arnaucube
/
asmt
mirror of https://github.com/arnaucube/asmt.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

full refactor 

- include the db package from aergo to this repository
- move sparse merkle tree code to smt directory
- add a new root package for being compatible with vocdoni/censusTree

Signed-off-by: p4u <pau@dabax.net>
master
p4u 3 years ago
parent
c679bcb817
commit
d855f4fb14
24 changed files with 3934 additions and 42 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +420
    -0
      censustree.go
  2. +90
    -0
      censustree_test.go
  3. +87
    -0
      db/db.go
  4. +276
    -0
      db/db_test.go
  5. +322
    -0
      db/leveldb.go
  6. +26
    -0
      db/logger.go
  7. +384
    -0
      db/memorydb.go
  8. +443
    -0
      db/perf_test.go
  9. +60
    -0
      db/types.go
  10. +5
    -1
      go.mod
  11. +1805
    -25
      go.sum
  12. +0
    -0
      smt/README.md
  13. +0
    -0
      smt/hash.go
  14. +0
    -0
      smt/pictures/batch.png
  15. +0
    -0
      smt/pictures/deleted.png
  16. +0
    -0
      smt/pictures/mod.png
  17. +0
    -0
      smt/pictures/smt.png
  18. +2
    -2
      smt/trie.go
  19. +1
    -1
      smt/trie_cache.go
  20. +0
    -0
      smt/trie_merkle_proof.go
  21. +0
    -0
      smt/trie_revert.go
  22. +12
    -12
      smt/trie_test.go
  23. +1
    -1
      smt/trie_tools.go
  24. +0
    -0
      smt/util.go

+ 420
- 0
censustree.go
View File

@ -0,0 +1,420 @@
package asmtree
import (
"bytes"
"fmt"
"path"
"sync/atomic"
"time"
"go.vocdoni.io/dvote/censustree"
"go.vocdoni.io/dvote/log"
"git.sr.ht/~sircmpwn/go-bare"
"github.com/p4u/asmt/db"
asmt "github.com/p4u/asmt/smt"
)
// We use go-bare for export/import the trie. In order to support
// big census (up to 8 Million entries) we need to increase the maximums.
const bareMaxArrayLength uint64 = 1024 * 1014 * 8 // 8 Million
const bareMaxUnmarshalBytes uint64 = 1024 * 1024 * 200 // 200 MiB
type Tree struct {
Tree *asmt.Trie
db db.DB
public uint32
lastAccessUnix int64 // a unix timestamp, used via sync/atomic
size uint64
snapshotRoot []byte // if not nil, this trie is considered an inmutable snapshot
snapshotSize uint64
}
type Proof struct {
Bitmap []byte
Length int
Siblings [][]byte
Value []byte
}
type exportElement struct {
Key []byte `bare:"key"`
Value []byte `bare:"value"`
}
type exportData struct {
Elements []exportElement `bare:"elements"`
}
const (
MaxKeySize = 256
MaxValueSize = 256
dbRootPrefix = "this is the last root for the SMT tree"
)
// NewTree initializes a new AergoSMT tree following the censustree.Tree interface specification.
func NewTree(name, storageDir string) (censustree.Tree, error) {
tr := &Tree{}
err := tr.Init(name, storageDir)
return tr, err
}
// newTree opens or creates a merkle tree under the given storage.
func newTree(name, storageDir string) (*asmt.Trie, db.DB, error) {
dir := path.Join(storageDir, name)
log.Debugf("creating new tree on %s", dir)
d := db.NewDB(db.LevelImpl, dir)
root := d.Get([]byte(dbRootPrefix))
tr := asmt.NewTrie(root, asmt.Hasher, d)
if root != nil {
if err := tr.LoadCache(root); err != nil {
return nil, nil, err
}
}
return tr, d, nil
}
// Init initializes a new asmt tree
func (t *Tree) Init(name, storageDir string) error {
var err error
t.Tree, t.db, err = newTree(name, storageDir)
t.updateAccessTime()
t.size = 0
return err
}
func (t *Tree) MaxKeySize() int {
return MaxKeySize
}
// LastAccess returns the last time the Tree was accessed, in the form of a unix
// timestamp.
func (t *Tree) LastAccess() int64 {
return atomic.LoadInt64(&t.lastAccessUnix)
}
func (t *Tree) updateAccessTime() {
atomic.StoreInt64(&t.lastAccessUnix, time.Now().Unix())
}
// Publish makes a merkle tree available for queries.
// Application layer should check IsPublish() before considering the Tree available.
func (t *Tree) Publish() {
atomic.StoreUint32(&t.public, 1)
}
// UnPublish makes a merkle tree not available for queries
func (t *Tree) UnPublish() {
atomic.StoreUint32(&t.public, 0)
}
// IsPublic returns true if the tree is available
func (t *Tree) IsPublic() bool {
return atomic.LoadUint32(&t.public) == 1
}
// Commit saves permanently the tree on disk
func (t *Tree) Commit() error {
if t.snapshotRoot != nil {
return fmt.Errorf("cannot commit to a snapshot trie")
}
err := t.Tree.Commit()
if err != nil {
return err
}
t.db.Set([]byte(dbRootPrefix), t.Root())
return nil
}
// Add adds a new claim to the merkle tree
// A claim is composed of two parts: index and value
// 1.index is mandatory, the data will be used for indexing the claim into to merkle tree
// 2.value is optional, the data will not affect the indexing
func (t *Tree) Add(index, value []byte) error {
t.updateAccessTime()
if t.snapshotRoot != nil {
return fmt.Errorf("cannot add to a snapshot trie")
}
if len(index) < 4 {
return fmt.Errorf("index too small (%d), minimum size is 4 bytes", len(index))
}
if len(value) > MaxValueSize {
return fmt.Errorf("index or value claim data too big")
}
_, err := t.Tree.Update([][]byte{asmt.Hasher(index)}, [][]byte{asmt.Hasher(value)})
if err != nil {
return err
}
atomic.StoreUint64(&t.size, 0) // TBD: improve this
return t.Commit()
}
// AddBatch adds a list of indexes and values.
// The commit to disk is executed only once.
// The values slince could be empty or as long as indexes.
func (t *Tree) AddBatch(indexes, values [][]byte) ([]int, error) {
var wrongIndexes []int
t.updateAccessTime()
if t.snapshotRoot != nil {
return wrongIndexes, fmt.Errorf("cannot add to a snapshot trie")
}
if len(values) > 0 && len(indexes) != len(values) {
return wrongIndexes, fmt.Errorf("indexes and values have different size")
}
var hashedIndexes [][]byte
var hashedValues [][]byte
var value []byte
for i, key := range indexes {
if len(key) < 4 {
wrongIndexes = append(wrongIndexes, i)
continue
}
value = nil
if len(values) > 0 {
if len(values[i]) > MaxValueSize {
wrongIndexes = append(wrongIndexes, i)
continue
}
value = values[i]
}
hashedIndexes = append(hashedIndexes, asmt.Hasher(key))
hashedValues = append(hashedValues, asmt.Hasher(value))
}
_, err := t.Tree.Update(hashedIndexes, hashedValues)
if err != nil {
return wrongIndexes, err
}
atomic.StoreUint64(&t.size, 0) // TBD: improve this
return wrongIndexes, t.Commit()
}
// Get returns the value of a key
func (t *Tree) Get(key []byte) []byte { // Do something with error
var value []byte
if t.snapshotRoot != nil {
value, _ = t.Tree.GetWithRoot(key, t.snapshotRoot)
} else {
value, _ = t.Tree.Get(key)
}
return value
}
// GenProof generates a merkle tree proof that can be later used on CheckProof() to validate it.
func (t *Tree) GenProof(index, value []byte) ([]byte, error) {
t.updateAccessTime()
var err error
var ap [][]byte
var pvalue []byte
var bitmap []byte
var length int
if t.snapshotRoot != nil {
bitmap, ap, length, _, _, pvalue, err = t.Tree.MerkleProofCompressedR(
asmt.Hasher(index),
t.snapshotRoot)
if err != nil {
return nil, err
}
} else {
bitmap, ap, length, _, _, pvalue, err = t.Tree.MerkleProofCompressed(
asmt.Hasher(index))
if err != nil {
return nil, err
}
}
//if !included {
// return nil, fmt.Errorf("not included")
//}
if !bytes.Equal(pvalue, asmt.Hasher(value)) {
return nil, fmt.Errorf("incorrect value on genProof")
}
return bare.Marshal(&Proof{Bitmap: bitmap, Length: length, Siblings: ap, Value: pvalue})
}
// CheckProof validates a merkle proof and its data.
func (t *Tree) CheckProof(index, value, root, mproof []byte) (bool, error) {
t.updateAccessTime()
p := Proof{}
if err := bare.Unmarshal(mproof, &p); err != nil {
return false, err
}
if !bytes.Equal(p.Value, asmt.Hasher(value)) {
return false, fmt.Errorf("values mismatch %x != %x", p.Value, asmt.Hasher(value))
}
if root != nil {
return t.Tree.VerifyInclusionWithRootC(
root,
p.Bitmap,
asmt.Hasher(index),
p.Value,
p.Siblings,
p.Length), nil
}
if t.snapshotRoot != nil {
return t.Tree.VerifyInclusionWithRootC(
t.snapshotRoot,
p.Bitmap,
asmt.Hasher(index),
p.Value,
p.Siblings,
p.Length), nil
}
return t.Tree.VerifyInclusionC(
p.Bitmap,
asmt.Hasher(index),
p.Value,
p.Siblings,
p.Length), nil
}
// Root returns the current root hash of the merkle tree
func (t *Tree) Root() []byte {
t.updateAccessTime()
if t.snapshotRoot != nil {
return t.snapshotRoot
}
return t.Tree.Root
}
// Dump returns the whole merkle tree serialized in a format that can be used on Import.
// Byte seralization is performed using bare message protocol, it is a 40% size win over JSON.
func (t *Tree) Dump(root []byte) ([]byte, error) {
t.updateAccessTime()
if root == nil && t.snapshotRoot != nil {
root = t.snapshotRoot
}
dump := exportData{}
t.iterateWithRoot(root, nil, func(k, v []byte) bool {
ee := exportElement{Key: make([]byte, len(k)), Value: make([]byte, len(v))}
// Copy elements since it's not safe to hold on to the []byte values from Iterate
copy(ee.Key, k[:])
copy(ee.Value, v[:])
dump.Elements = append(dump.Elements, ee)
return false
})
bare.MaxArrayLength(bareMaxArrayLength)
bare.MaxUnmarshalBytes(bareMaxUnmarshalBytes)
return bare.Marshal(&dump)
}
// String returns a human readable representation of the tree.
func (t *Tree) String() string {
s := bytes.Buffer{}
t.iterate(t.snapshotRoot, func(k, v []byte) bool {
s.WriteString(fmt.Sprintf("%x => %x\n", k, v))
return false
})
return s.String()
}
// Size returns the number of leaf nodes on the merkle tree.
// TO-DO: root is currently ignored
func (t *Tree) Size(root []byte) (int64, error) {
if t.snapshotRoot != nil {
return int64(t.snapshotSize), nil
}
return int64(t.count()), nil
}
// DumpPlain returns the entire list of added claims for a specific root hash.
// First return parametre are the indexes and second the values.
// If root is not specified, the last one is used.
func (t *Tree) DumpPlain(root []byte) ([][]byte, [][]byte, error) {
var indexes, values [][]byte
var err error
t.updateAccessTime()
t.iterateWithRoot(root, nil, func(k, v []byte) bool {
indexes = append(indexes, k)
values = append(values, v)
return false
})
return indexes, values, err
}
// ImportDump imports a partial or whole tree previously exported with Dump()
func (t *Tree) ImportDump(data []byte) error {
t.updateAccessTime()
if t.snapshotRoot != nil {
return fmt.Errorf("cannot import to a snapshot")
}
census := new(exportData)
bare.MaxArrayLength(bareMaxArrayLength)
bare.MaxUnmarshalBytes(bareMaxUnmarshalBytes)
if err := bare.Unmarshal(data, census); err != nil {
return fmt.Errorf("importdump cannot unmarshal data: %w", err)
}
keys := [][]byte{}
values := [][]byte{}
for _, ee := range census.Elements {
keys = append(keys, ee.Key)
values = append(values, ee.Value)
}
_, err := t.Tree.Update(keys, values)
if err != nil {
return err
}
atomic.StoreUint64(&t.size, 0) // TBD: improve this
return t.Commit()
}
// Snapshot returns a Tree instance of a exiting merkle root.
// A Snapshot cannot be modified.
func (t *Tree) Snapshot(root []byte) (censustree.Tree, error) {
exist, err := t.HashExists(root)
if err != nil {
return nil, err
}
if !exist {
return nil, fmt.Errorf("root %x does not exist, cannot build snapshot", root)
}
return &Tree{Tree: t.Tree, public: t.public, snapshotRoot: root, snapshotSize: t.count()}, nil
}
func (t *Tree) Close() error {
t.db.Close()
return nil
}
// HashExists checks if a hash exists as a node in the merkle tree
func (t *Tree) HashExists(hash []byte) (bool, error) {
t.updateAccessTime()
return t.Tree.TrieRootExists(hash), nil
}
func (t *Tree) count() uint64 {
if v := atomic.LoadUint64(&t.size); v != 0 {
return v
}
counter := uint64(0)
if err := t.Tree.Walk(t.snapshotRoot, func(*asmt.WalkResult) int32 {
counter++
return 0
}); err != nil {
return 0
}
atomic.StoreUint64(&t.size, counter)
return counter
}
func (t *Tree) iterate(prefix []byte, callback func(key, value []byte) bool) {
t.Tree.Walk(t.snapshotRoot, func(v *asmt.WalkResult) int32 {
if callback(v.Key, v.Value) {
return 1
} else {
return 0
}
})
}
func (t *Tree) iterateWithRoot(root, prefix []byte, callback func(key, value []byte) bool) {
t.Tree.Walk(root, func(v *asmt.WalkResult) int32 {
if callback(v.Key, v.Value) {
return 1
} else {
return 0
}
})
}

+ 90
- 0
censustree_test.go
View File

@ -0,0 +1,90 @@
package asmtree
import (
"bytes"
"fmt"
"testing"
)
func TestTree(t *testing.T) {
censusSize := 1000
storage := t.TempDir()
tr1 := &Tree{}
err := tr1.Init("test1", storage)
if err != nil {
t.Fatal(err)
}
for i := 0; i < censusSize; i++ {
if err = tr1.Add([]byte(fmt.Sprintf("number %d", i)),
[]byte(fmt.Sprintf("number %d value", i))); err != nil {
t.Fatal(err)
}
}
root1 := tr1.Root()
data, err := tr1.Dump(root1)
if err != nil {
t.Fatal(err)
}
t.Logf("dumped data size is: %d bytes", len(data))
tr2 := &Tree{}
err = tr2.Init("test2", storage)
if err != nil {
t.Fatal(err)
}
if err = tr2.ImportDump(data); err != nil {
t.Fatal(err)
}
root2 := tr2.Root()
if !bytes.Equal(root1, root2) {
t.Errorf("roots are different but they should be equal (%x != %x)", root1, root2)
}
// Try closing the storage and creating the tree again
tr2.Close()
err = tr2.Init("test2", storage)
if err != nil {
t.Fatal(err)
}
// Get the size
s, err := tr2.Size(nil)
if err != nil {
t.Errorf("cannot get te size of the tree after reopen: (%s)", err)
}
if s != int64(censusSize) {
t.Errorf("Size is wrong (have %d, expexted %d)", s, censusSize)
}
// Check Root is still the same
if !bytes.Equal(tr2.Root(), root2) {
t.Fatalf("after closing and opening the tree, the root is different")
}
// Generate a proof on tr1 and check validity on snapshot and tr2
proof1, err := tr1.GenProof([]byte("number 5"), []byte("number 5 value"))
if err != nil {
t.Error(err)
}
t.Logf("Proof Length: %d", len(proof1))
tr1s, err := tr1.Snapshot(root1)
if err != nil {
t.Fatal(err)
}
valid, err := tr1s.CheckProof([]byte("number 5"), []byte("number 5 value"), root1, proof1)
if err != nil {
t.Error(err)
}
if !valid {
t.Errorf("proof is invalid on snapshot")
}
valid, err = tr2.CheckProof([]byte("number 5"), []byte("number 5 value"), nil, proof1)
if err != nil {
t.Error(err)
}
if !valid {
t.Errorf("proof is invalid on tree2")
}
}

+ 87
- 0
db/db.go
View File

@ -0,0 +1,87 @@
/**
* @file
* @copyright defined in aergo/LICENSE.txt
*/
/*
Package db is an wrapper of key-value database implementations. Currently, this supports badgerdb (https://github.com/dgraph-io/badger).
Basic Usage
You can create database using a newdb func like this
database := NewDB(BadgerImpl, "./test")
A first argument is a backend db type to use, and a second is a root directory to store db files.
After creating db, you can write, read or delete single key-value using funcs in DB interface.
// write data
database.Set([]byte("key"), []byte("val"))
// read data
read := Get([]byte("key"))
// delete data
database.Delete([]byte("key"))
Transaction
A Transaction is a bulk set of operations to ensure atomic success or fail.
// create a new transaction
tx := database.NewTX(true)
// reserve writing
tx.Set([]byte("keyA"), []byte("valA"))
tx.Set([]byte("keyB"), []byte("valB"))
// Get will return a value reserved to write in this transaction
mustBeValA := tx.Get([]byte("keyA"))
// Perform writing
tx.Commit()
If you want to cancel and discard operations in tx, then you must call Discard() func to prevent a memory leack
// If you create a tx, but do not commit, than you have to call this
tx.Discard()
Iterator
An iteractor provides a way to get all keys sequentially.
// create an iterator that covers all range
for iter := database.Iterator(nil, nil); iter.Valid(); iter.Next() {
// print each key-value pair
fmt.Printf("%s = %s", string(iter.Key()), string(iter.Value()))
}
You can find more detail usages at a db_test.go file
*/
package db
import (
"fmt"
"github.com/aergoio/aergo-lib/log"
)
var dbImpls = map[ImplType]dbConstructor{}
var logger *extendedLog
func registorDBConstructor(dbimpl ImplType, constructor dbConstructor) {
dbImpls[dbimpl] = constructor
}
// NewDB creates new database or load existing database in the directory
func NewDB(dbimpltype ImplType, dir string) DB {
logger = &extendedLog{Logger: log.NewLogger("db")}
db, err := dbImpls[dbimpltype](dir)
if err != nil {
panic(fmt.Sprintf("Fail to Create New DB: %v", err))
}
return db
}
func convNilToBytes(byteArray []byte) []byte {
if byteArray == nil {
return []byte{}
}
return byteArray
}

+ 276
- 0
db/db_test.go
View File

@ -0,0 +1,276 @@
/**
* @file
* @copyright defined in aergo/LICENSE.txt
*/
package db
import (
"fmt"
"io/ioutil"
"log"
"os"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
)
const (
tmpDbTestKey1 = "tempkey1"
tmpDbTestKey2 = "tempkey2"
tmpDbTestStrVal1 = "val1"
tmpDbTestStrVal2 = "val2"
tmpDbTestIntVal1 = 1
tmpDbTestIntVal2 = 2
)
func createTmpDB(key ImplType) (dir string, db DB) {
dir, err := ioutil.TempDir("", string(key))
if err != nil {
log.Fatal(err)
}
db = NewDB(key, dir)
return
}
func setInitData(db DB) {
tx := db.NewTx()
tx.Set([]byte("1"), []byte("1"))
tx.Set([]byte("2"), []byte("2"))
tx.Set([]byte("3"), []byte("3"))
tx.Set([]byte("4"), []byte("4"))
tx.Set([]byte("5"), []byte("5"))
tx.Set([]byte("6"), []byte("6"))
tx.Set([]byte("7"), []byte("7"))
tx.Commit()
}
func TestGetSetDeleteExist(t *testing.T) {
// for each db implementation
for key := range dbImpls {
dir, db := createTmpDB(key)
// initial value of empty key must be empty byte
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
// set value
db.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
// check value set
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
assert.True(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
// delete value
db.Delete([]byte(tmpDbTestKey1))
// value must be erased
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
db.Close()
os.RemoveAll(dir)
}
}
func TestTransactionSet(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
// create a new writable tx
tx := db.NewTx()
// set the value in the tx
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
// the value will not visible at a db
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
tx.Commit()
// after commit, the value visible from the db
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
db.Close()
os.RemoveAll(dir)
}
}
func TestTransactionDiscard(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
// create a new writable tx
tx := db.NewTx()
// discard test
tx = db.NewTx()
// set the value in the tx
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal2))
// discard tx
tx.Discard()
assert.Panics(t, func() { tx.Commit() }, "commit after discard is not allowed")
// after discard, the value must be reset at the db
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
db.Close()
os.RemoveAll(dir)
}
}
func TestTransactionDelete(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
// create a new writable tx
tx := db.NewTx()
// set the value in the tx
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
// delete the value in the tx
tx.Delete([]byte(tmpDbTestKey1))
tx.Commit()
// after commit, chekc the value from the db
assert.Equal(t, "", string(db.Get([]byte(tmpDbTestKey1))), db.Type())
db.Close()
os.RemoveAll(dir)
}
}
func TestTransactionCommitTwice(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
// create a new writable tx
tx := db.NewTx()
// a first commit will success
tx.Commit()
// a second commit will cause panic
assert.Panics(t, func() { tx.Commit() })
db.Close()
os.RemoveAll(dir)
}
}
func TestBulk(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
// create a new Bulk instance
bulk := db.NewBulk()
// set the huge number of value in the bulk
for i := 0; i < 1000000; i++ {
bulk.Set([]byte(fmt.Sprintf("key%d", i)),
[]byte(tmpDbTestStrVal1))
}
bulk.Flush()
// after commit, the value visible from the db
for i := 0; i < 1000000; i++ {
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(fmt.Sprintf("key%d", i)))), db.Type())
}
db.Close()
os.RemoveAll(dir)
}
}
func TestIter(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
setInitData(db)
i := 1
for iter := db.Iterator(nil, nil); iter.Valid(); iter.Next() {
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
i++
}
db.Close()
os.RemoveAll(dir)
}
}
func TestRangeIter(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
setInitData(db)
// test iteration 2 -> 5
i := 2
for iter := db.Iterator([]byte("2"), []byte("5")); iter.Valid(); iter.Next() {
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
i++
}
assert.EqualValues(t, i, 5)
// nil sames with []byte("0")
// test iteration 0 -> 5
i = 1
for iter := db.Iterator(nil, []byte("5")); iter.Valid(); iter.Next() {
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
i++
}
assert.EqualValues(t, i, 5)
db.Close()
os.RemoveAll(dir)
}
}
func TestReverseIter(t *testing.T) {
for key := range dbImpls {
dir, db := createTmpDB(key)
setInitData(db)
// test reverse iteration 5 <- 2
i := 5
for iter := db.Iterator([]byte("5"), []byte("2")); iter.Valid(); iter.Next() {
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
i--
}
assert.EqualValues(t, i, 2)
// nil sames with []byte("0")
// test reverse iteration 5 -> 0
i = 5
for iter := db.Iterator([]byte("5"), nil); iter.Valid(); iter.Next() {
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
i--
}
assert.EqualValues(t, i, 0)
db.Close()
os.RemoveAll(dir)
}
}

+ 322
- 0
db/leveldb.go
View File

@ -0,0 +1,322 @@
/**
* @file
* @copyright defined in aergo/LICENSE.txt
*/
package db
import (
"bytes"
"fmt"
"path/filepath"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
)
// This function is always called first
func init() {
dbConstructor := func(dir string) (DB, error) {
return newLevelDB(dir)
}
registorDBConstructor(LevelImpl, dbConstructor)
}
func newLevelDB(dir string) (DB, error) {
dbPath := filepath.Join(dir, "data.db")
db, err := leveldb.OpenFile(dbPath, nil)
if err != nil {
return nil, err
}
database := &levelDB{
db: db,
}
return database, nil
}
//=========================================================
// DB Implementation
//=========================================================
// Enforce database and transaction implements interfaces
var _ DB = (*levelDB)(nil)
type levelDB struct {
db *leveldb.DB
}
func (db *levelDB) Type() string {
return "leveldb"
}
func (db *levelDB) Set(key, value []byte) {
key = convNilToBytes(key)
value = convNilToBytes(value)
err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
if err != nil {
panic(fmt.Sprintf("Database Error: %v", err))
}
}
func (db *levelDB) Delete(key []byte) {
key = convNilToBytes(key)
err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
if err != nil {
panic(fmt.Sprintf("Database Error: %v", err))
}
}
func (db *levelDB) Get(key []byte) []byte {
key = convNilToBytes(key)
res, err := db.db.Get(key, nil)
if err != nil {
if err == errors.ErrNotFound {
return []byte{}
}
panic(fmt.Sprintf("Database Error: %v", err))
}
return res
}
func (db *levelDB) Exist(key []byte) bool {
res, _ := db.db.Has(key, nil)
return res
}
func (db *levelDB) Close() {
db.db.Close()
}
func (db *levelDB) NewTx() Transaction {
batch := new(leveldb.Batch)
return &levelTransaction{db, batch, false, false}
}
func (db *levelDB) NewBulk() Bulk {
batch := new(leveldb.Batch)
return &levelBulk{db, batch, false, false}
}
//=========================================================
// Transaction Implementation
//=========================================================
type levelTransaction struct {
db *levelDB
tx *leveldb.Batch
isDiscard bool
isCommit bool
}
/*
func (transaction *levelTransaction) Get(key []byte) []byte {
panic(fmt.Sprintf("DO not support"))
}
*/
func (transaction *levelTransaction) Set(key, value []byte) {
transaction.tx.Put(key, value)
}
func (transaction *levelTransaction) Delete(key []byte) {
transaction.tx.Delete(key)
}
func (transaction *levelTransaction) Commit() {
if transaction.isDiscard {
panic("Commit after dicard tx is not allowed")
} else if transaction.isCommit {
panic("Commit occures two times")
}
err := transaction.db.db.Write(transaction.tx, &opt.WriteOptions{Sync: true})
if err != nil {
panic(fmt.Sprintf("Database Error: %v", err))
}
transaction.isCommit = true
}
func (transaction *levelTransaction) Discard() {
transaction.isDiscard = true
}
//=========================================================
// Bulk Implementation
//=========================================================
type levelBulk struct {
db *levelDB
tx *leveldb.Batch
isDiscard bool
isCommit bool
}
func (bulk *levelBulk) Set(key, value []byte) {
bulk.tx.Put(key, value)
}
func (bulk *levelBulk) Delete(key []byte) {
bulk.tx.Delete(key)
}
func (bulk *levelBulk) Flush() {
// do the same behavior that a transaction commit does
// db.write internally will handle large transaction
if bulk.isDiscard {
panic("Commit after dicard tx is not allowed")
} else if bulk.isCommit {
panic("Commit occures two times")
}
err := bulk.db.db.Write(bulk.tx, &opt.WriteOptions{Sync: true})
if err != nil {
panic(fmt.Sprintf("Database Error: %v", err))
}
bulk.isCommit = true
}
func (bulk *levelBulk) DiscardLast() {
bulk.isDiscard = true
}
//=========================================================
// Iterator Implementation
//=========================================================
type levelIterator struct {
start []byte
end []byte
reverse bool
iter iterator.Iterator
isInvalid bool
}
func (db *levelDB) Iterator(start, end []byte) Iterator {
var reverse bool
// if end is bigger then start, then reverse order
if bytes.Compare(start, end) == 1 {
reverse = true
} else {
reverse = false
}
iter := db.db.NewIterator(nil, nil)
if reverse {
if start == nil {
iter.Last()
} else {
valid := iter.Seek(start)
if valid {
soakey := iter.Key()
if bytes.Compare(start, soakey) < 0 {
iter.Prev()
}
} else {
iter.Last()
}
}
} else {
if start == nil {
iter.First()
} else {
iter.Seek(start)
}
}
return &levelIterator{
iter: iter,
start: start,
end: end,
reverse: reverse,
isInvalid: false,
}
}
func (iter *levelIterator) Next() {
if iter.Valid() {
if iter.reverse {
iter.iter.Prev()
} else {
iter.iter.Next()
}
} else {
panic("Iterator is Invalid")
}
}
func (iter *levelIterator) Valid() bool {
// Once invalid, forever invalid.
if iter.isInvalid {
return false
}
// Panic on DB error. No way to recover.
if err := iter.iter.Error(); err != nil {
panic(err)
}
// If source is invalid, invalid.
if !iter.iter.Valid() {
iter.isInvalid = true
return false
}
// If key is end or past it, invalid.
var end = iter.end
var key = iter.iter.Key()
if iter.reverse {
if end != nil && bytes.Compare(key, end) <= 0 {
iter.isInvalid = true
return false
}
} else {
if end != nil && bytes.Compare(end, key) <= 0 {
iter.isInvalid = true
return false
}
}
// Valid
return true
}
func (iter *levelIterator) Key() (key []byte) {
if !iter.Valid() {
panic("Iterator is invalid")
} else if err := iter.iter.Error(); err != nil {
panic(err)
}
originalKey := iter.iter.Key()
key = make([]byte, len(originalKey))
copy(key, originalKey)
return key
}
func (iter *levelIterator) Value() (value []byte) {
if !iter.Valid() {
panic("Iterator is invalid")
} else if err := iter.iter.Error(); err != nil {
panic(err)
}
originalValue := iter.iter.Value()
value = make([]byte, len(originalValue))
copy(value, originalValue)
return value
}

+ 26
- 0
db/logger.go
View File

@ -0,0 +1,26 @@
package db
import (
"github.com/aergoio/aergo-lib/log"
)
type extendedLog struct {
*log.Logger
}
func (l *extendedLog) Errorf(f string, v ...interface{}) {
l.Error().Msgf(f, v...)
}
func (l *extendedLog) Warningf(f string, v ...interface{}) {
l.Warn().Msgf(f, v...)
}
func (l *extendedLog) Infof(f string, v ...interface{}) {
// reduce info to debug level because infos at badgerdb are too detail
l.Debug().Msgf(f, v...) // INFO -> DEBUG
}
func (l *extendedLog) Debugf(f string, v ...interface{}) {
l.Debug().Msgf(f, v...)
}

+ 384
- 0
db/memorydb.go
View File

@ -0,0 +1,384 @@
/**
* @file
* @copyright defined in aergo/LICENSE.txt
*/
package db
import (
"bytes"
"container/list"
"encoding/gob"
"os"
"path"
"sort"
"sync"
)
// This function is always called first
func init() {
dbConstructor := func(dir string) (DB, error) {
return newMemoryDB(dir)
}
registorDBConstructor(MemoryImpl, dbConstructor)
}
func newMemoryDB(dir string) (DB, error) {
var db map[string][]byte
filePath := path.Join(dir, "database")
file, err := os.Open(filePath)
if err == nil {
decoder := gob.NewDecoder(file) //
err = decoder.Decode(&db)
if err != nil {
return nil, err
}
}
file.Close()
if db == nil {
db = make(map[string][]byte)
}
database := &memorydb{
db: db,
dir: filePath,
}
return database, nil
}
//=========================================================
// DB Implementation
//=========================================================
// Enforce database and transaction implements interfaces
var _ DB = (*memorydb)(nil)
type memorydb struct {
lock sync.Mutex
db map[string][]byte
dir string
}
func (db *memorydb) Type() string {
return "memorydb"
}
func (db *memorydb) Set(key, value []byte) {
db.lock.Lock()
defer db.lock.Unlock()
key = convNilToBytes(key)
value = convNilToBytes(value)
db.db[string(key)] = value
}
func (db *memorydb) Delete(key []byte) {
db.lock.Lock()
defer db.lock.Unlock()
key = convNilToBytes(key)
delete(db.db, string(key))
}
func (db *memorydb) Get(key []byte) []byte {
db.lock.Lock()
defer db.lock.Unlock()
key = convNilToBytes(key)
return db.db[string(key)]
}
func (db *memorydb) Exist(key []byte) bool {
db.lock.Lock()
defer db.lock.Unlock()
key = convNilToBytes(key)
_, ok := db.db[string(key)]
return ok
}
func (db *memorydb) Close() {
db.lock.Lock()
defer db.lock.Unlock()
file, err := os.OpenFile(db.dir, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err == nil {
encoder := gob.NewEncoder(file)
encoder.Encode(db.db)
}
file.Close()
}
func (db *memorydb) NewTx() Transaction {
return &memoryTransaction{
db: db,
opList: list.New(),
isDiscard: false,
isCommit: false,
}
}
func (db *memorydb) NewBulk() Bulk {
return &memoryBulk{
db: db,
opList: list.New(),
isDiscard: false,
isCommit: false,
}
}
//=========================================================
// Transaction Implementation
//=========================================================
type memoryTransaction struct {
txLock sync.Mutex
db *memorydb
opList *list.List
isDiscard bool
isCommit bool
}
type txOp struct {
isSet bool
key []byte
value []byte
}
func (transaction *memoryTransaction) Set(key, value []byte) {
transaction.txLock.Lock()
defer transaction.txLock.Unlock()
key = convNilToBytes(key)
value = convNilToBytes(value)
transaction.opList.PushBack(&txOp{true, key, value})
}
func (transaction *memoryTransaction) Delete(key []byte) {
transaction.txLock.Lock()
defer transaction.txLock.Unlock()
key = convNilToBytes(key)
transaction.opList.PushBack(&txOp{false, key, nil})
}
func (transaction *memoryTransaction) Commit() {
transaction.txLock.Lock()
defer transaction.txLock.Unlock()
if transaction.isDiscard {
panic("Commit after dicard tx is not allowed")
} else if transaction.isCommit {
panic("Commit occures two times")
}
db := transaction.db
db.lock.Lock()
defer db.lock.Unlock()
for e := transaction.opList.Front(); e != nil; e = e.Next() {
op := e.Value.(*txOp)
if op.isSet {
db.db[string(op.key)] = op.value
} else {
delete(db.db, string(op.key))
}
}
transaction.isCommit = true
}
func (transaction *memoryTransaction) Discard() {
transaction.txLock.Lock()
defer transaction.txLock.Unlock()
transaction.isDiscard = true
}
//=========================================================
// Bulk Implementation
//=========================================================
type memoryBulk struct {
txLock sync.Mutex
db *memorydb
opList *list.List
isDiscard bool
isCommit bool
}
func (bulk *memoryBulk) Set(key, value []byte) {
bulk.txLock.Lock()
defer bulk.txLock.Unlock()
key = convNilToBytes(key)
value = convNilToBytes(value)
bulk.opList.PushBack(&txOp{true, key, value})
}
func (bulk *memoryBulk) Delete(key []byte) {
bulk.txLock.Lock()
defer bulk.txLock.Unlock()
key = convNilToBytes(key)
bulk.opList.PushBack(&txOp{false, key, nil})
}
func (bulk *memoryBulk) Flush() {
bulk.txLock.Lock()
defer bulk.txLock.Unlock()
if bulk.isDiscard {
panic("Commit after dicard tx is not allowed")
} else if bulk.isCommit {
panic("Commit occures two times")
}
db := bulk.db
db.lock.Lock()
defer db.lock.Unlock()
for e := bulk.opList.Front(); e != nil; e = e.Next() {
op := e.Value.(*txOp)
if op.isSet {
db.db[string(op.key)] = op.value
} else {
delete(db.db, string(op.key))
}
}
bulk.isCommit = true
}
func (bulk *memoryBulk) DiscardLast() {
bulk.txLock.Lock()
defer bulk.txLock.Unlock()
bulk.isDiscard = true
}
//=========================================================
// Iterator Implementation
//=========================================================
type memoryIterator struct {
start []byte
end []byte
reverse bool
keys []string
isInvalid bool
cursor int
db *memorydb
}
func isKeyInRange(key []byte, start []byte, end []byte, reverse bool) bool {
if reverse {
if start != nil && bytes.Compare(start, key) < 0 {
return false
}
if end != nil && bytes.Compare(key, end) <= 0 {
return false
}
return true
}
if bytes.Compare(key, start) < 0 {
return false
}
if end != nil && bytes.Compare(end, key) <= 0 {
return false
}
return true
}
func (db *memorydb) Iterator(start, end []byte) Iterator {
db.lock.Lock()
defer db.lock.Unlock()
var reverse bool
// if end is bigger then start, then reverse order
if bytes.Compare(start, end) == 1 {
reverse = true
} else {
reverse = false
}
var keys sort.StringSlice
for key := range db.db {
if isKeyInRange([]byte(key), start, end, reverse) {
keys = append(keys, key)
}
}
if reverse {
sort.Sort(sort.Reverse(keys))
} else {
sort.Strings(keys)
}
return &memoryIterator{
start: start,
end: end,
reverse: reverse,
isInvalid: false,
keys: keys,
cursor: 0,
db: db,
}
}
func (iter *memoryIterator) Next() {
if !iter.Valid() {
panic("Iterator is Invalid")
}
iter.cursor++
}
func (iter *memoryIterator) Valid() bool {
// Once invalid, forever invalid.
if iter.isInvalid {
return false
}
return 0 <= iter.cursor && iter.cursor < len(iter.keys)
}
func (iter *memoryIterator) Key() (key []byte) {
if !iter.Valid() {
panic("Iterator is Invalid")
}
return []byte(iter.keys[iter.cursor])
}
func (iter *memoryIterator) Value() (value []byte) {
if !iter.Valid() {
panic("Iterator is Invalid")
}
key := []byte(iter.keys[iter.cursor])
return iter.db.Get(key)
}

+ 443
- 0
db/perf_test.go
View File

@ -0,0 +1,443 @@
package db
import (
"bytes"
"container/list"
"encoding/binary"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"reflect"
"testing"
"time"
"github.com/guptarohit/asciigraph"
)
type testKeyType int64
// Simple execution cmd
// go test -run=XXX -bench=. -benchmem -cpuprofile=cpu.out -memprofile=mem.out -timeout 20m -benchtime=1m
var valueLen = 256
var testSetSize = int64(10000000)
var batchSize = 100
// parameters for drawing graph
var graphCountingPeriod = 10000
var graphWidth = 120
var graphHeigh = 20
var deviceType = "ssd"
func init() {
rand.Seed(time.Now().Unix())
}
func int64ToBytes(i testKeyType) []byte {
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, uint64(i))
return buf
}
func printGraph(statics list.List) {
if statics.Len() == 0 {
return
}
// convert list to slice
var staticsSlice = make([]float64, statics.Len())
i := 0
for e := statics.Front(); e != nil; e = e.Next() {
staticsSlice[i] = e.Value.(float64)
i++
}
graph := asciigraph.Plot(staticsSlice, asciigraph.Width(graphWidth), asciigraph.Height(graphHeigh))
fmt.Println(graph)
}
func dirSizeKB(path string) (int64, error) {
var size int64
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
if !info.IsDir() {
size += info.Size()
}
return err
})
return size / 1024, err
}
func BenchmarkRandomWR(b *testing.B) {
// generate a common random data set
internal := map[testKeyType][]byte{}
for i := 0; i < int(testSetSize); i++ {
// generate a random value
token := make([]byte, valueLen)
internal[testKeyType(i)] = token
}
for dbType, dbConstructors := range dbImpls {
// create db
dbName := string(dbType)
tmpDir, _ := ioutil.TempDir("", dbName)
defer os.RemoveAll(tmpDir)
dbInstance, _ := dbConstructors(tmpDir)
var numOfWrite int64
var idx testKeyType
var statics list.List
var startTime time.Time
fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
fmt.Println("device type: ssd")
// write only
b.Run(dbName+"-write", func(b *testing.B) {
var i int
for i = 0; i < b.N; i++ {
if i != 0 && i%graphCountingPeriod == 0 {
startTime = time.Now()
}
idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
rand.Read(internal[idx]) // generate a random data
dbInstance.Set(
int64ToBytes(testKeyType(idx)),
internal[idx],
)
if i != 0 && i%graphCountingPeriod == 0 {
endTime := time.Now().Sub(startTime).Seconds()
statics.PushBack(endTime)
}
}
numOfWrite += int64(i)
})
printGraph(statics)
statics.Init()
// read only
b.Run(dbName+"-read", func(b *testing.B) {
for i := 0; i < b.N; i++ {
if i != 0 && i%graphCountingPeriod == 0 {
startTime = time.Now()
}
idx = testKeyType((int64(rand.Int()) % testSetSize))
originalVal := internal[idx]
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
if len(retrievedVal) != 0 {
if len(retrievedVal) != valueLen {
b.Errorf("Expected length %X for %v, got %X",
valueLen, idx, len(retrievedVal))
break
} else if !bytes.Equal(retrievedVal, originalVal) {
b.Errorf("Expected %v for %v, got %v",
originalVal, idx, retrievedVal)
break
}
}
if i != 0 && i%graphCountingPeriod == 0 {
endTime := time.Now().Sub(startTime).Seconds()
statics.PushBack(endTime)
}
}
})
printGraph(statics)
statics.Init()
// write and read
b.Run(dbName+"-write-read", func(b *testing.B) {
var i int
for i = 0; i < b.N; i++ {
if i != 0 && i%graphCountingPeriod == 0 {
startTime = time.Now()
}
idx = testKeyType(int64(rand.Int()) % testSetSize) // pick a random key
rand.Read(internal[idx]) // generate a random data
dbInstance.Set(
int64ToBytes(testKeyType(idx)),
internal[idx],
)
originalVal := internal[idx]
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
if len(retrievedVal) != 0 {
if len(retrievedVal) != valueLen {
b.Errorf("Expected length %X for %v, got %X",
valueLen, idx, len(retrievedVal))
break
} else if !bytes.Equal(retrievedVal, originalVal) {
b.Errorf("Expected %v for %v, got %v",
originalVal, idx, retrievedVal)
break
}
}
if i != 0 && i%graphCountingPeriod == 0 {
endTime := time.Now().Sub(startTime).Seconds()
statics.PushBack(endTime)
}
}
numOfWrite += int64(i)
})
printGraph(statics)
statics.Init()
// close
dbInstance.Close()
size, err := dirSizeKB(tmpDir)
if err != nil {
fmt.Println(err)
} else {
fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
}
}
}
func BenchmarkRandomBatchWR(b *testing.B) {
// generate a common random data set
internal := map[testKeyType][]byte{}
for i := 0; i < int(testSetSize); i++ {
// generate a random value
token := make([]byte, valueLen)
internal[testKeyType(i)] = token
}
for dbType, dbConstructors := range dbImpls {
// create db
dbName := string(dbType)
tmpDir, _ := ioutil.TempDir("", dbName)
defer os.RemoveAll(tmpDir)
dbInstance, _ := dbConstructors(tmpDir)
var numOfWrite int64
var idx testKeyType
var statics list.List
var startTime time.Time
fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
fmt.Println("device type: " + deviceType)
fmt.Printf("batch size: %d\n", batchSize)
// write only
b.Run(dbName+"-batch-write", func(b *testing.B) {
var i int
for i = 0; i < b.N; i++ {
if i != 0 && i%graphCountingPeriod == 0 {
startTime = time.Now()
}
tx := dbInstance.NewTx()
for j := 0; j < batchSize; j++ {
idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
rand.Read(internal[idx]) // generate a random data
tx.Set(
int64ToBytes(testKeyType(idx)),
internal[idx],
)
}
tx.Commit()
if i != 0 && i%graphCountingPeriod == 0 {
endTime := time.Now().Sub(startTime).Seconds()
statics.PushBack(endTime)
}
}
numOfWrite += int64(i)
})
// print a graph
printGraph(statics)
statics.Init()
// read only
b.Run(dbName+"-read", func(b *testing.B) {
for i := 0; i < b.N; i++ {
if i != 0 && i%graphCountingPeriod == 0 {
startTime = time.Now()
}
idx = testKeyType((int64(rand.Int()) % testSetSize))
originalVal := internal[idx]
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
if len(retrievedVal) != 0 {
if len(retrievedVal) != valueLen {
b.Errorf("Expected length %X for %v, got %X",
valueLen, idx, len(retrievedVal))
break
} else if !bytes.Equal(retrievedVal, originalVal) {
b.Errorf("Expected %v for %v, got %v",
originalVal, idx, retrievedVal)
break
}
}
if i != 0 && i%graphCountingPeriod == 0 {
endTime := time.Now().Sub(startTime).Seconds()
statics.PushBack(endTime)
}
}
})
// close
dbInstance.Close()
printGraph(statics)
size, err := dirSizeKB(tmpDir)
if err != nil {
fmt.Println(err)
} else {
fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
}
}
}
/*
[Refined Test Result]
goos: windows
goarch: amd64
pkg: github.com/aergoio/aergo-lib/db
[badgerdb-ssd-single]
testset_size: 10000000
key_len: 8
val_len: 256
device type: ssd
BenchmarkRandomWR/badgerdb-write-12 30000000 29536 ns/op 4983 B/op 84 allocs/op
BenchmarkRandomWR/badgerdb-read-12 20000000 47739 ns/op 13154 B/op 60 allocs/op
BenchmarkRandomWR/badgerdb-write-read-12 20000000 41990 ns/op 10317 B/op 140 allocs/op
* Total size of badgerdb db: 14544473 kb, Size of 1 write: 286 byte
[leveldb-ssd-single]
testset_size: 10000000
key_len: 8
val_len: 256
device type: ssd
BenchmarkRandomWR/leveldb-write-12 1000000 1006445 ns/op 532 B/op 5 allocs/op
BenchmarkRandomWR/leveldb-read-12 50000000 18941 ns/op 1160 B/op 17 allocs/op
BenchmarkRandomWR/leveldb-write-read-12 1000000 986440 ns/op 1272 B/op 11 allocs/op
* Total size of leveldb db: 567386 kb, Size of 1 write: 250 byte
[badgerdb-ssd-batch100]
testset_size: 10000000
key_len: 8
val_len: 256
device type: ssd
batch size: 100
BenchmarkRandomBatchWR/badgerdb-batch-write-12 2000000 612737 ns/op 383751 B/op 4576 allocs/op
BenchmarkRandomBatchWR/badgerdb-read-12 20000000 53298 ns/op 22177 B/op 71 allocs/op
* Total size of badgerdb db: 74546912 kb, Size of 1 write: 25359 byte
[leveldb-ssd-batch100]
testset_size: 10000000
key_len: 8
val_len: 256
device type: ssd
batch size: 100
BenchmarkRandomBatchWR/leveldb-batch-write-12 300000 6778458 ns/op 127466 B/op 399 allocs/op
BenchmarkRandomBatchWR/leveldb-read-12 5000000 141176 ns/op 4547 B/op 34 allocs/op
* Total size of leveldb db: 2540271 kb, Size of 1 write: 8388 byte
[badgerdb-ssd-batch1000]
testset_size: 10000000
key_len: 8
val_len: 256
device type: ssd
batch size: 1000
BenchmarkRandomBatchWR1000/badgerdb-batch-write-12 200000 5507060 ns/op 3504344 B/op 42743 allocs/op
BenchmarkRandomBatchWR1000/badgerdb-read-12 20000000 52636 ns/op 21867 B/op 73 allocs/op
* Total size of badgerdb db: 51315516 kb, Size of 1 write: 250103 byte
[leveldb-ssd-batch100]
testset_size: 10000000
key_len: 8
val_len: 256
device type: ssd
batch size: 1000
BenchmarkRandomBatchWR1000/leveldb-batch-write-12 20000 70431430 ns/op 1081956 B/op 3906 allocs/op
BenchmarkRandomBatchWR1000/leveldb-read-12 5000000 138074 ns/op 4194 B/op 33 allocs/op
* Total size of leveldb db: 2529393 kb, Size of 1 write: 86046 byte
[badgerdb-hdd-single]
testset_size: 10000000
key_len: 8
val_len: 256
device type: hdd
BenchmarkHddRandomWR/badgerdb-write-12 30000000 27901 ns/op 4809 B/op 82 allocs/op
BenchmarkHddRandomWR/badgerdb-read-12 20000000 45765 ns/op 14321 B/op 68 allocs/op
BenchmarkHddRandomWR/badgerdb-write-read-12 20000000 39404 ns/op 8693 B/op 124 allocs/op
* Total size of badgerdb db: 14697136 kb, Size of 1 write: 289 byte
[leveldb-hdd-single]
testset_size: 10000000
key_len: 8
val_len: 256
device type: hdd
BenchmarkHddRandomWR/leveldb-write-12 50000 20975160 ns/op 551 B/op 4 allocs/op
BenchmarkHddRandomWR/leveldb-read-12 100000000 10375 ns/op 988 B/op 14 allocs/op
BenchmarkHddRandomWR/leveldb-write-read-12 50000 21144257 ns/op 994 B/op 9 allocs/op
* Total size of leveldb db: 31940 kb, Size of 1 write: 272 byte
[badgerdb-hdd-batch100]
testset_size: 10000000
key_len: 8
val_len: 256
device type: hdd
batch size: 100
BenchmarkHddRandomBatchWR/badgerdb-batch-write-12 1000000 675351 ns/op 329398 B/op 4007 allocs/op
BenchmarkHddRandomBatchWR/badgerdb-read-12 10000000 87933 ns/op 22713 B/op 85 allocs/op
* Total size of badgerdb db: 24285947 kb, Size of 1 write: 24620 byte
[leveldb-hdd-batch100]
testset_size: 10000000
key_len: 8
val_len: 256
device type: hdd
batch size: 100
BenchmarkHddRandomBatchWR/leveldb-batch-write-12 30000 45769594 ns/op 136783 B/op 399 allocs/op
BenchmarkHddRandomBatchWR/leveldb-read-12 20000000 30579 ns/op 3386 B/op 19 allocs/op
* Total size of leveldb db: 878403 kb, Size of 1 write: 22430 byte
[badgerdb-hdd-batch1000]
testset_size: 10000000
key_len: 8
val_len: 256
device type: hdd
batch size: 1000
BenchmarkHddRandomBatchWR1000/badgerdb-batch-write-12 200000 6599637 ns/op 3724911 B/op 43392 allocs/op
BenchmarkHddRandomBatchWR1000/badgerdb-read-12 10000000 99201 ns/op 31710 B/op 105 allocs/op
* Total size of badgerdb db: 51157857 kb, Size of 1 write: 249335 byte
[leveldb-hdd-batch1000]
testset_size: 10000000
key_len: 8
val_len: 256
device type: hdd
batch size: 1000
BenchmarkHddRandomBatchWR1000/leveldb-batch-write-12 10000 209360868 ns/op 1027546 B/op 3363 allocs/op
BenchmarkHddRandomBatchWR1000/leveldb-read-12 10000000 87929 ns/op 3971 B/op 29 allocs/op
* Total size of leveldb db: 1689120 kb, Size of 1 write: 171236 byte
PASS
ok github.com/aergoio/aergo-lib/db 33617.451s
*/

+ 60
- 0
db/types.go
View File

@ -0,0 +1,60 @@
/**
* @file
* @copyright defined in aergo/LICENSE.txt
*/
package db
// ImplType represents implementators of a DB interface
type ImplType string
const (
// LevelImpl represents a name of DB interface implementation using leveldb
LevelImpl ImplType = "leveldb"
// MemoryImpl represents a name of DB interface implementation in memory
MemoryImpl ImplType = "memorydb"
)
type dbConstructor func(dir string) (DB, error)
// DB is an general interface to access at storage data
type DB interface {
Type() string
Set(key, value []byte)
Delete(key []byte)
Get(key []byte) []byte
Exist(key []byte) bool
Iterator(start, end []byte) Iterator
NewTx() Transaction
NewBulk() Bulk
Close()
//Print()
//Stats() map[string]string
}
// Transaction is used to batch multiple operations
type Transaction interface {
// Get(key []byte) []byte
Set(key, value []byte)
Delete(key []byte)
Commit()
Discard()
}
// Bulk is used to batch multiple transactions
// This will internally commit transactions when reach maximum tx size
type Bulk interface {
Set(key, value []byte)
Delete(key []byte)
Flush()
DiscardLast()
}
// Iterator is used to navigate specific key ranges
type Iterator interface {
Next()
Valid() bool
Key() []byte
Value() []byte
}

+ 5
- 1
go.mod
View File

@ -3,7 +3,11 @@ module github.com/p4u/asmt
go 1.16 go 1.16
require ( require (
git.sr.ht/~sircmpwn/go-bare v0.0.0-20210227202403-5dae5c48f917
github.com/aergoio/aergo-lib v1.0.2 github.com/aergoio/aergo-lib v1.0.2
github.com/spf13/afero v1.2.1 // indirect
github.com/guptarohit/asciigraph v0.4.1
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
go.vocdoni.io/dvote v1.0.0
golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
) )

+ 1805
- 25
go.sum
File diff suppressed because it is too large
View File


README.md → smt/README.md
View File


hash.go → smt/hash.go
View File


pictures/batch.png → smt/pictures/batch.png
View File


pictures/deleted.png → smt/pictures/deleted.png
View File


pictures/mod.png → smt/pictures/mod.png
View File


pictures/smt.png → smt/pictures/smt.png
View File


trie.go → smt/trie.go
View File

@ -10,7 +10,7 @@ import (
"fmt" "fmt"
"sync" "sync"
"github.com/aergoio/aergo-lib/db"
"github.com/p4u/asmt/db"
) )
// Trie is a modified sparse Merkle tree. // Trie is a modified sparse Merkle tree.
@ -27,7 +27,7 @@ type Trie struct {
lock sync.RWMutex lock sync.RWMutex
// hash is the hash function used in the trie // hash is the hash function used in the trie
hash func(data ...[]byte) []byte hash func(data ...[]byte) []byte
// TrieHeight is the number if bits in a key
// TrieHeight is the number of bits in a key
TrieHeight int TrieHeight int
// LoadDbCounter counts the nb of db reads in on update // LoadDbCounter counts the nb of db reads in on update
LoadDbCounter int LoadDbCounter int

trie_cache.go → smt/trie_cache.go
View File

@ -8,7 +8,7 @@ package trie
import ( import (
"sync" "sync"
"github.com/aergoio/aergo-lib/db"
"github.com/p4u/asmt/db"
) )
// DbTx represents Set and Delete interface to store data // DbTx represents Set and Delete interface to store data

trie_merkle_proof.go → smt/trie_merkle_proof.go
View File


trie_revert.go → smt/trie_revert.go
View File


trie_test.go → smt/trie_test.go
View File

@ -20,7 +20,7 @@ import (
"sort" "sort"
"testing" "testing"
"github.com/aergoio/aergo-lib/db"
"github.com/p4u/asmt/db"
) )
func TestTrieEmpty(t *testing.T) { func TestTrieEmpty(t *testing.T) {
@ -137,7 +137,7 @@ func TestTriePublicUpdateAndGet(t *testing.T) {
func TestGetWithRoot(t *testing.T) { func TestGetWithRoot(t *testing.T) {
dbPath := t.TempDir() dbPath := t.TempDir()
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
smt.CacheHeightLimit = 0 smt.CacheHeightLimit = 0
@ -199,7 +199,7 @@ func TestGetWithRoot(t *testing.T) {
func TestTrieWalk(t *testing.T) { func TestTrieWalk(t *testing.T) {
dbPath := t.TempDir() dbPath := t.TempDir()
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
smt.CacheHeightLimit = 0 smt.CacheHeightLimit = 0
@ -455,7 +455,7 @@ func TestTrieCommit(t *testing.T) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
keys := getFreshData(10, 32) keys := getFreshData(10, 32)
@ -479,7 +479,7 @@ func TestTrieStageUpdates(t *testing.T) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
keys := getFreshData(10, 32) keys := getFreshData(10, 32)
@ -505,7 +505,7 @@ func TestTrieRevert(t *testing.T) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
@ -594,7 +594,7 @@ func TestTrieRaisesError(t *testing.T) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
// Add data to empty trie // Add data to empty trie
@ -644,7 +644,7 @@ func TestTrieLoadCache(t *testing.T) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
// Test size of cache // Test size of cache
@ -745,7 +745,7 @@ func TestStash(t *testing.T) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
// Add data to empty trie // Add data to empty trie
keys := getFreshData(20, 32) keys := getFreshData(20, 32)
@ -836,7 +836,7 @@ func BenchmarkCacheHeightLimit233(b *testing.B) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
smt.CacheHeightLimit = 233 smt.CacheHeightLimit = 233
benchmark10MAccounts10Ktps(smt, b) benchmark10MAccounts10Ktps(smt, b)
@ -848,7 +848,7 @@ func BenchmarkCacheHeightLimit238(b *testing.B) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
smt.CacheHeightLimit = 238 smt.CacheHeightLimit = 238
benchmark10MAccounts10Ktps(smt, b) benchmark10MAccounts10Ktps(smt, b)
@ -860,7 +860,7 @@ func BenchmarkCacheHeightLimit245(b *testing.B) {
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
_ = os.MkdirAll(dbPath, 0711) _ = os.MkdirAll(dbPath, 0711)
} }
st := db.NewDB(db.BadgerImpl, dbPath)
st := db.NewDB(db.LevelImpl, dbPath)
smt := NewTrie(nil, Hasher, st) smt := NewTrie(nil, Hasher, st)
smt.CacheHeightLimit = 245 smt.CacheHeightLimit = 245
benchmark10MAccounts10Ktps(smt, b) benchmark10MAccounts10Ktps(smt, b)

trie_tools.go → smt/trie_tools.go
View File

@ -11,7 +11,7 @@ import (
"sync" "sync"
"sync/atomic" "sync/atomic"
"github.com/aergoio/aergo-lib/db"
"github.com/p4u/asmt/db"
) )
// LoadCache loads the first layers of the merkle tree given a root // LoadCache loads the first layers of the merkle tree given a root

util.go → smt/util.go
View File


Write Preview
Loading…
Cancel
Save