full refactor

- include the db package from aergo to this repository
- move sparse merkle tree code to smt directory
- add a new root package for being compatible with vocdoni/censusTree

Signed-off-by: p4u <pau@dabax.net>

censustree.go

@@ -0,0 +1,420 @@
+package asmtree
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+	"sync/atomic"
+	"time"
+
+	"go.vocdoni.io/dvote/censustree"
+	"go.vocdoni.io/dvote/log"
+
+	"git.sr.ht/~sircmpwn/go-bare"
+	"github.com/p4u/asmt/db"
+	asmt "github.com/p4u/asmt/smt"
+)
+
+// We use go-bare for export/import the trie. In order to support
+// big census (up to 8 Million entries) we need to increase the maximums.
+const bareMaxArrayLength uint64 = 1024 * 1014 * 8 // 8 Million
+
+const bareMaxUnmarshalBytes uint64 = 1024 * 1024 * 200 // 200 MiB
+
+type Tree struct {
+	Tree           *asmt.Trie
+	db             db.DB
+	public         uint32
+	lastAccessUnix int64 // a unix timestamp, used via sync/atomic
+	size           uint64
+	snapshotRoot   []byte // if not nil, this trie is considered an inmutable snapshot
+	snapshotSize   uint64
+}
+
+type Proof struct {
+	Bitmap   []byte
+	Length   int
+	Siblings [][]byte
+	Value    []byte
+}
+
+type exportElement struct {
+	Key   []byte `bare:"key"`
+	Value []byte `bare:"value"`
+}
+
+type exportData struct {
+	Elements []exportElement `bare:"elements"`
+}
+
+const (
+	MaxKeySize   = 256
+	MaxValueSize = 256
+	dbRootPrefix = "this is the last root for the SMT tree"
+)
+
+// NewTree initializes a new AergoSMT tree following the censustree.Tree interface specification.
+func NewTree(name, storageDir string) (censustree.Tree, error) {
+	tr := &Tree{}
+	err := tr.Init(name, storageDir)
+	return tr, err
+}
+
+// newTree opens or creates a merkle tree under the given storage.
+func newTree(name, storageDir string) (*asmt.Trie, db.DB, error) {
+	dir := path.Join(storageDir, name)
+	log.Debugf("creating new tree on %s", dir)
+	d := db.NewDB(db.LevelImpl, dir)
+	root := d.Get([]byte(dbRootPrefix))
+	tr := asmt.NewTrie(root, asmt.Hasher, d)
+	if root != nil {
+		if err := tr.LoadCache(root); err != nil {
+			return nil, nil, err
+		}
+	}
+	return tr, d, nil
+}
+
+// Init initializes a new asmt tree
+func (t *Tree) Init(name, storageDir string) error {
+	var err error
+	t.Tree, t.db, err = newTree(name, storageDir)
+	t.updateAccessTime()
+	t.size = 0
+	return err
+}
+
+func (t *Tree) MaxKeySize() int {
+	return MaxKeySize
+}
+
+// LastAccess returns the last time the Tree was accessed, in the form of a unix
+// timestamp.
+func (t *Tree) LastAccess() int64 {
+	return atomic.LoadInt64(&t.lastAccessUnix)
+}
+
+func (t *Tree) updateAccessTime() {
+	atomic.StoreInt64(&t.lastAccessUnix, time.Now().Unix())
+}
+
+// Publish makes a merkle tree available for queries.
+// Application layer should check IsPublish() before considering the Tree available.
+func (t *Tree) Publish() {
+	atomic.StoreUint32(&t.public, 1)
+}
+
+// UnPublish makes a merkle tree not available for queries
+func (t *Tree) UnPublish() {
+	atomic.StoreUint32(&t.public, 0)
+}
+
+// IsPublic returns true if the tree is available
+func (t *Tree) IsPublic() bool {
+	return atomic.LoadUint32(&t.public) == 1
+}
+
+// Commit saves permanently the tree on disk
+func (t *Tree) Commit() error {
+	if t.snapshotRoot != nil {
+		return fmt.Errorf("cannot commit to a snapshot trie")
+	}
+	err := t.Tree.Commit()
+	if err != nil {
+		return err
+	}
+	t.db.Set([]byte(dbRootPrefix), t.Root())
+	return nil
+}
+
+// Add adds a new claim to the merkle tree
+// A claim is composed of two parts: index and value
+//  1.index is mandatory, the data will be used for indexing the claim into to merkle tree
+//  2.value is optional, the data will not affect the indexing
+func (t *Tree) Add(index, value []byte) error {
+	t.updateAccessTime()
+	if t.snapshotRoot != nil {
+		return fmt.Errorf("cannot add to a snapshot trie")
+	}
+	if len(index) < 4 {
+		return fmt.Errorf("index too small (%d), minimum size is 4 bytes", len(index))
+	}
+	if len(value) > MaxValueSize {
+		return fmt.Errorf("index or value claim data too big")
+	}
+	_, err := t.Tree.Update([][]byte{asmt.Hasher(index)}, [][]byte{asmt.Hasher(value)})
+	if err != nil {
+		return err
+	}
+	atomic.StoreUint64(&t.size, 0) // TBD: improve this
+	return t.Commit()
+}
+
+// AddBatch adds a list of indexes and values.
+// The commit to disk is executed only once.
+// The values slince could be empty or as long as indexes.
+func (t *Tree) AddBatch(indexes, values [][]byte) ([]int, error) {
+	var wrongIndexes []int
+	t.updateAccessTime()
+	if t.snapshotRoot != nil {
+		return wrongIndexes, fmt.Errorf("cannot add to a snapshot trie")
+	}
+	if len(values) > 0 && len(indexes) != len(values) {
+		return wrongIndexes, fmt.Errorf("indexes and values have different size")
+	}
+	var hashedIndexes [][]byte
+	var hashedValues [][]byte
+	var value []byte
+	for i, key := range indexes {
+		if len(key) < 4 {
+			wrongIndexes = append(wrongIndexes, i)
+			continue
+		}
+		value = nil
+		if len(values) > 0 {
+			if len(values[i]) > MaxValueSize {
+				wrongIndexes = append(wrongIndexes, i)
+				continue
+			}
+			value = values[i]
+		}
+		hashedIndexes = append(hashedIndexes, asmt.Hasher(key))
+		hashedValues = append(hashedValues, asmt.Hasher(value))
+	}
+	_, err := t.Tree.Update(hashedIndexes, hashedValues)
+	if err != nil {
+		return wrongIndexes, err
+	}
+	atomic.StoreUint64(&t.size, 0) // TBD: improve this
+	return wrongIndexes, t.Commit()
+}
+
+// Get returns the value of a key
+func (t *Tree) Get(key []byte) []byte { // Do something with error
+	var value []byte
+	if t.snapshotRoot != nil {
+		value, _ = t.Tree.GetWithRoot(key, t.snapshotRoot)
+	} else {
+		value, _ = t.Tree.Get(key)
+	}
+	return value
+}
+
+// GenProof generates a merkle tree proof that can be later used on CheckProof() to validate it.
+func (t *Tree) GenProof(index, value []byte) ([]byte, error) {
+	t.updateAccessTime()
+	var err error
+	var ap [][]byte
+	var pvalue []byte
+	var bitmap []byte
+	var length int
+	if t.snapshotRoot != nil {
+		bitmap, ap, length, _, _, pvalue, err = t.Tree.MerkleProofCompressedR(
+			asmt.Hasher(index),
+			t.snapshotRoot)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		bitmap, ap, length, _, _, pvalue, err = t.Tree.MerkleProofCompressed(
+			asmt.Hasher(index))
+		if err != nil {
+			return nil, err
+		}
+	}
+	//if !included {
+	//	return nil, fmt.Errorf("not included")
+	//}
+	if !bytes.Equal(pvalue, asmt.Hasher(value)) {
+		return nil, fmt.Errorf("incorrect value on genProof")
+	}
+	return bare.Marshal(&Proof{Bitmap: bitmap, Length: length, Siblings: ap, Value: pvalue})
+}
+
+// CheckProof validates a merkle proof and its data.
+func (t *Tree) CheckProof(index, value, root, mproof []byte) (bool, error) {
+	t.updateAccessTime()
+	p := Proof{}
+	if err := bare.Unmarshal(mproof, &p); err != nil {
+		return false, err
+	}
+	if !bytes.Equal(p.Value, asmt.Hasher(value)) {
+		return false, fmt.Errorf("values mismatch %x != %x", p.Value, asmt.Hasher(value))
+	}
+	if root != nil {
+		return t.Tree.VerifyInclusionWithRootC(
+			root,
+			p.Bitmap,
+			asmt.Hasher(index),
+			p.Value,
+			p.Siblings,
+			p.Length), nil
+	}
+	if t.snapshotRoot != nil {
+		return t.Tree.VerifyInclusionWithRootC(
+			t.snapshotRoot,
+			p.Bitmap,
+			asmt.Hasher(index),
+			p.Value,
+			p.Siblings,
+			p.Length), nil
+	}
+	return t.Tree.VerifyInclusionC(
+		p.Bitmap,
+		asmt.Hasher(index),
+		p.Value,
+		p.Siblings,
+		p.Length), nil
+}
+
+// Root returns the current root hash of the merkle tree
+func (t *Tree) Root() []byte {
+	t.updateAccessTime()
+	if t.snapshotRoot != nil {
+		return t.snapshotRoot
+	}
+	return t.Tree.Root
+}
+
+// Dump returns the whole merkle tree serialized in a format that can be used on Import.
+// Byte seralization is performed using bare message protocol, it is a 40% size win over JSON.
+func (t *Tree) Dump(root []byte) ([]byte, error) {
+	t.updateAccessTime()
+	if root == nil && t.snapshotRoot != nil {
+		root = t.snapshotRoot
+	}
+	dump := exportData{}
+	t.iterateWithRoot(root, nil, func(k, v []byte) bool {
+		ee := exportElement{Key: make([]byte, len(k)), Value: make([]byte, len(v))}
+		// Copy elements since it's not safe to hold on to the []byte values from Iterate
+		copy(ee.Key, k[:])
+		copy(ee.Value, v[:])
+		dump.Elements = append(dump.Elements, ee)
+		return false
+	})
+	bare.MaxArrayLength(bareMaxArrayLength)
+	bare.MaxUnmarshalBytes(bareMaxUnmarshalBytes)
+
+	return bare.Marshal(&dump)
+}
+
+// String returns a human readable representation of the tree.
+func (t *Tree) String() string {
+	s := bytes.Buffer{}
+	t.iterate(t.snapshotRoot, func(k, v []byte) bool {
+		s.WriteString(fmt.Sprintf("%x => %x\n", k, v))
+		return false
+	})
+	return s.String()
+}
+
+// Size returns the number of leaf nodes on the merkle tree.
+// TO-DO: root is currently ignored
+func (t *Tree) Size(root []byte) (int64, error) {
+	if t.snapshotRoot != nil {
+		return int64(t.snapshotSize), nil
+	}
+	return int64(t.count()), nil
+}
+
+// DumpPlain returns the entire list of added claims for a specific root hash.
+// First return parametre are the indexes and second the values.
+// If root is not specified, the last one is used.
+func (t *Tree) DumpPlain(root []byte) ([][]byte, [][]byte, error) {
+	var indexes, values [][]byte
+	var err error
+	t.updateAccessTime()
+
+	t.iterateWithRoot(root, nil, func(k, v []byte) bool {
+		indexes = append(indexes, k)
+		values = append(values, v)
+		return false
+	})
+
+	return indexes, values, err
+}
+
+// ImportDump imports a partial or whole tree previously exported with Dump()
+func (t *Tree) ImportDump(data []byte) error {
+	t.updateAccessTime()
+	if t.snapshotRoot != nil {
+		return fmt.Errorf("cannot import to a snapshot")
+	}
+	census := new(exportData)
+	bare.MaxArrayLength(bareMaxArrayLength)
+	bare.MaxUnmarshalBytes(bareMaxUnmarshalBytes)
+	if err := bare.Unmarshal(data, census); err != nil {
+		return fmt.Errorf("importdump cannot unmarshal data: %w", err)
+	}
+	keys := [][]byte{}
+	values := [][]byte{}
+	for _, ee := range census.Elements {
+		keys = append(keys, ee.Key)
+		values = append(values, ee.Value)
+	}
+	_, err := t.Tree.Update(keys, values)
+	if err != nil {
+		return err
+	}
+	atomic.StoreUint64(&t.size, 0) // TBD: improve this
+	return t.Commit()
+}
+
+// Snapshot returns a Tree instance of a exiting merkle root.
+// A Snapshot cannot be modified.
+func (t *Tree) Snapshot(root []byte) (censustree.Tree, error) {
+	exist, err := t.HashExists(root)
+	if err != nil {
+		return nil, err
+	}
+	if !exist {
+		return nil, fmt.Errorf("root %x does not exist, cannot build snapshot", root)
+	}
+	return &Tree{Tree: t.Tree, public: t.public, snapshotRoot: root, snapshotSize: t.count()}, nil
+}
+
+func (t *Tree) Close() error {
+	t.db.Close()
+	return nil
+}
+
+// HashExists checks if a hash exists as a node in the merkle tree
+func (t *Tree) HashExists(hash []byte) (bool, error) {
+	t.updateAccessTime()
+	return t.Tree.TrieRootExists(hash), nil
+}
+
+func (t *Tree) count() uint64 {
+	if v := atomic.LoadUint64(&t.size); v != 0 {
+		return v
+	}
+	counter := uint64(0)
+	if err := t.Tree.Walk(t.snapshotRoot, func(*asmt.WalkResult) int32 {
+		counter++
+		return 0
+	}); err != nil {
+		return 0
+	}
+	atomic.StoreUint64(&t.size, counter)
+	return counter
+}
+
+func (t *Tree) iterate(prefix []byte, callback func(key, value []byte) bool) {
+	t.Tree.Walk(t.snapshotRoot, func(v *asmt.WalkResult) int32 {
+		if callback(v.Key, v.Value) {
+			return 1
+		} else {
+			return 0
+		}
+	})
+}
+
+func (t *Tree) iterateWithRoot(root, prefix []byte, callback func(key, value []byte) bool) {
+	t.Tree.Walk(root, func(v *asmt.WalkResult) int32 {
+		if callback(v.Key, v.Value) {
+			return 1
+		} else {
+			return 0
+		}
+	})
+}

censustree_test.go

@@ -0,0 +1,90 @@
+package asmtree
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+)
+
+func TestTree(t *testing.T) {
+	censusSize := 1000
+	storage := t.TempDir()
+	tr1 := &Tree{}
+	err := tr1.Init("test1", storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < censusSize; i++ {
+		if err = tr1.Add([]byte(fmt.Sprintf("number %d", i)),
+			[]byte(fmt.Sprintf("number %d value", i))); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	root1 := tr1.Root()
+	data, err := tr1.Dump(root1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("dumped data size is: %d bytes", len(data))
+
+	tr2 := &Tree{}
+	err = tr2.Init("test2", storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = tr2.ImportDump(data); err != nil {
+		t.Fatal(err)
+	}
+	root2 := tr2.Root()
+	if !bytes.Equal(root1, root2) {
+		t.Errorf("roots are different but they should be equal (%x != %x)", root1, root2)
+	}
+
+	// Try closing the storage and creating the tree again
+	tr2.Close()
+	err = tr2.Init("test2", storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Get the size
+	s, err := tr2.Size(nil)
+	if err != nil {
+		t.Errorf("cannot get te size of the tree after reopen: (%s)", err)
+	}
+	if s != int64(censusSize) {
+		t.Errorf("Size is wrong (have %d, expexted %d)", s, censusSize)
+	}
+
+	// Check Root is still the same
+	if !bytes.Equal(tr2.Root(), root2) {
+		t.Fatalf("after closing and opening the tree, the root is different")
+	}
+
+	// Generate a proof on tr1 and check validity on snapshot and tr2
+	proof1, err := tr1.GenProof([]byte("number 5"), []byte("number 5 value"))
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("Proof Length: %d", len(proof1))
+	tr1s, err := tr1.Snapshot(root1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	valid, err := tr1s.CheckProof([]byte("number 5"), []byte("number 5 value"), root1, proof1)
+	if err != nil {
+		t.Error(err)
+	}
+	if !valid {
+		t.Errorf("proof is invalid on snapshot")
+	}
+	valid, err = tr2.CheckProof([]byte("number 5"), []byte("number 5 value"), nil, proof1)
+	if err != nil {
+		t.Error(err)
+	}
+	if !valid {
+		t.Errorf("proof is invalid on tree2")
+	}
+
+}

db/db.go

@@ -0,0 +1,87 @@
+/**
+ *  @file
+ *  @copyright defined in aergo/LICENSE.txt
+ */
+
+/*
+Package db is an wrapper of key-value database implementations. Currently, this supports badgerdb (https://github.com/dgraph-io/badger).
+
+Basic Usage
+
+You can create database using a newdb func like this
+ database := NewDB(BadgerImpl, "./test")
+
+A first argument is a backend db type to use, and a second is a root directory to store db files.
+After creating db, you can write, read or delete single key-value using funcs in DB interface.
+ // write data
+ database.Set([]byte("key"), []byte("val"))
+
+ // read data
+ read := Get([]byte("key"))
+
+ // delete data
+ database.Delete([]byte("key"))
+
+Transaction
+
+A Transaction is a bulk set of operations to ensure atomic success or fail.
+ // create a new transaction
+ tx := database.NewTX(true)
+
+ // reserve writing
+ tx.Set([]byte("keyA"), []byte("valA"))
+ tx.Set([]byte("keyB"), []byte("valB"))
+
+ // Get will return a value reserved to write in this transaction
+ mustBeValA := tx.Get([]byte("keyA"))
+
+ // Perform writing
+ tx.Commit()
+If you want to cancel and discard operations in tx, then you must call Discard() func to prevent a memory leack
+ // If you create a tx, but do not commit, than you have to call this
+ tx.Discard()
+
+Iterator
+
+An iteractor provides a way to get all keys sequentially.
+ // create an iterator that covers all range
+ for iter := database.Iterator(nil, nil); iter.Valid(); iter.Next() {
+	 // print each key-value pair
+	 fmt.Printf("%s = %s", string(iter.Key()), string(iter.Value()))
+ }
+
+You can find more detail usages at a db_test.go file
+*/
+package db
+
+import (
+	"fmt"
+
+	"github.com/aergoio/aergo-lib/log"
+)
+
+var dbImpls = map[ImplType]dbConstructor{}
+var logger *extendedLog
+
+func registorDBConstructor(dbimpl ImplType, constructor dbConstructor) {
+	dbImpls[dbimpl] = constructor
+}
+
+// NewDB creates new database or load existing database in the directory
+func NewDB(dbimpltype ImplType, dir string) DB {
+	logger = &extendedLog{Logger: log.NewLogger("db")}
+	db, err := dbImpls[dbimpltype](dir)
+
+	if err != nil {
+		panic(fmt.Sprintf("Fail to Create New DB: %v", err))
+	}
+
+	return db
+}
+
+func convNilToBytes(byteArray []byte) []byte {
+	if byteArray == nil {
+		return []byte{}
+	}
+	return byteArray
+}

db/db_test.go

@@ -0,0 +1,276 @@
+/**
+ *  @file
+ *  @copyright defined in aergo/LICENSE.txt
+ */
+package db
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	tmpDbTestKey1    = "tempkey1"
+	tmpDbTestKey2    = "tempkey2"
+	tmpDbTestStrVal1 = "val1"
+	tmpDbTestStrVal2 = "val2"
+	tmpDbTestIntVal1 = 1
+	tmpDbTestIntVal2 = 2
+)
+
+func createTmpDB(key ImplType) (dir string, db DB) {
+	dir, err := ioutil.TempDir("", string(key))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	db = NewDB(key, dir)
+
+	return
+}
+
+func setInitData(db DB) {
+	tx := db.NewTx()
+
+	tx.Set([]byte("1"), []byte("1"))
+	tx.Set([]byte("2"), []byte("2"))
+	tx.Set([]byte("3"), []byte("3"))
+	tx.Set([]byte("4"), []byte("4"))
+	tx.Set([]byte("5"), []byte("5"))
+	tx.Set([]byte("6"), []byte("6"))
+	tx.Set([]byte("7"), []byte("7"))
+
+	tx.Commit()
+}
+
+func TestGetSetDeleteExist(t *testing.T) {
+	// for each db implementation
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		// initial value of empty key must be empty byte
+		assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
+		assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
+
+		// set value
+		db.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
+
+		// check value set
+		assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
+		assert.True(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
+
+		// delete value
+		db.Delete([]byte(tmpDbTestKey1))
+
+		// value must be erased
+		assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
+		assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestTransactionSet(t *testing.T) {
+
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		// create a new writable tx
+		tx := db.NewTx()
+
+		// set the value in the tx
+		tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
+		// the value will not visible at a db
+		assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
+
+		tx.Commit()
+
+		// after commit, the value visible from the db
+		assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestTransactionDiscard(t *testing.T) {
+
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		// create a new writable tx
+		tx := db.NewTx()
+		// discard test
+		tx = db.NewTx()
+		// set the value in the tx
+		tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal2))
+
+		// discard tx
+		tx.Discard()
+
+		assert.Panics(t, func() { tx.Commit() }, "commit after discard is not allowed")
+
+		// after discard, the value must be reset at the db
+		assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestTransactionDelete(t *testing.T) {
+
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		// create a new writable tx
+		tx := db.NewTx()
+
+		// set the value in the tx
+		tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
+
+		// delete the value in the tx
+		tx.Delete([]byte(tmpDbTestKey1))
+
+		tx.Commit()
+
+		// after commit, chekc the value from the db
+		assert.Equal(t, "", string(db.Get([]byte(tmpDbTestKey1))), db.Type())
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestTransactionCommitTwice(t *testing.T) {
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		// create a new writable tx
+		tx := db.NewTx()
+
+		// a first commit will success
+		tx.Commit()
+
+		// a second commit will cause panic
+		assert.Panics(t, func() { tx.Commit() })
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestBulk(t *testing.T) {
+
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		// create a new Bulk instance
+		bulk := db.NewBulk()
+
+		// set the huge number of value in the bulk
+		for i := 0; i < 1000000; i++ {
+			bulk.Set([]byte(fmt.Sprintf("key%d", i)),
+				[]byte(tmpDbTestStrVal1))
+		}
+
+		bulk.Flush()
+
+		// after commit, the value visible from the db
+
+		for i := 0; i < 1000000; i++ {
+			assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(fmt.Sprintf("key%d", i)))), db.Type())
+		}
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestIter(t *testing.T) {
+
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		setInitData(db)
+
+		i := 1
+
+		for iter := db.Iterator(nil, nil); iter.Valid(); iter.Next() {
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
+			i++
+		}
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestRangeIter(t *testing.T) {
+
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		setInitData(db)
+
+		// test iteration 2 -> 5
+		i := 2
+		for iter := db.Iterator([]byte("2"), []byte("5")); iter.Valid(); iter.Next() {
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
+			i++
+		}
+		assert.EqualValues(t, i, 5)
+
+		// nil sames with []byte("0")
+		// test iteration 0 -> 5
+		i = 1
+		for iter := db.Iterator(nil, []byte("5")); iter.Valid(); iter.Next() {
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
+			i++
+		}
+		assert.EqualValues(t, i, 5)
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}
+
+func TestReverseIter(t *testing.T) {
+
+	for key := range dbImpls {
+		dir, db := createTmpDB(key)
+
+		setInitData(db)
+
+		// test reverse iteration 5 <- 2
+		i := 5
+		for iter := db.Iterator([]byte("5"), []byte("2")); iter.Valid(); iter.Next() {
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
+			i--
+		}
+		assert.EqualValues(t, i, 2)
+
+		// nil sames with []byte("0")
+		// test reverse iteration 5 -> 0
+		i = 5
+		for iter := db.Iterator([]byte("5"), nil); iter.Valid(); iter.Next() {
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
+			assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
+			i--
+		}
+		assert.EqualValues(t, i, 0)
+
+		db.Close()
+		os.RemoveAll(dir)
+	}
+}

db/leveldb.go

@@ -0,0 +1,322 @@
+/**
+ *  @file
+ *  @copyright defined in aergo/LICENSE.txt
+ */
+
+package db
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+// This function is always called first
+func init() {
+	dbConstructor := func(dir string) (DB, error) {
+		return newLevelDB(dir)
+	}
+	registorDBConstructor(LevelImpl, dbConstructor)
+}
+
+func newLevelDB(dir string) (DB, error) {
+	dbPath := filepath.Join(dir, "data.db")
+
+	db, err := leveldb.OpenFile(dbPath, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	database := &levelDB{
+		db: db,
+	}
+	return database, nil
+}
+
+//=========================================================
+// DB Implementation
+//=========================================================
+
+// Enforce database and transaction implements interfaces
+var _ DB = (*levelDB)(nil)
+
+type levelDB struct {
+	db *leveldb.DB
+}
+
+func (db *levelDB) Type() string {
+	return "leveldb"
+}
+
+func (db *levelDB) Set(key, value []byte) {
+	key = convNilToBytes(key)
+	value = convNilToBytes(value)
+
+	err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
+
+	if err != nil {
+		panic(fmt.Sprintf("Database Error: %v", err))
+	}
+}
+
+func (db *levelDB) Delete(key []byte) {
+	key = convNilToBytes(key)
+
+	err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
+
+	if err != nil {
+		panic(fmt.Sprintf("Database Error: %v", err))
+	}
+}
+
+func (db *levelDB) Get(key []byte) []byte {
+	key = convNilToBytes(key)
+	res, err := db.db.Get(key, nil)
+	if err != nil {
+		if err == errors.ErrNotFound {
+			return []byte{}
+		}
+		panic(fmt.Sprintf("Database Error: %v", err))
+	}
+	return res
+}
+
+func (db *levelDB) Exist(key []byte) bool {
+	res, _ := db.db.Has(key, nil)
+	return res
+}
+
+func (db *levelDB) Close() {
+	db.db.Close()
+}
+
+func (db *levelDB) NewTx() Transaction {
+	batch := new(leveldb.Batch)
+
+	return &levelTransaction{db, batch, false, false}
+}
+
+func (db *levelDB) NewBulk() Bulk {
+	batch := new(leveldb.Batch)
+
+	return &levelBulk{db, batch, false, false}
+}
+
+//=========================================================
+// Transaction Implementation
+//=========================================================
+
+type levelTransaction struct {
+	db        *levelDB
+	tx        *leveldb.Batch
+	isDiscard bool
+	isCommit  bool
+}
+
+/*
+func (transaction *levelTransaction) Get(key []byte) []byte {
+	panic(fmt.Sprintf("DO not support"))
+}
+*/
+
+func (transaction *levelTransaction) Set(key, value []byte) {
+	transaction.tx.Put(key, value)
+}
+
+func (transaction *levelTransaction) Delete(key []byte) {
+	transaction.tx.Delete(key)
+}
+
+func (transaction *levelTransaction) Commit() {
+	if transaction.isDiscard {
+		panic("Commit after dicard tx is not allowed")
+	} else if transaction.isCommit {
+		panic("Commit occures two times")
+	}
+	err := transaction.db.db.Write(transaction.tx, &opt.WriteOptions{Sync: true})
+	if err != nil {
+		panic(fmt.Sprintf("Database Error: %v", err))
+	}
+	transaction.isCommit = true
+}
+
+func (transaction *levelTransaction) Discard() {
+	transaction.isDiscard = true
+}
+
+//=========================================================
+// Bulk Implementation
+//=========================================================
+
+type levelBulk struct {
+	db        *levelDB
+	tx        *leveldb.Batch
+	isDiscard bool
+	isCommit  bool
+}
+
+func (bulk *levelBulk) Set(key, value []byte) {
+	bulk.tx.Put(key, value)
+}
+
+func (bulk *levelBulk) Delete(key []byte) {
+	bulk.tx.Delete(key)
+}
+
+func (bulk *levelBulk) Flush() {
+	// do the same behavior that a transaction commit does
+	// db.write internally will handle large transaction
+	if bulk.isDiscard {
+		panic("Commit after dicard tx is not allowed")
+	} else if bulk.isCommit {
+		panic("Commit occures two times")
+	}
+
+	err := bulk.db.db.Write(bulk.tx, &opt.WriteOptions{Sync: true})
+	if err != nil {
+		panic(fmt.Sprintf("Database Error: %v", err))
+	}
+	bulk.isCommit = true
+}
+
+func (bulk *levelBulk) DiscardLast() {
+	bulk.isDiscard = true
+}
+
+//=========================================================
+// Iterator Implementation
+//=========================================================
+
+type levelIterator struct {
+	start     []byte
+	end       []byte
+	reverse   bool
+	iter      iterator.Iterator
+	isInvalid bool
+}
+
+func (db *levelDB) Iterator(start, end []byte) Iterator {
+	var reverse bool
+
+	// if end is bigger then start, then reverse order
+	if bytes.Compare(start, end) == 1 {
+		reverse = true
+	} else {
+		reverse = false
+	}
+
+	iter := db.db.NewIterator(nil, nil)
+
+	if reverse {
+		if start == nil {
+			iter.Last()
+		} else {
+			valid := iter.Seek(start)
+			if valid {
+				soakey := iter.Key()
+				if bytes.Compare(start, soakey) < 0 {
+					iter.Prev()
+				}
+			} else {
+				iter.Last()
+			}
+		}
+	} else {
+		if start == nil {
+			iter.First()
+		} else {
+			iter.Seek(start)
+		}
+	}
+	return &levelIterator{
+		iter:      iter,
+		start:     start,
+		end:       end,
+		reverse:   reverse,
+		isInvalid: false,
+	}
+}
+
+func (iter *levelIterator) Next() {
+	if iter.Valid() {
+		if iter.reverse {
+			iter.iter.Prev()
+		} else {
+			iter.iter.Next()
+		}
+	} else {
+		panic("Iterator is Invalid")
+	}
+}
+
+func (iter *levelIterator) Valid() bool {
+
+	// Once invalid, forever invalid.
+	if iter.isInvalid {
+		return false
+	}
+
+	// Panic on DB error.  No way to recover.
+	if err := iter.iter.Error(); err != nil {
+		panic(err)
+	}
+
+	// If source is invalid, invalid.
+	if !iter.iter.Valid() {
+		iter.isInvalid = true
+		return false
+	}
+
+	// If key is end or past it, invalid.
+	var end = iter.end
+	var key = iter.iter.Key()
+
+	if iter.reverse {
+		if end != nil && bytes.Compare(key, end) <= 0 {
+			iter.isInvalid = true
+			return false
+		}
+	} else {
+		if end != nil && bytes.Compare(end, key) <= 0 {
+			iter.isInvalid = true
+			return false
+		}
+	}
+
+	// Valid
+	return true
+}
+
+func (iter *levelIterator) Key() (key []byte) {
+	if !iter.Valid() {
+		panic("Iterator is invalid")
+	} else if err := iter.iter.Error(); err != nil {
+		panic(err)
+	}
+
+	originalKey := iter.iter.Key()
+
+	key = make([]byte, len(originalKey))
+	copy(key, originalKey)
+
+	return key
+}
+
+func (iter *levelIterator) Value() (value []byte) {
+	if !iter.Valid() {
+		panic("Iterator is invalid")
+	} else if err := iter.iter.Error(); err != nil {
+		panic(err)
+	}
+	originalValue := iter.iter.Value()
+
+	value = make([]byte, len(originalValue))
+	copy(value, originalValue)
+
+	return value
+}

db/logger.go

@@ -0,0 +1,26 @@
+package db
+
+import (
+	"github.com/aergoio/aergo-lib/log"
+)
+
+type extendedLog struct {
+	*log.Logger
+}
+
+func (l *extendedLog) Errorf(f string, v ...interface{}) {
+	l.Error().Msgf(f, v...)
+}
+
+func (l *extendedLog) Warningf(f string, v ...interface{}) {
+	l.Warn().Msgf(f, v...)
+}
+
+func (l *extendedLog) Infof(f string, v ...interface{}) {
+	// reduce info to debug level because infos at badgerdb are too detail
+	l.Debug().Msgf(f, v...) // INFO -> DEBUG
+}
+
+func (l *extendedLog) Debugf(f string, v ...interface{}) {
+	l.Debug().Msgf(f, v...)
+}

db/memorydb.go

@@ -0,0 +1,384 @@
+/**
+ *  @file
+ *  @copyright defined in aergo/LICENSE.txt
+ */
+
+package db
+
+import (
+	"bytes"
+	"container/list"
+	"encoding/gob"
+	"os"
+	"path"
+	"sort"
+	"sync"
+)
+
+// This function is always called first
+func init() {
+	dbConstructor := func(dir string) (DB, error) {
+		return newMemoryDB(dir)
+	}
+	registorDBConstructor(MemoryImpl, dbConstructor)
+}
+
+func newMemoryDB(dir string) (DB, error) {
+	var db map[string][]byte
+
+	filePath := path.Join(dir, "database")
+
+	file, err := os.Open(filePath)
+	if err == nil {
+		decoder := gob.NewDecoder(file) //
+		err = decoder.Decode(&db)
+
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	file.Close()
+
+	if db == nil {
+		db = make(map[string][]byte)
+	}
+
+	database := &memorydb{
+		db:  db,
+		dir: filePath,
+	}
+
+	return database, nil
+}
+
+//=========================================================
+// DB Implementation
+//=========================================================
+
+// Enforce database and transaction implements interfaces
+var _ DB = (*memorydb)(nil)
+
+type memorydb struct {
+	lock sync.Mutex
+	db   map[string][]byte
+	dir  string
+}
+
+func (db *memorydb) Type() string {
+	return "memorydb"
+}
+
+func (db *memorydb) Set(key, value []byte) {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	key = convNilToBytes(key)
+	value = convNilToBytes(value)
+
+	db.db[string(key)] = value
+}
+
+func (db *memorydb) Delete(key []byte) {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	key = convNilToBytes(key)
+
+	delete(db.db, string(key))
+}
+
+func (db *memorydb) Get(key []byte) []byte {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	key = convNilToBytes(key)
+
+	return db.db[string(key)]
+}
+
+func (db *memorydb) Exist(key []byte) bool {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	key = convNilToBytes(key)
+
+	_, ok := db.db[string(key)]
+
+	return ok
+}
+
+func (db *memorydb) Close() {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	file, err := os.OpenFile(db.dir, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err == nil {
+		encoder := gob.NewEncoder(file)
+		encoder.Encode(db.db)
+	}
+	file.Close()
+}
+
+func (db *memorydb) NewTx() Transaction {
+
+	return &memoryTransaction{
+		db:        db,
+		opList:    list.New(),
+		isDiscard: false,
+		isCommit:  false,
+	}
+}
+
+func (db *memorydb) NewBulk() Bulk {
+
+	return &memoryBulk{
+		db:        db,
+		opList:    list.New(),
+		isDiscard: false,
+		isCommit:  false,
+	}
+}
+
+//=========================================================
+// Transaction Implementation
+//=========================================================
+
+type memoryTransaction struct {
+	txLock    sync.Mutex
+	db        *memorydb
+	opList    *list.List
+	isDiscard bool
+	isCommit  bool
+}
+
+type txOp struct {
+	isSet bool
+	key   []byte
+	value []byte
+}
+
+func (transaction *memoryTransaction) Set(key, value []byte) {
+	transaction.txLock.Lock()
+	defer transaction.txLock.Unlock()
+
+	key = convNilToBytes(key)
+	value = convNilToBytes(value)
+
+	transaction.opList.PushBack(&txOp{true, key, value})
+}
+
+func (transaction *memoryTransaction) Delete(key []byte) {
+	transaction.txLock.Lock()
+	defer transaction.txLock.Unlock()
+
+	key = convNilToBytes(key)
+
+	transaction.opList.PushBack(&txOp{false, key, nil})
+}
+
+func (transaction *memoryTransaction) Commit() {
+	transaction.txLock.Lock()
+	defer transaction.txLock.Unlock()
+
+	if transaction.isDiscard {
+		panic("Commit after dicard tx is not allowed")
+	} else if transaction.isCommit {
+		panic("Commit occures two times")
+	}
+
+	db := transaction.db
+
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	for e := transaction.opList.Front(); e != nil; e = e.Next() {
+		op := e.Value.(*txOp)
+		if op.isSet {
+			db.db[string(op.key)] = op.value
+		} else {
+			delete(db.db, string(op.key))
+		}
+	}
+
+	transaction.isCommit = true
+}
+
+func (transaction *memoryTransaction) Discard() {
+	transaction.txLock.Lock()
+	defer transaction.txLock.Unlock()
+
+	transaction.isDiscard = true
+}
+
+//=========================================================
+// Bulk Implementation
+//=========================================================
+
+type memoryBulk struct {
+	txLock    sync.Mutex
+	db        *memorydb
+	opList    *list.List
+	isDiscard bool
+	isCommit  bool
+}
+
+func (bulk *memoryBulk) Set(key, value []byte) {
+	bulk.txLock.Lock()
+	defer bulk.txLock.Unlock()
+
+	key = convNilToBytes(key)
+	value = convNilToBytes(value)
+
+	bulk.opList.PushBack(&txOp{true, key, value})
+}
+
+func (bulk *memoryBulk) Delete(key []byte) {
+	bulk.txLock.Lock()
+	defer bulk.txLock.Unlock()
+
+	key = convNilToBytes(key)
+
+	bulk.opList.PushBack(&txOp{false, key, nil})
+}
+
+func (bulk *memoryBulk) Flush() {
+	bulk.txLock.Lock()
+	defer bulk.txLock.Unlock()
+
+	if bulk.isDiscard {
+		panic("Commit after dicard tx is not allowed")
+	} else if bulk.isCommit {
+		panic("Commit occures two times")
+	}
+
+	db := bulk.db
+
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	for e := bulk.opList.Front(); e != nil; e = e.Next() {
+		op := e.Value.(*txOp)
+		if op.isSet {
+			db.db[string(op.key)] = op.value
+		} else {
+			delete(db.db, string(op.key))
+		}
+	}
+
+	bulk.isCommit = true
+}
+
+func (bulk *memoryBulk) DiscardLast() {
+	bulk.txLock.Lock()
+	defer bulk.txLock.Unlock()
+
+	bulk.isDiscard = true
+}
+
+//=========================================================
+// Iterator Implementation
+//=========================================================
+
+type memoryIterator struct {
+	start     []byte
+	end       []byte
+	reverse   bool
+	keys      []string
+	isInvalid bool
+	cursor    int
+	db        *memorydb
+}
+
+func isKeyInRange(key []byte, start []byte, end []byte, reverse bool) bool {
+	if reverse {
+		if start != nil && bytes.Compare(start, key) < 0 {
+			return false
+		}
+		if end != nil && bytes.Compare(key, end) <= 0 {
+			return false
+		}
+		return true
+	}
+
+	if bytes.Compare(key, start) < 0 {
+		return false
+	}
+	if end != nil && bytes.Compare(end, key) <= 0 {
+		return false
+	}
+	return true
+
+}
+
+func (db *memorydb) Iterator(start, end []byte) Iterator {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	var reverse bool
+
+	// if end is bigger then start, then reverse order
+	if bytes.Compare(start, end) == 1 {
+		reverse = true
+	} else {
+		reverse = false
+	}
+
+	var keys sort.StringSlice
+
+	for key := range db.db {
+		if isKeyInRange([]byte(key), start, end, reverse) {
+			keys = append(keys, key)
+		}
+	}
+	if reverse {
+		sort.Sort(sort.Reverse(keys))
+	} else {
+		sort.Strings(keys)
+	}
+
+	return &memoryIterator{
+		start:     start,
+		end:       end,
+		reverse:   reverse,
+		isInvalid: false,
+		keys:      keys,
+		cursor:    0,
+		db:        db,
+	}
+}
+
+func (iter *memoryIterator) Next() {
+	if !iter.Valid() {
+		panic("Iterator is Invalid")
+	}
+
+	iter.cursor++
+}
+
+func (iter *memoryIterator) Valid() bool {
+	// Once invalid, forever invalid.
+	if iter.isInvalid {
+		return false
+	}
+
+	return 0 <= iter.cursor && iter.cursor < len(iter.keys)
+}
+
+func (iter *memoryIterator) Key() (key []byte) {
+	if !iter.Valid() {
+		panic("Iterator is Invalid")
+	}
+
+	return []byte(iter.keys[iter.cursor])
+}
+
+func (iter *memoryIterator) Value() (value []byte) {
+	if !iter.Valid() {
+		panic("Iterator is Invalid")
+	}
+
+	key := []byte(iter.keys[iter.cursor])
+
+	return iter.db.Get(key)
+}

db/perf_test.go

@@ -0,0 +1,443 @@
+package db
+
+import (
+	"bytes"
+	"container/list"
+	"encoding/binary"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/guptarohit/asciigraph"
+)
+
+type testKeyType int64
+
+// Simple execution cmd
+// go test -run=XXX -bench=. -benchmem -cpuprofile=cpu.out -memprofile=mem.out -timeout 20m -benchtime=1m
+
+var valueLen = 256
+var testSetSize = int64(10000000)
+var batchSize = 100
+
+// parameters for drawing graph
+var graphCountingPeriod = 10000
+var graphWidth = 120
+var graphHeigh = 20
+
+var deviceType = "ssd"
+
+func init() {
+	rand.Seed(time.Now().Unix())
+}
+
+func int64ToBytes(i testKeyType) []byte {
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, uint64(i))
+	return buf
+}
+
+func printGraph(statics list.List) {
+	if statics.Len() == 0 {
+		return
+	}
+	// convert list to slice
+	var staticsSlice = make([]float64, statics.Len())
+	i := 0
+	for e := statics.Front(); e != nil; e = e.Next() {
+		staticsSlice[i] = e.Value.(float64)
+		i++
+	}
+
+	graph := asciigraph.Plot(staticsSlice, asciigraph.Width(graphWidth), asciigraph.Height(graphHeigh))
+	fmt.Println(graph)
+}
+
+func dirSizeKB(path string) (int64, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return err
+	})
+	return size / 1024, err
+}
+
+func BenchmarkRandomWR(b *testing.B) {
+
+	// generate a common random data set
+	internal := map[testKeyType][]byte{}
+	for i := 0; i < int(testSetSize); i++ {
+		// generate a random value
+		token := make([]byte, valueLen)
+		internal[testKeyType(i)] = token
+	}
+
+	for dbType, dbConstructors := range dbImpls {
+
+		// create db
+		dbName := string(dbType)
+		tmpDir, _ := ioutil.TempDir("", dbName)
+		defer os.RemoveAll(tmpDir)
+
+		dbInstance, _ := dbConstructors(tmpDir)
+		var numOfWrite int64
+		var idx testKeyType
+		var statics list.List
+		var startTime time.Time
+
+		fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
+			dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
+		fmt.Println("device type: ssd")
+
+		// write only
+		b.Run(dbName+"-write", func(b *testing.B) {
+			var i int
+			for i = 0; i < b.N; i++ {
+				if i != 0 && i%graphCountingPeriod == 0 {
+					startTime = time.Now()
+				}
+				idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
+				rand.Read(internal[idx])                             // generate a random data
+
+				dbInstance.Set(
+					int64ToBytes(testKeyType(idx)),
+					internal[idx],
+				)
+				if i != 0 && i%graphCountingPeriod == 0 {
+					endTime := time.Now().Sub(startTime).Seconds()
+					statics.PushBack(endTime)
+				}
+			}
+			numOfWrite += int64(i)
+		})
+
+		printGraph(statics)
+		statics.Init()
+
+		// read only
+		b.Run(dbName+"-read", func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				if i != 0 && i%graphCountingPeriod == 0 {
+					startTime = time.Now()
+				}
+				idx = testKeyType((int64(rand.Int()) % testSetSize))
+				originalVal := internal[idx]
+
+				retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
+
+				if len(retrievedVal) != 0 {
+					if len(retrievedVal) != valueLen {
+						b.Errorf("Expected length %X for %v, got %X",
+							valueLen, idx, len(retrievedVal))
+						break
+					} else if !bytes.Equal(retrievedVal, originalVal) {
+						b.Errorf("Expected %v for %v, got %v",
+							originalVal, idx, retrievedVal)
+						break
+					}
+				}
+
+				if i != 0 && i%graphCountingPeriod == 0 {
+					endTime := time.Now().Sub(startTime).Seconds()
+					statics.PushBack(endTime)
+				}
+			}
+		})
+
+		printGraph(statics)
+		statics.Init()
+
+		// write and read
+		b.Run(dbName+"-write-read", func(b *testing.B) {
+			var i int
+			for i = 0; i < b.N; i++ {
+				if i != 0 && i%graphCountingPeriod == 0 {
+					startTime = time.Now()
+				}
+				idx = testKeyType(int64(rand.Int()) % testSetSize) // pick a random key
+				rand.Read(internal[idx])                           // generate a random data
+
+				dbInstance.Set(
+					int64ToBytes(testKeyType(idx)),
+					internal[idx],
+				)
+
+				originalVal := internal[idx]
+
+				retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
+
+				if len(retrievedVal) != 0 {
+					if len(retrievedVal) != valueLen {
+						b.Errorf("Expected length %X for %v, got %X",
+							valueLen, idx, len(retrievedVal))
+						break
+					} else if !bytes.Equal(retrievedVal, originalVal) {
+						b.Errorf("Expected %v for %v, got %v",
+							originalVal, idx, retrievedVal)
+						break
+					}
+				}
+
+				if i != 0 && i%graphCountingPeriod == 0 {
+					endTime := time.Now().Sub(startTime).Seconds()
+					statics.PushBack(endTime)
+				}
+			}
+			numOfWrite += int64(i)
+		})
+
+		printGraph(statics)
+		statics.Init()
+
+		// close
+		dbInstance.Close()
+
+		size, err := dirSizeKB(tmpDir)
+		if err != nil {
+			fmt.Println(err)
+		} else {
+			fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
+		}
+	}
+}
+
+func BenchmarkRandomBatchWR(b *testing.B) {
+
+	// generate a common random data set
+	internal := map[testKeyType][]byte{}
+	for i := 0; i < int(testSetSize); i++ {
+		// generate a random value
+		token := make([]byte, valueLen)
+		internal[testKeyType(i)] = token
+	}
+
+	for dbType, dbConstructors := range dbImpls {
+
+		// create db
+		dbName := string(dbType)
+		tmpDir, _ := ioutil.TempDir("", dbName)
+		defer os.RemoveAll(tmpDir)
+
+		dbInstance, _ := dbConstructors(tmpDir)
+		var numOfWrite int64
+		var idx testKeyType
+		var statics list.List
+		var startTime time.Time
+
+		fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
+			dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
+		fmt.Println("device type: " + deviceType)
+		fmt.Printf("batch size: %d\n", batchSize)
+
+		// write only
+		b.Run(dbName+"-batch-write", func(b *testing.B) {
+			var i int
+
+			for i = 0; i < b.N; i++ {
+				if i != 0 && i%graphCountingPeriod == 0 {
+					startTime = time.Now()
+				}
+				tx := dbInstance.NewTx()
+
+				for j := 0; j < batchSize; j++ {
+					idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
+					rand.Read(internal[idx])                             // generate a random data
+
+					tx.Set(
+						int64ToBytes(testKeyType(idx)),
+						internal[idx],
+					)
+				}
+				tx.Commit()
+				if i != 0 && i%graphCountingPeriod == 0 {
+					endTime := time.Now().Sub(startTime).Seconds()
+					statics.PushBack(endTime)
+				}
+			}
+			numOfWrite += int64(i)
+		})
+
+		// print a graph
+		printGraph(statics)
+		statics.Init()
+
+		// read only
+		b.Run(dbName+"-read", func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				if i != 0 && i%graphCountingPeriod == 0 {
+					startTime = time.Now()
+				}
+
+				idx = testKeyType((int64(rand.Int()) % testSetSize))
+				originalVal := internal[idx]
+
+				retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
+
+				if len(retrievedVal) != 0 {
+					if len(retrievedVal) != valueLen {
+						b.Errorf("Expected length %X for %v, got %X",
+							valueLen, idx, len(retrievedVal))
+						break
+					} else if !bytes.Equal(retrievedVal, originalVal) {
+						b.Errorf("Expected %v for %v, got %v",
+							originalVal, idx, retrievedVal)
+						break
+					}
+				}
+				if i != 0 && i%graphCountingPeriod == 0 {
+					endTime := time.Now().Sub(startTime).Seconds()
+					statics.PushBack(endTime)
+				}
+			}
+		})
+
+		// close
+		dbInstance.Close()
+
+		printGraph(statics)
+
+		size, err := dirSizeKB(tmpDir)
+		if err != nil {
+			fmt.Println(err)
+		} else {
+			fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
+		}
+	}
+}
+
+/*
+[Refined Test Result]
+goos: windows
+goarch: amd64
+pkg: github.com/aergoio/aergo-lib/db
+
+[badgerdb-ssd-single]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: ssd
+BenchmarkRandomWR/badgerdb-write-12 	            30000000	     29536 ns/op	    4983 B/op	      84 allocs/op
+BenchmarkRandomWR/badgerdb-read-12  	            20000000	     47739 ns/op	   13154 B/op	      60 allocs/op
+BenchmarkRandomWR/badgerdb-write-read-12         	20000000	     41990 ns/op	   10317 B/op	     140 allocs/op
+* Total size of badgerdb db: 14544473 kb, Size of 1 write: 286 byte
+
+[leveldb-ssd-single]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: ssd
+BenchmarkRandomWR/leveldb-write-12               	 1000000	   1006445 ns/op	     532 B/op	       5 allocs/op
+BenchmarkRandomWR/leveldb-read-12                	50000000	     18941 ns/op	    1160 B/op	      17 allocs/op
+BenchmarkRandomWR/leveldb-write-read-12          	 1000000	    986440 ns/op	    1272 B/op	      11 allocs/op
+* Total size of leveldb db: 567386 kb, Size of 1 write: 250 byte
+
+[badgerdb-ssd-batch100]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: ssd
+batch size: 100
+BenchmarkRandomBatchWR/badgerdb-batch-write-12   	 2000000	    612737 ns/op	  383751 B/op	    4576 allocs/op
+BenchmarkRandomBatchWR/badgerdb-read-12          	20000000	     53298 ns/op	   22177 B/op	      71 allocs/op
+* Total size of badgerdb db: 74546912 kb, Size of 1 write: 25359 byte
+
+[leveldb-ssd-batch100]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: ssd
+batch size: 100
+BenchmarkRandomBatchWR/leveldb-batch-write-12    	  300000	   6778458 ns/op	  127466 B/op	     399 allocs/op
+BenchmarkRandomBatchWR/leveldb-read-12           	 5000000	    141176 ns/op	    4547 B/op	      34 allocs/op
+* Total size of leveldb db: 2540271 kb, Size of 1 write: 8388 byte
+
+[badgerdb-ssd-batch1000]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: ssd
+batch size: 1000
+BenchmarkRandomBatchWR1000/badgerdb-batch-write-12         	  200000	   5507060 ns/op	 3504344 B/op	   42743 allocs/op
+BenchmarkRandomBatchWR1000/badgerdb-read-12                	20000000	     52636 ns/op	   21867 B/op	      73 allocs/op
+* Total size of badgerdb db: 51315516 kb, Size of 1 write: 250103 byte
+
+[leveldb-ssd-batch100]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: ssd
+batch size: 1000
+BenchmarkRandomBatchWR1000/leveldb-batch-write-12   	  	   20000	  70431430 ns/op	 1081956 B/op	    3906 allocs/op
+BenchmarkRandomBatchWR1000/leveldb-read-12                 	 5000000	    138074 ns/op	    4194 B/op	      33 allocs/op
+* Total size of leveldb db: 2529393 kb, Size of 1 write: 86046 byte
+
+[badgerdb-hdd-single]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: hdd
+BenchmarkHddRandomWR/badgerdb-write-12                     	30000000	     27901 ns/op	    4809 B/op	      82 allocs/op
+BenchmarkHddRandomWR/badgerdb-read-12                      	20000000	     45765 ns/op	   14321 B/op	      68 allocs/op
+BenchmarkHddRandomWR/badgerdb-write-read-12                 20000000	     39404 ns/op	    8693 B/op	     124 allocs/op
+* Total size of badgerdb db: 14697136 kb, Size of 1 write: 289 byte
+
+[leveldb-hdd-single]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: hdd
+BenchmarkHddRandomWR/leveldb-write-12                      	   50000	  20975160 ns/op	     551 B/op	       4 allocs/op
+BenchmarkHddRandomWR/leveldb-read-12                       100000000	     10375 ns/op	     988 B/op	      14 allocs/op
+BenchmarkHddRandomWR/leveldb-write-read-12                 	   50000	  21144257 ns/op	     994 B/op	       9 allocs/op
+* Total size of leveldb db: 31940 kb, Size of 1 write: 272 byte
+
+[badgerdb-hdd-batch100]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: hdd
+batch size: 100
+BenchmarkHddRandomBatchWR/badgerdb-batch-write-12          	 1000000	    675351 ns/op	  329398 B/op	    4007 allocs/op
+BenchmarkHddRandomBatchWR/badgerdb-read-12                 	10000000	     87933 ns/op	   22713 B/op	      85 allocs/op
+* Total size of badgerdb db: 24285947 kb, Size of 1 write: 24620 byte
+
+[leveldb-hdd-batch100]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: hdd
+batch size: 100
+BenchmarkHddRandomBatchWR/leveldb-batch-write-12           	   30000	  45769594 ns/op	  136783 B/op	     399 allocs/op
+BenchmarkHddRandomBatchWR/leveldb-read-12                  	20000000	     30579 ns/op	    3386 B/op	      19 allocs/op
+* Total size of leveldb db: 878403 kb, Size of 1 write: 22430 byte
+
+[badgerdb-hdd-batch1000]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: hdd
+batch size: 1000
+BenchmarkHddRandomBatchWR1000/badgerdb-batch-write-12      	  200000	   6599637 ns/op	 3724911 B/op	   43392 allocs/op
+BenchmarkHddRandomBatchWR1000/badgerdb-read-12             	10000000	     99201 ns/op	   31710 B/op	     105 allocs/op
+* Total size of badgerdb db: 51157857 kb, Size of 1 write: 249335 byte
+
+[leveldb-hdd-batch1000]
+testset_size: 10000000
+key_len: 8
+val_len: 256
+device type: hdd
+batch size: 1000
+BenchmarkHddRandomBatchWR1000/leveldb-batch-write-12       	   10000	 209360868 ns/op	 1027546 B/op	    3363 allocs/op
+BenchmarkHddRandomBatchWR1000/leveldb-read-12              	10000000	     87929 ns/op	    3971 B/op	      29 allocs/op
+* Total size of leveldb db: 1689120 kb, Size of 1 write: 171236 byte
+
+PASS
+ok  	github.com/aergoio/aergo-lib/db	33617.451s
+*/

db/types.go

@@ -0,0 +1,60 @@
+/**
+ *  @file
+ *  @copyright defined in aergo/LICENSE.txt
+ */
+
+package db
+
+// ImplType represents implementators of a DB interface
+type ImplType string
+
+const (
+	// LevelImpl represents a name of DB interface implementation using leveldb
+	LevelImpl ImplType = "leveldb"
+
+	// MemoryImpl represents a name of DB interface implementation in memory
+	MemoryImpl ImplType = "memorydb"
+)
+
+type dbConstructor func(dir string) (DB, error)
+
+// DB is an general interface to access at storage data
+type DB interface {
+	Type() string
+	Set(key, value []byte)
+	Delete(key []byte)
+	Get(key []byte) []byte
+	Exist(key []byte) bool
+	Iterator(start, end []byte) Iterator
+	NewTx() Transaction
+	NewBulk() Bulk
+	Close()
+	//Print()
+	//Stats() map[string]string
+}
+
+// Transaction is used to batch multiple operations
+type Transaction interface {
+	//	Get(key []byte) []byte
+	Set(key, value []byte)
+	Delete(key []byte)
+	Commit()
+	Discard()
+}
+
+// Bulk is used to batch multiple transactions
+// This will internally commit transactions when reach maximum tx size
+type Bulk interface {
+	Set(key, value []byte)
+	Delete(key []byte)
+	Flush()
+	DiscardLast()
+}
+
+// Iterator is used to navigate specific key ranges
+type Iterator interface {
+	Next()
+	Valid() bool
+	Key() []byte
+	Value() []byte
+}

go.mod

@@ -3,7 +3,11 @@ module github.com/p4u/asmt
 go 1.16
 
 require (
+	git.sr.ht/~sircmpwn/go-bare v0.0.0-20210227202403-5dae5c48f917
 	github.com/aergoio/aergo-lib v1.0.2
-	github.com/spf13/afero v1.2.1 // indirect
+	github.com/guptarohit/asciigraph v0.4.1
+	github.com/stretchr/testify v1.7.0
+	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
+	go.vocdoni.io/dvote v1.0.0
 	golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
 )

smt/trie.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/aergoio/aergo-lib/db"
+	"github.com/p4u/asmt/db"
 )
 
 // Trie is a modified sparse Merkle tree.
@@ -27,7 +27,7 @@ type Trie struct {
 	lock sync.RWMutex
 	// hash is the hash function used in the trie
 	hash func(data ...[]byte) []byte
-	// TrieHeight is the number if bits in a key
+	// TrieHeight is the number of bits in a key
 	TrieHeight int
 	// LoadDbCounter counts the nb of db reads in on update
 	LoadDbCounter int

smt/trie_cache.go

@@ -8,7 +8,7 @@ package trie
 import (
 	"sync"
 
-	"github.com/aergoio/aergo-lib/db"
+	"github.com/p4u/asmt/db"
 )
 
 // DbTx represents Set and Delete interface to store data

smt/trie_test.go

@@ -20,7 +20,7 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/aergoio/aergo-lib/db"
+	"github.com/p4u/asmt/db"
 )
 
 func TestTrieEmpty(t *testing.T) {
@@ -137,7 +137,7 @@ func TestTriePublicUpdateAndGet(t *testing.T) {
 
 func TestGetWithRoot(t *testing.T) {
 	dbPath := t.TempDir()
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 	smt := NewTrie(nil, Hasher, st)
 	smt.CacheHeightLimit = 0
 
@@ -199,7 +199,7 @@ func TestGetWithRoot(t *testing.T) {
 
 func TestTrieWalk(t *testing.T) {
 	dbPath := t.TempDir()
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 
 	smt := NewTrie(nil, Hasher, st)
 	smt.CacheHeightLimit = 0
@@ -455,7 +455,7 @@ func TestTrieCommit(t *testing.T) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 
 	smt := NewTrie(nil, Hasher, st)
 	keys := getFreshData(10, 32)
@@ -479,7 +479,7 @@ func TestTrieStageUpdates(t *testing.T) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 
 	smt := NewTrie(nil, Hasher, st)
 	keys := getFreshData(10, 32)
@@ -505,7 +505,7 @@ func TestTrieRevert(t *testing.T) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 
 	smt := NewTrie(nil, Hasher, st)
 
@@ -594,7 +594,7 @@ func TestTrieRaisesError(t *testing.T) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 
 	smt := NewTrie(nil, Hasher, st)
 	// Add data to empty trie
@@ -644,7 +644,7 @@ func TestTrieLoadCache(t *testing.T) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 
 	smt := NewTrie(nil, Hasher, st)
 	// Test size of cache
@@ -745,7 +745,7 @@ func TestStash(t *testing.T) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 	smt := NewTrie(nil, Hasher, st)
 	// Add data to empty trie
 	keys := getFreshData(20, 32)
@@ -836,7 +836,7 @@ func BenchmarkCacheHeightLimit233(b *testing.B) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 	smt := NewTrie(nil, Hasher, st)
 	smt.CacheHeightLimit = 233
 	benchmark10MAccounts10Ktps(smt, b)
@@ -848,7 +848,7 @@ func BenchmarkCacheHeightLimit238(b *testing.B) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 	smt := NewTrie(nil, Hasher, st)
 	smt.CacheHeightLimit = 238
 	benchmark10MAccounts10Ktps(smt, b)
@@ -860,7 +860,7 @@ func BenchmarkCacheHeightLimit245(b *testing.B) {
 	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
 		_ = os.MkdirAll(dbPath, 0711)
 	}
-	st := db.NewDB(db.BadgerImpl, dbPath)
+	st := db.NewDB(db.LevelImpl, dbPath)
 	smt := NewTrie(nil, Hasher, st)
 	smt.CacheHeightLimit = 245
 	benchmark10MAccounts10Ktps(smt, b)

smt/trie_tools.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/aergoio/aergo-lib/db"
+	"github.com/p4u/asmt/db"
 )
 
 // LoadCache loads the first layers of the merkle tree given a root