mirror of
https://github.com/arnaucube/asmt.git
synced 2026-02-06 19:06:46 +01:00
full refactor
- include the db package from aergo to this repository - move sparse merkle tree code to smt directory - add a new root package for being compatible with vocdoni/censusTree Signed-off-by: p4u <pau@dabax.net>
This commit is contained in:
p4u
420
censustree.go
Normal file
420
censustree.go
Normal file
@@ -0,0 +1,420 @@
|
||||
package asmtree
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.vocdoni.io/dvote/censustree"
|
||||
"go.vocdoni.io/dvote/log"
|
||||
|
||||
"git.sr.ht/~sircmpwn/go-bare"
|
||||
"github.com/p4u/asmt/db"
|
||||
asmt "github.com/p4u/asmt/smt"
|
||||
)
|
||||
|
||||
// We use go-bare for export/import the trie. In order to support
|
||||
// big census (up to 8 Million entries) we need to increase the maximums.
|
||||
const bareMaxArrayLength uint64 = 1024 * 1014 * 8 // 8 Million
|
||||
|
||||
const bareMaxUnmarshalBytes uint64 = 1024 * 1024 * 200 // 200 MiB
|
||||
|
||||
type Tree struct {
|
||||
Tree *asmt.Trie
|
||||
db db.DB
|
||||
public uint32
|
||||
lastAccessUnix int64 // a unix timestamp, used via sync/atomic
|
||||
size uint64
|
||||
snapshotRoot []byte // if not nil, this trie is considered an inmutable snapshot
|
||||
snapshotSize uint64
|
||||
}
|
||||
|
||||
type Proof struct {
|
||||
Bitmap []byte
|
||||
Length int
|
||||
Siblings [][]byte
|
||||
Value []byte
|
||||
}
|
||||
|
||||
type exportElement struct {
|
||||
Key []byte `bare:"key"`
|
||||
Value []byte `bare:"value"`
|
||||
}
|
||||
|
||||
type exportData struct {
|
||||
Elements []exportElement `bare:"elements"`
|
||||
}
|
||||
|
||||
const (
|
||||
MaxKeySize = 256
|
||||
MaxValueSize = 256
|
||||
dbRootPrefix = "this is the last root for the SMT tree"
|
||||
)
|
||||
|
||||
// NewTree initializes a new AergoSMT tree following the censustree.Tree interface specification.
|
||||
func NewTree(name, storageDir string) (censustree.Tree, error) {
|
||||
tr := &Tree{}
|
||||
err := tr.Init(name, storageDir)
|
||||
return tr, err
|
||||
}
|
||||
|
||||
// newTree opens or creates a merkle tree under the given storage.
|
||||
func newTree(name, storageDir string) (*asmt.Trie, db.DB, error) {
|
||||
dir := path.Join(storageDir, name)
|
||||
log.Debugf("creating new tree on %s", dir)
|
||||
d := db.NewDB(db.LevelImpl, dir)
|
||||
root := d.Get([]byte(dbRootPrefix))
|
||||
tr := asmt.NewTrie(root, asmt.Hasher, d)
|
||||
if root != nil {
|
||||
if err := tr.LoadCache(root); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return tr, d, nil
|
||||
}
|
||||
|
||||
// Init initializes a new asmt tree
|
||||
func (t *Tree) Init(name, storageDir string) error {
|
||||
var err error
|
||||
t.Tree, t.db, err = newTree(name, storageDir)
|
||||
t.updateAccessTime()
|
||||
t.size = 0
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *Tree) MaxKeySize() int {
|
||||
return MaxKeySize
|
||||
}
|
||||
|
||||
// LastAccess returns the last time the Tree was accessed, in the form of a unix
|
||||
// timestamp.
|
||||
func (t *Tree) LastAccess() int64 {
|
||||
return atomic.LoadInt64(&t.lastAccessUnix)
|
||||
}
|
||||
|
||||
func (t *Tree) updateAccessTime() {
|
||||
atomic.StoreInt64(&t.lastAccessUnix, time.Now().Unix())
|
||||
}
|
||||
|
||||
// Publish makes a merkle tree available for queries.
|
||||
// Application layer should check IsPublish() before considering the Tree available.
|
||||
func (t *Tree) Publish() {
|
||||
atomic.StoreUint32(&t.public, 1)
|
||||
}
|
||||
|
||||
// UnPublish makes a merkle tree not available for queries
|
||||
func (t *Tree) UnPublish() {
|
||||
atomic.StoreUint32(&t.public, 0)
|
||||
}
|
||||
|
||||
// IsPublic returns true if the tree is available
|
||||
func (t *Tree) IsPublic() bool {
|
||||
return atomic.LoadUint32(&t.public) == 1
|
||||
}
|
||||
|
||||
// Commit saves permanently the tree on disk
|
||||
func (t *Tree) Commit() error {
|
||||
if t.snapshotRoot != nil {
|
||||
return fmt.Errorf("cannot commit to a snapshot trie")
|
||||
}
|
||||
err := t.Tree.Commit()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.db.Set([]byte(dbRootPrefix), t.Root())
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add adds a new claim to the merkle tree
|
||||
// A claim is composed of two parts: index and value
|
||||
// 1.index is mandatory, the data will be used for indexing the claim into to merkle tree
|
||||
// 2.value is optional, the data will not affect the indexing
|
||||
func (t *Tree) Add(index, value []byte) error {
|
||||
t.updateAccessTime()
|
||||
if t.snapshotRoot != nil {
|
||||
return fmt.Errorf("cannot add to a snapshot trie")
|
||||
}
|
||||
if len(index) < 4 {
|
||||
return fmt.Errorf("index too small (%d), minimum size is 4 bytes", len(index))
|
||||
}
|
||||
if len(value) > MaxValueSize {
|
||||
return fmt.Errorf("index or value claim data too big")
|
||||
}
|
||||
_, err := t.Tree.Update([][]byte{asmt.Hasher(index)}, [][]byte{asmt.Hasher(value)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
atomic.StoreUint64(&t.size, 0) // TBD: improve this
|
||||
return t.Commit()
|
||||
}
|
||||
|
||||
// AddBatch adds a list of indexes and values.
|
||||
// The commit to disk is executed only once.
|
||||
// The values slince could be empty or as long as indexes.
|
||||
func (t *Tree) AddBatch(indexes, values [][]byte) ([]int, error) {
|
||||
var wrongIndexes []int
|
||||
t.updateAccessTime()
|
||||
if t.snapshotRoot != nil {
|
||||
return wrongIndexes, fmt.Errorf("cannot add to a snapshot trie")
|
||||
}
|
||||
if len(values) > 0 && len(indexes) != len(values) {
|
||||
return wrongIndexes, fmt.Errorf("indexes and values have different size")
|
||||
}
|
||||
var hashedIndexes [][]byte
|
||||
var hashedValues [][]byte
|
||||
var value []byte
|
||||
for i, key := range indexes {
|
||||
if len(key) < 4 {
|
||||
wrongIndexes = append(wrongIndexes, i)
|
||||
continue
|
||||
}
|
||||
value = nil
|
||||
if len(values) > 0 {
|
||||
if len(values[i]) > MaxValueSize {
|
||||
wrongIndexes = append(wrongIndexes, i)
|
||||
continue
|
||||
}
|
||||
value = values[i]
|
||||
}
|
||||
hashedIndexes = append(hashedIndexes, asmt.Hasher(key))
|
||||
hashedValues = append(hashedValues, asmt.Hasher(value))
|
||||
}
|
||||
_, err := t.Tree.Update(hashedIndexes, hashedValues)
|
||||
if err != nil {
|
||||
return wrongIndexes, err
|
||||
}
|
||||
atomic.StoreUint64(&t.size, 0) // TBD: improve this
|
||||
return wrongIndexes, t.Commit()
|
||||
}
|
||||
|
||||
// Get returns the value of a key
|
||||
func (t *Tree) Get(key []byte) []byte { // Do something with error
|
||||
var value []byte
|
||||
if t.snapshotRoot != nil {
|
||||
value, _ = t.Tree.GetWithRoot(key, t.snapshotRoot)
|
||||
} else {
|
||||
value, _ = t.Tree.Get(key)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// GenProof generates a merkle tree proof that can be later used on CheckProof() to validate it.
|
||||
func (t *Tree) GenProof(index, value []byte) ([]byte, error) {
|
||||
t.updateAccessTime()
|
||||
var err error
|
||||
var ap [][]byte
|
||||
var pvalue []byte
|
||||
var bitmap []byte
|
||||
var length int
|
||||
if t.snapshotRoot != nil {
|
||||
bitmap, ap, length, _, _, pvalue, err = t.Tree.MerkleProofCompressedR(
|
||||
asmt.Hasher(index),
|
||||
t.snapshotRoot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
bitmap, ap, length, _, _, pvalue, err = t.Tree.MerkleProofCompressed(
|
||||
asmt.Hasher(index))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
//if !included {
|
||||
// return nil, fmt.Errorf("not included")
|
||||
//}
|
||||
if !bytes.Equal(pvalue, asmt.Hasher(value)) {
|
||||
return nil, fmt.Errorf("incorrect value on genProof")
|
||||
}
|
||||
return bare.Marshal(&Proof{Bitmap: bitmap, Length: length, Siblings: ap, Value: pvalue})
|
||||
}
|
||||
|
||||
// CheckProof validates a merkle proof and its data.
|
||||
func (t *Tree) CheckProof(index, value, root, mproof []byte) (bool, error) {
|
||||
t.updateAccessTime()
|
||||
p := Proof{}
|
||||
if err := bare.Unmarshal(mproof, &p); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !bytes.Equal(p.Value, asmt.Hasher(value)) {
|
||||
return false, fmt.Errorf("values mismatch %x != %x", p.Value, asmt.Hasher(value))
|
||||
}
|
||||
if root != nil {
|
||||
return t.Tree.VerifyInclusionWithRootC(
|
||||
root,
|
||||
p.Bitmap,
|
||||
asmt.Hasher(index),
|
||||
p.Value,
|
||||
p.Siblings,
|
||||
p.Length), nil
|
||||
}
|
||||
if t.snapshotRoot != nil {
|
||||
return t.Tree.VerifyInclusionWithRootC(
|
||||
t.snapshotRoot,
|
||||
p.Bitmap,
|
||||
asmt.Hasher(index),
|
||||
p.Value,
|
||||
p.Siblings,
|
||||
p.Length), nil
|
||||
}
|
||||
return t.Tree.VerifyInclusionC(
|
||||
p.Bitmap,
|
||||
asmt.Hasher(index),
|
||||
p.Value,
|
||||
p.Siblings,
|
||||
p.Length), nil
|
||||
}
|
||||
|
||||
// Root returns the current root hash of the merkle tree
|
||||
func (t *Tree) Root() []byte {
|
||||
t.updateAccessTime()
|
||||
if t.snapshotRoot != nil {
|
||||
return t.snapshotRoot
|
||||
}
|
||||
return t.Tree.Root
|
||||
}
|
||||
|
||||
// Dump returns the whole merkle tree serialized in a format that can be used on Import.
|
||||
// Byte seralization is performed using bare message protocol, it is a 40% size win over JSON.
|
||||
func (t *Tree) Dump(root []byte) ([]byte, error) {
|
||||
t.updateAccessTime()
|
||||
if root == nil && t.snapshotRoot != nil {
|
||||
root = t.snapshotRoot
|
||||
}
|
||||
dump := exportData{}
|
||||
t.iterateWithRoot(root, nil, func(k, v []byte) bool {
|
||||
ee := exportElement{Key: make([]byte, len(k)), Value: make([]byte, len(v))}
|
||||
// Copy elements since it's not safe to hold on to the []byte values from Iterate
|
||||
copy(ee.Key, k[:])
|
||||
copy(ee.Value, v[:])
|
||||
dump.Elements = append(dump.Elements, ee)
|
||||
return false
|
||||
})
|
||||
bare.MaxArrayLength(bareMaxArrayLength)
|
||||
bare.MaxUnmarshalBytes(bareMaxUnmarshalBytes)
|
||||
|
||||
return bare.Marshal(&dump)
|
||||
}
|
||||
|
||||
// String returns a human readable representation of the tree.
|
||||
func (t *Tree) String() string {
|
||||
s := bytes.Buffer{}
|
||||
t.iterate(t.snapshotRoot, func(k, v []byte) bool {
|
||||
s.WriteString(fmt.Sprintf("%x => %x\n", k, v))
|
||||
return false
|
||||
})
|
||||
return s.String()
|
||||
}
|
||||
|
||||
// Size returns the number of leaf nodes on the merkle tree.
|
||||
// TO-DO: root is currently ignored
|
||||
func (t *Tree) Size(root []byte) (int64, error) {
|
||||
if t.snapshotRoot != nil {
|
||||
return int64(t.snapshotSize), nil
|
||||
}
|
||||
return int64(t.count()), nil
|
||||
}
|
||||
|
||||
// DumpPlain returns the entire list of added claims for a specific root hash.
|
||||
// First return parametre are the indexes and second the values.
|
||||
// If root is not specified, the last one is used.
|
||||
func (t *Tree) DumpPlain(root []byte) ([][]byte, [][]byte, error) {
|
||||
var indexes, values [][]byte
|
||||
var err error
|
||||
t.updateAccessTime()
|
||||
|
||||
t.iterateWithRoot(root, nil, func(k, v []byte) bool {
|
||||
indexes = append(indexes, k)
|
||||
values = append(values, v)
|
||||
return false
|
||||
})
|
||||
|
||||
return indexes, values, err
|
||||
}
|
||||
|
||||
// ImportDump imports a partial or whole tree previously exported with Dump()
|
||||
func (t *Tree) ImportDump(data []byte) error {
|
||||
t.updateAccessTime()
|
||||
if t.snapshotRoot != nil {
|
||||
return fmt.Errorf("cannot import to a snapshot")
|
||||
}
|
||||
census := new(exportData)
|
||||
bare.MaxArrayLength(bareMaxArrayLength)
|
||||
bare.MaxUnmarshalBytes(bareMaxUnmarshalBytes)
|
||||
if err := bare.Unmarshal(data, census); err != nil {
|
||||
return fmt.Errorf("importdump cannot unmarshal data: %w", err)
|
||||
}
|
||||
keys := [][]byte{}
|
||||
values := [][]byte{}
|
||||
for _, ee := range census.Elements {
|
||||
keys = append(keys, ee.Key)
|
||||
values = append(values, ee.Value)
|
||||
}
|
||||
_, err := t.Tree.Update(keys, values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
atomic.StoreUint64(&t.size, 0) // TBD: improve this
|
||||
return t.Commit()
|
||||
}
|
||||
|
||||
// Snapshot returns a Tree instance of a exiting merkle root.
|
||||
// A Snapshot cannot be modified.
|
||||
func (t *Tree) Snapshot(root []byte) (censustree.Tree, error) {
|
||||
exist, err := t.HashExists(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exist {
|
||||
return nil, fmt.Errorf("root %x does not exist, cannot build snapshot", root)
|
||||
}
|
||||
return &Tree{Tree: t.Tree, public: t.public, snapshotRoot: root, snapshotSize: t.count()}, nil
|
||||
}
|
||||
|
||||
func (t *Tree) Close() error {
|
||||
t.db.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// HashExists checks if a hash exists as a node in the merkle tree
|
||||
func (t *Tree) HashExists(hash []byte) (bool, error) {
|
||||
t.updateAccessTime()
|
||||
return t.Tree.TrieRootExists(hash), nil
|
||||
}
|
||||
|
||||
func (t *Tree) count() uint64 {
|
||||
if v := atomic.LoadUint64(&t.size); v != 0 {
|
||||
return v
|
||||
}
|
||||
counter := uint64(0)
|
||||
if err := t.Tree.Walk(t.snapshotRoot, func(*asmt.WalkResult) int32 {
|
||||
counter++
|
||||
return 0
|
||||
}); err != nil {
|
||||
return 0
|
||||
}
|
||||
atomic.StoreUint64(&t.size, counter)
|
||||
return counter
|
||||
}
|
||||
|
||||
func (t *Tree) iterate(prefix []byte, callback func(key, value []byte) bool) {
|
||||
t.Tree.Walk(t.snapshotRoot, func(v *asmt.WalkResult) int32 {
|
||||
if callback(v.Key, v.Value) {
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Tree) iterateWithRoot(root, prefix []byte, callback func(key, value []byte) bool) {
|
||||
t.Tree.Walk(root, func(v *asmt.WalkResult) int32 {
|
||||
if callback(v.Key, v.Value) {
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
}
|
||||
90
censustree_test.go
Normal file
90
censustree_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package asmtree
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTree(t *testing.T) {
|
||||
censusSize := 1000
|
||||
storage := t.TempDir()
|
||||
tr1 := &Tree{}
|
||||
err := tr1.Init("test1", storage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < censusSize; i++ {
|
||||
if err = tr1.Add([]byte(fmt.Sprintf("number %d", i)),
|
||||
[]byte(fmt.Sprintf("number %d value", i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
root1 := tr1.Root()
|
||||
data, err := tr1.Dump(root1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("dumped data size is: %d bytes", len(data))
|
||||
|
||||
tr2 := &Tree{}
|
||||
err = tr2.Init("test2", storage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = tr2.ImportDump(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root2 := tr2.Root()
|
||||
if !bytes.Equal(root1, root2) {
|
||||
t.Errorf("roots are different but they should be equal (%x != %x)", root1, root2)
|
||||
}
|
||||
|
||||
// Try closing the storage and creating the tree again
|
||||
tr2.Close()
|
||||
err = tr2.Init("test2", storage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Get the size
|
||||
s, err := tr2.Size(nil)
|
||||
if err != nil {
|
||||
t.Errorf("cannot get te size of the tree after reopen: (%s)", err)
|
||||
}
|
||||
if s != int64(censusSize) {
|
||||
t.Errorf("Size is wrong (have %d, expexted %d)", s, censusSize)
|
||||
}
|
||||
|
||||
// Check Root is still the same
|
||||
if !bytes.Equal(tr2.Root(), root2) {
|
||||
t.Fatalf("after closing and opening the tree, the root is different")
|
||||
}
|
||||
|
||||
// Generate a proof on tr1 and check validity on snapshot and tr2
|
||||
proof1, err := tr1.GenProof([]byte("number 5"), []byte("number 5 value"))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
t.Logf("Proof Length: %d", len(proof1))
|
||||
tr1s, err := tr1.Snapshot(root1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
valid, err := tr1s.CheckProof([]byte("number 5"), []byte("number 5 value"), root1, proof1)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !valid {
|
||||
t.Errorf("proof is invalid on snapshot")
|
||||
}
|
||||
valid, err = tr2.CheckProof([]byte("number 5"), []byte("number 5 value"), nil, proof1)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !valid {
|
||||
t.Errorf("proof is invalid on tree2")
|
||||
}
|
||||
|
||||
}
|
||||
87
db/db.go
Normal file
87
db/db.go
Normal file
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
/*
|
||||
Package db is an wrapper of key-value database implementations. Currently, this supports badgerdb (https://github.com/dgraph-io/badger).
|
||||
|
||||
Basic Usage
|
||||
|
||||
You can create database using a newdb func like this
|
||||
database := NewDB(BadgerImpl, "./test")
|
||||
|
||||
A first argument is a backend db type to use, and a second is a root directory to store db files.
|
||||
After creating db, you can write, read or delete single key-value using funcs in DB interface.
|
||||
// write data
|
||||
database.Set([]byte("key"), []byte("val"))
|
||||
|
||||
// read data
|
||||
read := Get([]byte("key"))
|
||||
|
||||
// delete data
|
||||
database.Delete([]byte("key"))
|
||||
|
||||
Transaction
|
||||
|
||||
A Transaction is a bulk set of operations to ensure atomic success or fail.
|
||||
// create a new transaction
|
||||
tx := database.NewTX(true)
|
||||
|
||||
// reserve writing
|
||||
tx.Set([]byte("keyA"), []byte("valA"))
|
||||
tx.Set([]byte("keyB"), []byte("valB"))
|
||||
|
||||
// Get will return a value reserved to write in this transaction
|
||||
mustBeValA := tx.Get([]byte("keyA"))
|
||||
|
||||
// Perform writing
|
||||
tx.Commit()
|
||||
If you want to cancel and discard operations in tx, then you must call Discard() func to prevent a memory leack
|
||||
// If you create a tx, but do not commit, than you have to call this
|
||||
tx.Discard()
|
||||
|
||||
Iterator
|
||||
|
||||
An iteractor provides a way to get all keys sequentially.
|
||||
// create an iterator that covers all range
|
||||
for iter := database.Iterator(nil, nil); iter.Valid(); iter.Next() {
|
||||
// print each key-value pair
|
||||
fmt.Printf("%s = %s", string(iter.Key()), string(iter.Value()))
|
||||
}
|
||||
|
||||
You can find more detail usages at a db_test.go file
|
||||
*/
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/aergoio/aergo-lib/log"
|
||||
)
|
||||
|
||||
var dbImpls = map[ImplType]dbConstructor{}
|
||||
var logger *extendedLog
|
||||
|
||||
func registorDBConstructor(dbimpl ImplType, constructor dbConstructor) {
|
||||
dbImpls[dbimpl] = constructor
|
||||
}
|
||||
|
||||
// NewDB creates new database or load existing database in the directory
|
||||
func NewDB(dbimpltype ImplType, dir string) DB {
|
||||
logger = &extendedLog{Logger: log.NewLogger("db")}
|
||||
db, err := dbImpls[dbimpltype](dir)
|
||||
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Fail to Create New DB: %v", err))
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
|
||||
func convNilToBytes(byteArray []byte) []byte {
|
||||
if byteArray == nil {
|
||||
return []byte{}
|
||||
}
|
||||
return byteArray
|
||||
}
|
||||
276
db/db_test.go
Normal file
276
db/db_test.go
Normal file
@@ -0,0 +1,276 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
tmpDbTestKey1 = "tempkey1"
|
||||
tmpDbTestKey2 = "tempkey2"
|
||||
tmpDbTestStrVal1 = "val1"
|
||||
tmpDbTestStrVal2 = "val2"
|
||||
tmpDbTestIntVal1 = 1
|
||||
tmpDbTestIntVal2 = 2
|
||||
)
|
||||
|
||||
func createTmpDB(key ImplType) (dir string, db DB) {
|
||||
dir, err := ioutil.TempDir("", string(key))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
db = NewDB(key, dir)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func setInitData(db DB) {
|
||||
tx := db.NewTx()
|
||||
|
||||
tx.Set([]byte("1"), []byte("1"))
|
||||
tx.Set([]byte("2"), []byte("2"))
|
||||
tx.Set([]byte("3"), []byte("3"))
|
||||
tx.Set([]byte("4"), []byte("4"))
|
||||
tx.Set([]byte("5"), []byte("5"))
|
||||
tx.Set([]byte("6"), []byte("6"))
|
||||
tx.Set([]byte("7"), []byte("7"))
|
||||
|
||||
tx.Commit()
|
||||
}
|
||||
|
||||
func TestGetSetDeleteExist(t *testing.T) {
|
||||
// for each db implementation
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// initial value of empty key must be empty byte
|
||||
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
|
||||
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
// set value
|
||||
db.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
|
||||
|
||||
// check value set
|
||||
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
|
||||
assert.True(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
// delete value
|
||||
db.Delete([]byte(tmpDbTestKey1))
|
||||
|
||||
// value must be erased
|
||||
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
|
||||
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionSet(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
|
||||
// set the value in the tx
|
||||
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
|
||||
// the value will not visible at a db
|
||||
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
tx.Commit()
|
||||
|
||||
// after commit, the value visible from the db
|
||||
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionDiscard(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
// discard test
|
||||
tx = db.NewTx()
|
||||
// set the value in the tx
|
||||
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal2))
|
||||
|
||||
// discard tx
|
||||
tx.Discard()
|
||||
|
||||
assert.Panics(t, func() { tx.Commit() }, "commit after discard is not allowed")
|
||||
|
||||
// after discard, the value must be reset at the db
|
||||
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionDelete(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
|
||||
// set the value in the tx
|
||||
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
|
||||
|
||||
// delete the value in the tx
|
||||
tx.Delete([]byte(tmpDbTestKey1))
|
||||
|
||||
tx.Commit()
|
||||
|
||||
// after commit, chekc the value from the db
|
||||
assert.Equal(t, "", string(db.Get([]byte(tmpDbTestKey1))), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionCommitTwice(t *testing.T) {
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
|
||||
// a first commit will success
|
||||
tx.Commit()
|
||||
|
||||
// a second commit will cause panic
|
||||
assert.Panics(t, func() { tx.Commit() })
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBulk(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new Bulk instance
|
||||
bulk := db.NewBulk()
|
||||
|
||||
// set the huge number of value in the bulk
|
||||
for i := 0; i < 1000000; i++ {
|
||||
bulk.Set([]byte(fmt.Sprintf("key%d", i)),
|
||||
[]byte(tmpDbTestStrVal1))
|
||||
}
|
||||
|
||||
bulk.Flush()
|
||||
|
||||
// after commit, the value visible from the db
|
||||
|
||||
for i := 0; i < 1000000; i++ {
|
||||
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(fmt.Sprintf("key%d", i)))), db.Type())
|
||||
}
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIter(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
setInitData(db)
|
||||
|
||||
i := 1
|
||||
|
||||
for iter := db.Iterator(nil, nil); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
i++
|
||||
}
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRangeIter(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
setInitData(db)
|
||||
|
||||
// test iteration 2 -> 5
|
||||
i := 2
|
||||
for iter := db.Iterator([]byte("2"), []byte("5")); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i++
|
||||
}
|
||||
assert.EqualValues(t, i, 5)
|
||||
|
||||
// nil sames with []byte("0")
|
||||
// test iteration 0 -> 5
|
||||
i = 1
|
||||
for iter := db.Iterator(nil, []byte("5")); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i++
|
||||
}
|
||||
assert.EqualValues(t, i, 5)
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReverseIter(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
setInitData(db)
|
||||
|
||||
// test reverse iteration 5 <- 2
|
||||
i := 5
|
||||
for iter := db.Iterator([]byte("5"), []byte("2")); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i--
|
||||
}
|
||||
assert.EqualValues(t, i, 2)
|
||||
|
||||
// nil sames with []byte("0")
|
||||
// test reverse iteration 5 -> 0
|
||||
i = 5
|
||||
for iter := db.Iterator([]byte("5"), nil); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i--
|
||||
}
|
||||
assert.EqualValues(t, i, 0)
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
322
db/leveldb.go
Normal file
322
db/leveldb.go
Normal file
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
// This function is always called first
|
||||
func init() {
|
||||
dbConstructor := func(dir string) (DB, error) {
|
||||
return newLevelDB(dir)
|
||||
}
|
||||
registorDBConstructor(LevelImpl, dbConstructor)
|
||||
}
|
||||
|
||||
func newLevelDB(dir string) (DB, error) {
|
||||
dbPath := filepath.Join(dir, "data.db")
|
||||
|
||||
db, err := leveldb.OpenFile(dbPath, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
database := &levelDB{
|
||||
db: db,
|
||||
}
|
||||
return database, nil
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// DB Implementation
|
||||
//=========================================================
|
||||
|
||||
// Enforce database and transaction implements interfaces
|
||||
var _ DB = (*levelDB)(nil)
|
||||
|
||||
type levelDB struct {
|
||||
db *leveldb.DB
|
||||
}
|
||||
|
||||
func (db *levelDB) Type() string {
|
||||
return "leveldb"
|
||||
}
|
||||
|
||||
func (db *levelDB) Set(key, value []byte) {
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
|
||||
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
func (db *levelDB) Delete(key []byte) {
|
||||
key = convNilToBytes(key)
|
||||
|
||||
err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
|
||||
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
func (db *levelDB) Get(key []byte) []byte {
|
||||
key = convNilToBytes(key)
|
||||
res, err := db.db.Get(key, nil)
|
||||
if err != nil {
|
||||
if err == errors.ErrNotFound {
|
||||
return []byte{}
|
||||
}
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (db *levelDB) Exist(key []byte) bool {
|
||||
res, _ := db.db.Has(key, nil)
|
||||
return res
|
||||
}
|
||||
|
||||
func (db *levelDB) Close() {
|
||||
db.db.Close()
|
||||
}
|
||||
|
||||
func (db *levelDB) NewTx() Transaction {
|
||||
batch := new(leveldb.Batch)
|
||||
|
||||
return &levelTransaction{db, batch, false, false}
|
||||
}
|
||||
|
||||
func (db *levelDB) NewBulk() Bulk {
|
||||
batch := new(leveldb.Batch)
|
||||
|
||||
return &levelBulk{db, batch, false, false}
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Transaction Implementation
|
||||
//=========================================================
|
||||
|
||||
type levelTransaction struct {
|
||||
db *levelDB
|
||||
tx *leveldb.Batch
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
/*
|
||||
func (transaction *levelTransaction) Get(key []byte) []byte {
|
||||
panic(fmt.Sprintf("DO not support"))
|
||||
}
|
||||
*/
|
||||
|
||||
func (transaction *levelTransaction) Set(key, value []byte) {
|
||||
transaction.tx.Put(key, value)
|
||||
}
|
||||
|
||||
func (transaction *levelTransaction) Delete(key []byte) {
|
||||
transaction.tx.Delete(key)
|
||||
}
|
||||
|
||||
func (transaction *levelTransaction) Commit() {
|
||||
if transaction.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if transaction.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
err := transaction.db.db.Write(transaction.tx, &opt.WriteOptions{Sync: true})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
transaction.isCommit = true
|
||||
}
|
||||
|
||||
func (transaction *levelTransaction) Discard() {
|
||||
transaction.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Bulk Implementation
|
||||
//=========================================================
|
||||
|
||||
type levelBulk struct {
|
||||
db *levelDB
|
||||
tx *leveldb.Batch
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) Set(key, value []byte) {
|
||||
bulk.tx.Put(key, value)
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) Delete(key []byte) {
|
||||
bulk.tx.Delete(key)
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) Flush() {
|
||||
// do the same behavior that a transaction commit does
|
||||
// db.write internally will handle large transaction
|
||||
if bulk.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if bulk.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
|
||||
err := bulk.db.db.Write(bulk.tx, &opt.WriteOptions{Sync: true})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
bulk.isCommit = true
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) DiscardLast() {
|
||||
bulk.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Iterator Implementation
|
||||
//=========================================================
|
||||
|
||||
type levelIterator struct {
|
||||
start []byte
|
||||
end []byte
|
||||
reverse bool
|
||||
iter iterator.Iterator
|
||||
isInvalid bool
|
||||
}
|
||||
|
||||
func (db *levelDB) Iterator(start, end []byte) Iterator {
|
||||
var reverse bool
|
||||
|
||||
// if end is bigger then start, then reverse order
|
||||
if bytes.Compare(start, end) == 1 {
|
||||
reverse = true
|
||||
} else {
|
||||
reverse = false
|
||||
}
|
||||
|
||||
iter := db.db.NewIterator(nil, nil)
|
||||
|
||||
if reverse {
|
||||
if start == nil {
|
||||
iter.Last()
|
||||
} else {
|
||||
valid := iter.Seek(start)
|
||||
if valid {
|
||||
soakey := iter.Key()
|
||||
if bytes.Compare(start, soakey) < 0 {
|
||||
iter.Prev()
|
||||
}
|
||||
} else {
|
||||
iter.Last()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if start == nil {
|
||||
iter.First()
|
||||
} else {
|
||||
iter.Seek(start)
|
||||
}
|
||||
}
|
||||
return &levelIterator{
|
||||
iter: iter,
|
||||
start: start,
|
||||
end: end,
|
||||
reverse: reverse,
|
||||
isInvalid: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Next() {
|
||||
if iter.Valid() {
|
||||
if iter.reverse {
|
||||
iter.iter.Prev()
|
||||
} else {
|
||||
iter.iter.Next()
|
||||
}
|
||||
} else {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Valid() bool {
|
||||
|
||||
// Once invalid, forever invalid.
|
||||
if iter.isInvalid {
|
||||
return false
|
||||
}
|
||||
|
||||
// Panic on DB error. No way to recover.
|
||||
if err := iter.iter.Error(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// If source is invalid, invalid.
|
||||
if !iter.iter.Valid() {
|
||||
iter.isInvalid = true
|
||||
return false
|
||||
}
|
||||
|
||||
// If key is end or past it, invalid.
|
||||
var end = iter.end
|
||||
var key = iter.iter.Key()
|
||||
|
||||
if iter.reverse {
|
||||
if end != nil && bytes.Compare(key, end) <= 0 {
|
||||
iter.isInvalid = true
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
if end != nil && bytes.Compare(end, key) <= 0 {
|
||||
iter.isInvalid = true
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Valid
|
||||
return true
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Key() (key []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is invalid")
|
||||
} else if err := iter.iter.Error(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
originalKey := iter.iter.Key()
|
||||
|
||||
key = make([]byte, len(originalKey))
|
||||
copy(key, originalKey)
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Value() (value []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is invalid")
|
||||
} else if err := iter.iter.Error(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
originalValue := iter.iter.Value()
|
||||
|
||||
value = make([]byte, len(originalValue))
|
||||
copy(value, originalValue)
|
||||
|
||||
return value
|
||||
}
|
||||
26
db/logger.go
Normal file
26
db/logger.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"github.com/aergoio/aergo-lib/log"
|
||||
)
|
||||
|
||||
type extendedLog struct {
|
||||
*log.Logger
|
||||
}
|
||||
|
||||
func (l *extendedLog) Errorf(f string, v ...interface{}) {
|
||||
l.Error().Msgf(f, v...)
|
||||
}
|
||||
|
||||
func (l *extendedLog) Warningf(f string, v ...interface{}) {
|
||||
l.Warn().Msgf(f, v...)
|
||||
}
|
||||
|
||||
func (l *extendedLog) Infof(f string, v ...interface{}) {
|
||||
// reduce info to debug level because infos at badgerdb are too detail
|
||||
l.Debug().Msgf(f, v...) // INFO -> DEBUG
|
||||
}
|
||||
|
||||
func (l *extendedLog) Debugf(f string, v ...interface{}) {
|
||||
l.Debug().Msgf(f, v...)
|
||||
}
|
||||
384
db/memorydb.go
Normal file
384
db/memorydb.go
Normal file
@@ -0,0 +1,384 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"encoding/gob"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// This function is always called first
|
||||
func init() {
|
||||
dbConstructor := func(dir string) (DB, error) {
|
||||
return newMemoryDB(dir)
|
||||
}
|
||||
registorDBConstructor(MemoryImpl, dbConstructor)
|
||||
}
|
||||
|
||||
func newMemoryDB(dir string) (DB, error) {
|
||||
var db map[string][]byte
|
||||
|
||||
filePath := path.Join(dir, "database")
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err == nil {
|
||||
decoder := gob.NewDecoder(file) //
|
||||
err = decoder.Decode(&db)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
file.Close()
|
||||
|
||||
if db == nil {
|
||||
db = make(map[string][]byte)
|
||||
}
|
||||
|
||||
database := &memorydb{
|
||||
db: db,
|
||||
dir: filePath,
|
||||
}
|
||||
|
||||
return database, nil
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// DB Implementation
|
||||
//=========================================================
|
||||
|
||||
// Enforce database and transaction implements interfaces
|
||||
var _ DB = (*memorydb)(nil)
|
||||
|
||||
type memorydb struct {
|
||||
lock sync.Mutex
|
||||
db map[string][]byte
|
||||
dir string
|
||||
}
|
||||
|
||||
func (db *memorydb) Type() string {
|
||||
return "memorydb"
|
||||
}
|
||||
|
||||
func (db *memorydb) Set(key, value []byte) {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
db.db[string(key)] = value
|
||||
}
|
||||
|
||||
func (db *memorydb) Delete(key []byte) {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
delete(db.db, string(key))
|
||||
}
|
||||
|
||||
func (db *memorydb) Get(key []byte) []byte {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
return db.db[string(key)]
|
||||
}
|
||||
|
||||
func (db *memorydb) Exist(key []byte) bool {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
_, ok := db.db[string(key)]
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
func (db *memorydb) Close() {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
file, err := os.OpenFile(db.dir, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
if err == nil {
|
||||
encoder := gob.NewEncoder(file)
|
||||
encoder.Encode(db.db)
|
||||
}
|
||||
file.Close()
|
||||
}
|
||||
|
||||
func (db *memorydb) NewTx() Transaction {
|
||||
|
||||
return &memoryTransaction{
|
||||
db: db,
|
||||
opList: list.New(),
|
||||
isDiscard: false,
|
||||
isCommit: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (db *memorydb) NewBulk() Bulk {
|
||||
|
||||
return &memoryBulk{
|
||||
db: db,
|
||||
opList: list.New(),
|
||||
isDiscard: false,
|
||||
isCommit: false,
|
||||
}
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Transaction Implementation
|
||||
//=========================================================
|
||||
|
||||
type memoryTransaction struct {
|
||||
txLock sync.Mutex
|
||||
db *memorydb
|
||||
opList *list.List
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
type txOp struct {
|
||||
isSet bool
|
||||
key []byte
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Set(key, value []byte) {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
transaction.opList.PushBack(&txOp{true, key, value})
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Delete(key []byte) {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
transaction.opList.PushBack(&txOp{false, key, nil})
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Commit() {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
if transaction.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if transaction.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
|
||||
db := transaction.db
|
||||
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
for e := transaction.opList.Front(); e != nil; e = e.Next() {
|
||||
op := e.Value.(*txOp)
|
||||
if op.isSet {
|
||||
db.db[string(op.key)] = op.value
|
||||
} else {
|
||||
delete(db.db, string(op.key))
|
||||
}
|
||||
}
|
||||
|
||||
transaction.isCommit = true
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Discard() {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
transaction.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Bulk Implementation
|
||||
//=========================================================
|
||||
|
||||
type memoryBulk struct {
|
||||
txLock sync.Mutex
|
||||
db *memorydb
|
||||
opList *list.List
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) Set(key, value []byte) {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
bulk.opList.PushBack(&txOp{true, key, value})
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) Delete(key []byte) {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
bulk.opList.PushBack(&txOp{false, key, nil})
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) Flush() {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
if bulk.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if bulk.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
|
||||
db := bulk.db
|
||||
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
for e := bulk.opList.Front(); e != nil; e = e.Next() {
|
||||
op := e.Value.(*txOp)
|
||||
if op.isSet {
|
||||
db.db[string(op.key)] = op.value
|
||||
} else {
|
||||
delete(db.db, string(op.key))
|
||||
}
|
||||
}
|
||||
|
||||
bulk.isCommit = true
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) DiscardLast() {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
bulk.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Iterator Implementation
|
||||
//=========================================================
|
||||
|
||||
type memoryIterator struct {
|
||||
start []byte
|
||||
end []byte
|
||||
reverse bool
|
||||
keys []string
|
||||
isInvalid bool
|
||||
cursor int
|
||||
db *memorydb
|
||||
}
|
||||
|
||||
func isKeyInRange(key []byte, start []byte, end []byte, reverse bool) bool {
|
||||
if reverse {
|
||||
if start != nil && bytes.Compare(start, key) < 0 {
|
||||
return false
|
||||
}
|
||||
if end != nil && bytes.Compare(key, end) <= 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if bytes.Compare(key, start) < 0 {
|
||||
return false
|
||||
}
|
||||
if end != nil && bytes.Compare(end, key) <= 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
|
||||
}
|
||||
|
||||
func (db *memorydb) Iterator(start, end []byte) Iterator {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
var reverse bool
|
||||
|
||||
// if end is bigger then start, then reverse order
|
||||
if bytes.Compare(start, end) == 1 {
|
||||
reverse = true
|
||||
} else {
|
||||
reverse = false
|
||||
}
|
||||
|
||||
var keys sort.StringSlice
|
||||
|
||||
for key := range db.db {
|
||||
if isKeyInRange([]byte(key), start, end, reverse) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
if reverse {
|
||||
sort.Sort(sort.Reverse(keys))
|
||||
} else {
|
||||
sort.Strings(keys)
|
||||
}
|
||||
|
||||
return &memoryIterator{
|
||||
start: start,
|
||||
end: end,
|
||||
reverse: reverse,
|
||||
isInvalid: false,
|
||||
keys: keys,
|
||||
cursor: 0,
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Next() {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
|
||||
iter.cursor++
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Valid() bool {
|
||||
// Once invalid, forever invalid.
|
||||
if iter.isInvalid {
|
||||
return false
|
||||
}
|
||||
|
||||
return 0 <= iter.cursor && iter.cursor < len(iter.keys)
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Key() (key []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
|
||||
return []byte(iter.keys[iter.cursor])
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Value() (value []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
|
||||
key := []byte(iter.keys[iter.cursor])
|
||||
|
||||
return iter.db.Get(key)
|
||||
}
|
||||
443
db/perf_test.go
Normal file
443
db/perf_test.go
Normal file
@@ -0,0 +1,443 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/guptarohit/asciigraph"
|
||||
)
|
||||
|
||||
type testKeyType int64
|
||||
|
||||
// Simple execution cmd
|
||||
// go test -run=XXX -bench=. -benchmem -cpuprofile=cpu.out -memprofile=mem.out -timeout 20m -benchtime=1m
|
||||
|
||||
var valueLen = 256
|
||||
var testSetSize = int64(10000000)
|
||||
var batchSize = 100
|
||||
|
||||
// parameters for drawing graph
|
||||
var graphCountingPeriod = 10000
|
||||
var graphWidth = 120
|
||||
var graphHeigh = 20
|
||||
|
||||
var deviceType = "ssd"
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().Unix())
|
||||
}
|
||||
|
||||
func int64ToBytes(i testKeyType) []byte {
|
||||
buf := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(buf, uint64(i))
|
||||
return buf
|
||||
}
|
||||
|
||||
func printGraph(statics list.List) {
|
||||
if statics.Len() == 0 {
|
||||
return
|
||||
}
|
||||
// convert list to slice
|
||||
var staticsSlice = make([]float64, statics.Len())
|
||||
i := 0
|
||||
for e := statics.Front(); e != nil; e = e.Next() {
|
||||
staticsSlice[i] = e.Value.(float64)
|
||||
i++
|
||||
}
|
||||
|
||||
graph := asciigraph.Plot(staticsSlice, asciigraph.Width(graphWidth), asciigraph.Height(graphHeigh))
|
||||
fmt.Println(graph)
|
||||
}
|
||||
|
||||
func dirSizeKB(path string) (int64, error) {
|
||||
var size int64
|
||||
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
if !info.IsDir() {
|
||||
size += info.Size()
|
||||
}
|
||||
return err
|
||||
})
|
||||
return size / 1024, err
|
||||
}
|
||||
|
||||
func BenchmarkRandomWR(b *testing.B) {
|
||||
|
||||
// generate a common random data set
|
||||
internal := map[testKeyType][]byte{}
|
||||
for i := 0; i < int(testSetSize); i++ {
|
||||
// generate a random value
|
||||
token := make([]byte, valueLen)
|
||||
internal[testKeyType(i)] = token
|
||||
}
|
||||
|
||||
for dbType, dbConstructors := range dbImpls {
|
||||
|
||||
// create db
|
||||
dbName := string(dbType)
|
||||
tmpDir, _ := ioutil.TempDir("", dbName)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbInstance, _ := dbConstructors(tmpDir)
|
||||
var numOfWrite int64
|
||||
var idx testKeyType
|
||||
var statics list.List
|
||||
var startTime time.Time
|
||||
|
||||
fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
|
||||
dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
|
||||
fmt.Println("device type: ssd")
|
||||
|
||||
// write only
|
||||
b.Run(dbName+"-write", func(b *testing.B) {
|
||||
var i int
|
||||
for i = 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
|
||||
rand.Read(internal[idx]) // generate a random data
|
||||
|
||||
dbInstance.Set(
|
||||
int64ToBytes(testKeyType(idx)),
|
||||
internal[idx],
|
||||
)
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
numOfWrite += int64(i)
|
||||
})
|
||||
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// read only
|
||||
b.Run(dbName+"-read", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize))
|
||||
originalVal := internal[idx]
|
||||
|
||||
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
|
||||
|
||||
if len(retrievedVal) != 0 {
|
||||
if len(retrievedVal) != valueLen {
|
||||
b.Errorf("Expected length %X for %v, got %X",
|
||||
valueLen, idx, len(retrievedVal))
|
||||
break
|
||||
} else if !bytes.Equal(retrievedVal, originalVal) {
|
||||
b.Errorf("Expected %v for %v, got %v",
|
||||
originalVal, idx, retrievedVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// write and read
|
||||
b.Run(dbName+"-write-read", func(b *testing.B) {
|
||||
var i int
|
||||
for i = 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
idx = testKeyType(int64(rand.Int()) % testSetSize) // pick a random key
|
||||
rand.Read(internal[idx]) // generate a random data
|
||||
|
||||
dbInstance.Set(
|
||||
int64ToBytes(testKeyType(idx)),
|
||||
internal[idx],
|
||||
)
|
||||
|
||||
originalVal := internal[idx]
|
||||
|
||||
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
|
||||
|
||||
if len(retrievedVal) != 0 {
|
||||
if len(retrievedVal) != valueLen {
|
||||
b.Errorf("Expected length %X for %v, got %X",
|
||||
valueLen, idx, len(retrievedVal))
|
||||
break
|
||||
} else if !bytes.Equal(retrievedVal, originalVal) {
|
||||
b.Errorf("Expected %v for %v, got %v",
|
||||
originalVal, idx, retrievedVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
numOfWrite += int64(i)
|
||||
})
|
||||
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// close
|
||||
dbInstance.Close()
|
||||
|
||||
size, err := dirSizeKB(tmpDir)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
} else {
|
||||
fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRandomBatchWR(b *testing.B) {
|
||||
|
||||
// generate a common random data set
|
||||
internal := map[testKeyType][]byte{}
|
||||
for i := 0; i < int(testSetSize); i++ {
|
||||
// generate a random value
|
||||
token := make([]byte, valueLen)
|
||||
internal[testKeyType(i)] = token
|
||||
}
|
||||
|
||||
for dbType, dbConstructors := range dbImpls {
|
||||
|
||||
// create db
|
||||
dbName := string(dbType)
|
||||
tmpDir, _ := ioutil.TempDir("", dbName)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbInstance, _ := dbConstructors(tmpDir)
|
||||
var numOfWrite int64
|
||||
var idx testKeyType
|
||||
var statics list.List
|
||||
var startTime time.Time
|
||||
|
||||
fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
|
||||
dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
|
||||
fmt.Println("device type: " + deviceType)
|
||||
fmt.Printf("batch size: %d\n", batchSize)
|
||||
|
||||
// write only
|
||||
b.Run(dbName+"-batch-write", func(b *testing.B) {
|
||||
var i int
|
||||
|
||||
for i = 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
tx := dbInstance.NewTx()
|
||||
|
||||
for j := 0; j < batchSize; j++ {
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
|
||||
rand.Read(internal[idx]) // generate a random data
|
||||
|
||||
tx.Set(
|
||||
int64ToBytes(testKeyType(idx)),
|
||||
internal[idx],
|
||||
)
|
||||
}
|
||||
tx.Commit()
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
numOfWrite += int64(i)
|
||||
})
|
||||
|
||||
// print a graph
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// read only
|
||||
b.Run(dbName+"-read", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize))
|
||||
originalVal := internal[idx]
|
||||
|
||||
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
|
||||
|
||||
if len(retrievedVal) != 0 {
|
||||
if len(retrievedVal) != valueLen {
|
||||
b.Errorf("Expected length %X for %v, got %X",
|
||||
valueLen, idx, len(retrievedVal))
|
||||
break
|
||||
} else if !bytes.Equal(retrievedVal, originalVal) {
|
||||
b.Errorf("Expected %v for %v, got %v",
|
||||
originalVal, idx, retrievedVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// close
|
||||
dbInstance.Close()
|
||||
|
||||
printGraph(statics)
|
||||
|
||||
size, err := dirSizeKB(tmpDir)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
} else {
|
||||
fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
[Refined Test Result]
|
||||
goos: windows
|
||||
goarch: amd64
|
||||
pkg: github.com/aergoio/aergo-lib/db
|
||||
|
||||
[badgerdb-ssd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
BenchmarkRandomWR/badgerdb-write-12 30000000 29536 ns/op 4983 B/op 84 allocs/op
|
||||
BenchmarkRandomWR/badgerdb-read-12 20000000 47739 ns/op 13154 B/op 60 allocs/op
|
||||
BenchmarkRandomWR/badgerdb-write-read-12 20000000 41990 ns/op 10317 B/op 140 allocs/op
|
||||
* Total size of badgerdb db: 14544473 kb, Size of 1 write: 286 byte
|
||||
|
||||
[leveldb-ssd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
BenchmarkRandomWR/leveldb-write-12 1000000 1006445 ns/op 532 B/op 5 allocs/op
|
||||
BenchmarkRandomWR/leveldb-read-12 50000000 18941 ns/op 1160 B/op 17 allocs/op
|
||||
BenchmarkRandomWR/leveldb-write-read-12 1000000 986440 ns/op 1272 B/op 11 allocs/op
|
||||
* Total size of leveldb db: 567386 kb, Size of 1 write: 250 byte
|
||||
|
||||
[badgerdb-ssd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 100
|
||||
BenchmarkRandomBatchWR/badgerdb-batch-write-12 2000000 612737 ns/op 383751 B/op 4576 allocs/op
|
||||
BenchmarkRandomBatchWR/badgerdb-read-12 20000000 53298 ns/op 22177 B/op 71 allocs/op
|
||||
* Total size of badgerdb db: 74546912 kb, Size of 1 write: 25359 byte
|
||||
|
||||
[leveldb-ssd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 100
|
||||
BenchmarkRandomBatchWR/leveldb-batch-write-12 300000 6778458 ns/op 127466 B/op 399 allocs/op
|
||||
BenchmarkRandomBatchWR/leveldb-read-12 5000000 141176 ns/op 4547 B/op 34 allocs/op
|
||||
* Total size of leveldb db: 2540271 kb, Size of 1 write: 8388 byte
|
||||
|
||||
[badgerdb-ssd-batch1000]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 1000
|
||||
BenchmarkRandomBatchWR1000/badgerdb-batch-write-12 200000 5507060 ns/op 3504344 B/op 42743 allocs/op
|
||||
BenchmarkRandomBatchWR1000/badgerdb-read-12 20000000 52636 ns/op 21867 B/op 73 allocs/op
|
||||
* Total size of badgerdb db: 51315516 kb, Size of 1 write: 250103 byte
|
||||
|
||||
[leveldb-ssd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 1000
|
||||
BenchmarkRandomBatchWR1000/leveldb-batch-write-12 20000 70431430 ns/op 1081956 B/op 3906 allocs/op
|
||||
BenchmarkRandomBatchWR1000/leveldb-read-12 5000000 138074 ns/op 4194 B/op 33 allocs/op
|
||||
* Total size of leveldb db: 2529393 kb, Size of 1 write: 86046 byte
|
||||
|
||||
[badgerdb-hdd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
BenchmarkHddRandomWR/badgerdb-write-12 30000000 27901 ns/op 4809 B/op 82 allocs/op
|
||||
BenchmarkHddRandomWR/badgerdb-read-12 20000000 45765 ns/op 14321 B/op 68 allocs/op
|
||||
BenchmarkHddRandomWR/badgerdb-write-read-12 20000000 39404 ns/op 8693 B/op 124 allocs/op
|
||||
* Total size of badgerdb db: 14697136 kb, Size of 1 write: 289 byte
|
||||
|
||||
[leveldb-hdd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
BenchmarkHddRandomWR/leveldb-write-12 50000 20975160 ns/op 551 B/op 4 allocs/op
|
||||
BenchmarkHddRandomWR/leveldb-read-12 100000000 10375 ns/op 988 B/op 14 allocs/op
|
||||
BenchmarkHddRandomWR/leveldb-write-read-12 50000 21144257 ns/op 994 B/op 9 allocs/op
|
||||
* Total size of leveldb db: 31940 kb, Size of 1 write: 272 byte
|
||||
|
||||
[badgerdb-hdd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 100
|
||||
BenchmarkHddRandomBatchWR/badgerdb-batch-write-12 1000000 675351 ns/op 329398 B/op 4007 allocs/op
|
||||
BenchmarkHddRandomBatchWR/badgerdb-read-12 10000000 87933 ns/op 22713 B/op 85 allocs/op
|
||||
* Total size of badgerdb db: 24285947 kb, Size of 1 write: 24620 byte
|
||||
|
||||
[leveldb-hdd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 100
|
||||
BenchmarkHddRandomBatchWR/leveldb-batch-write-12 30000 45769594 ns/op 136783 B/op 399 allocs/op
|
||||
BenchmarkHddRandomBatchWR/leveldb-read-12 20000000 30579 ns/op 3386 B/op 19 allocs/op
|
||||
* Total size of leveldb db: 878403 kb, Size of 1 write: 22430 byte
|
||||
|
||||
[badgerdb-hdd-batch1000]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 1000
|
||||
BenchmarkHddRandomBatchWR1000/badgerdb-batch-write-12 200000 6599637 ns/op 3724911 B/op 43392 allocs/op
|
||||
BenchmarkHddRandomBatchWR1000/badgerdb-read-12 10000000 99201 ns/op 31710 B/op 105 allocs/op
|
||||
* Total size of badgerdb db: 51157857 kb, Size of 1 write: 249335 byte
|
||||
|
||||
[leveldb-hdd-batch1000]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 1000
|
||||
BenchmarkHddRandomBatchWR1000/leveldb-batch-write-12 10000 209360868 ns/op 1027546 B/op 3363 allocs/op
|
||||
BenchmarkHddRandomBatchWR1000/leveldb-read-12 10000000 87929 ns/op 3971 B/op 29 allocs/op
|
||||
* Total size of leveldb db: 1689120 kb, Size of 1 write: 171236 byte
|
||||
|
||||
PASS
|
||||
ok github.com/aergoio/aergo-lib/db 33617.451s
|
||||
*/
|
||||
60
db/types.go
Normal file
60
db/types.go
Normal file
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
package db
|
||||
|
||||
// ImplType represents implementators of a DB interface
|
||||
type ImplType string
|
||||
|
||||
const (
|
||||
// LevelImpl represents a name of DB interface implementation using leveldb
|
||||
LevelImpl ImplType = "leveldb"
|
||||
|
||||
// MemoryImpl represents a name of DB interface implementation in memory
|
||||
MemoryImpl ImplType = "memorydb"
|
||||
)
|
||||
|
||||
type dbConstructor func(dir string) (DB, error)
|
||||
|
||||
// DB is an general interface to access at storage data
|
||||
type DB interface {
|
||||
Type() string
|
||||
Set(key, value []byte)
|
||||
Delete(key []byte)
|
||||
Get(key []byte) []byte
|
||||
Exist(key []byte) bool
|
||||
Iterator(start, end []byte) Iterator
|
||||
NewTx() Transaction
|
||||
NewBulk() Bulk
|
||||
Close()
|
||||
//Print()
|
||||
//Stats() map[string]string
|
||||
}
|
||||
|
||||
// Transaction is used to batch multiple operations
|
||||
type Transaction interface {
|
||||
// Get(key []byte) []byte
|
||||
Set(key, value []byte)
|
||||
Delete(key []byte)
|
||||
Commit()
|
||||
Discard()
|
||||
}
|
||||
|
||||
// Bulk is used to batch multiple transactions
|
||||
// This will internally commit transactions when reach maximum tx size
|
||||
type Bulk interface {
|
||||
Set(key, value []byte)
|
||||
Delete(key []byte)
|
||||
Flush()
|
||||
DiscardLast()
|
||||
}
|
||||
|
||||
// Iterator is used to navigate specific key ranges
|
||||
type Iterator interface {
|
||||
Next()
|
||||
Valid() bool
|
||||
Key() []byte
|
||||
Value() []byte
|
||||
}
|
||||
6
go.mod
6
go.mod
@@ -3,7 +3,11 @@ module github.com/p4u/asmt
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
git.sr.ht/~sircmpwn/go-bare v0.0.0-20210227202403-5dae5c48f917
|
||||
github.com/aergoio/aergo-lib v1.0.2
|
||||
github.com/spf13/afero v1.2.1 // indirect
|
||||
github.com/guptarohit/asciigraph v0.4.1
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
|
||||
go.vocdoni.io/dvote v1.0.0
|
||||
golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
|
||||
)
|
||||
|
||||
|
Before Width: | Height: | Size: 68 KiB After Width: | Height: | Size: 68 KiB |
|
Before Width: | Height: | Size: 85 KiB After Width: | Height: | Size: 85 KiB |
|
Before Width: | Height: | Size: 75 KiB After Width: | Height: | Size: 75 KiB |
|
Before Width: | Height: | Size: 68 KiB After Width: | Height: | Size: 68 KiB |
@@ -10,7 +10,7 @@ import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/aergoio/aergo-lib/db"
|
||||
"github.com/p4u/asmt/db"
|
||||
)
|
||||
|
||||
// Trie is a modified sparse Merkle tree.
|
||||
@@ -27,7 +27,7 @@ type Trie struct {
|
||||
lock sync.RWMutex
|
||||
// hash is the hash function used in the trie
|
||||
hash func(data ...[]byte) []byte
|
||||
// TrieHeight is the number if bits in a key
|
||||
// TrieHeight is the number of bits in a key
|
||||
TrieHeight int
|
||||
// LoadDbCounter counts the nb of db reads in on update
|
||||
LoadDbCounter int
|
||||
@@ -8,7 +8,7 @@ package trie
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/aergoio/aergo-lib/db"
|
||||
"github.com/p4u/asmt/db"
|
||||
)
|
||||
|
||||
// DbTx represents Set and Delete interface to store data
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/aergoio/aergo-lib/db"
|
||||
"github.com/p4u/asmt/db"
|
||||
)
|
||||
|
||||
func TestTrieEmpty(t *testing.T) {
|
||||
@@ -137,7 +137,7 @@ func TestTriePublicUpdateAndGet(t *testing.T) {
|
||||
|
||||
func TestGetWithRoot(t *testing.T) {
|
||||
dbPath := t.TempDir()
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
smt.CacheHeightLimit = 0
|
||||
|
||||
@@ -199,7 +199,7 @@ func TestGetWithRoot(t *testing.T) {
|
||||
|
||||
func TestTrieWalk(t *testing.T) {
|
||||
dbPath := t.TempDir()
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
smt.CacheHeightLimit = 0
|
||||
@@ -455,7 +455,7 @@ func TestTrieCommit(t *testing.T) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
keys := getFreshData(10, 32)
|
||||
@@ -479,7 +479,7 @@ func TestTrieStageUpdates(t *testing.T) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
keys := getFreshData(10, 32)
|
||||
@@ -505,7 +505,7 @@ func TestTrieRevert(t *testing.T) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
|
||||
@@ -594,7 +594,7 @@ func TestTrieRaisesError(t *testing.T) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
// Add data to empty trie
|
||||
@@ -644,7 +644,7 @@ func TestTrieLoadCache(t *testing.T) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
// Test size of cache
|
||||
@@ -745,7 +745,7 @@ func TestStash(t *testing.T) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
// Add data to empty trie
|
||||
keys := getFreshData(20, 32)
|
||||
@@ -836,7 +836,7 @@ func BenchmarkCacheHeightLimit233(b *testing.B) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
smt.CacheHeightLimit = 233
|
||||
benchmark10MAccounts10Ktps(smt, b)
|
||||
@@ -848,7 +848,7 @@ func BenchmarkCacheHeightLimit238(b *testing.B) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
smt.CacheHeightLimit = 238
|
||||
benchmark10MAccounts10Ktps(smt, b)
|
||||
@@ -860,7 +860,7 @@ func BenchmarkCacheHeightLimit245(b *testing.B) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dbPath, 0711)
|
||||
}
|
||||
st := db.NewDB(db.BadgerImpl, dbPath)
|
||||
st := db.NewDB(db.LevelImpl, dbPath)
|
||||
smt := NewTrie(nil, Hasher, st)
|
||||
smt.CacheHeightLimit = 245
|
||||
benchmark10MAccounts10Ktps(smt, b)
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/aergoio/aergo-lib/db"
|
||||
"github.com/p4u/asmt/db"
|
||||
)
|
||||
|
||||
// LoadCache loads the first layers of the merkle tree given a root
|
||||
Reference in New Issue
Block a user