mirror of
https://github.com/arnaucube/asmt.git
synced 2026-02-06 19:06:46 +01:00
full refactor
- include the db package from aergo to this repository - move sparse merkle tree code to smt directory - add a new root package for being compatible with vocdoni/censusTree Signed-off-by: p4u <pau@dabax.net>
This commit is contained in:
p4u
87
db/db.go
Normal file
87
db/db.go
Normal file
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
/*
|
||||
Package db is an wrapper of key-value database implementations. Currently, this supports badgerdb (https://github.com/dgraph-io/badger).
|
||||
|
||||
Basic Usage
|
||||
|
||||
You can create database using a newdb func like this
|
||||
database := NewDB(BadgerImpl, "./test")
|
||||
|
||||
A first argument is a backend db type to use, and a second is a root directory to store db files.
|
||||
After creating db, you can write, read or delete single key-value using funcs in DB interface.
|
||||
// write data
|
||||
database.Set([]byte("key"), []byte("val"))
|
||||
|
||||
// read data
|
||||
read := Get([]byte("key"))
|
||||
|
||||
// delete data
|
||||
database.Delete([]byte("key"))
|
||||
|
||||
Transaction
|
||||
|
||||
A Transaction is a bulk set of operations to ensure atomic success or fail.
|
||||
// create a new transaction
|
||||
tx := database.NewTX(true)
|
||||
|
||||
// reserve writing
|
||||
tx.Set([]byte("keyA"), []byte("valA"))
|
||||
tx.Set([]byte("keyB"), []byte("valB"))
|
||||
|
||||
// Get will return a value reserved to write in this transaction
|
||||
mustBeValA := tx.Get([]byte("keyA"))
|
||||
|
||||
// Perform writing
|
||||
tx.Commit()
|
||||
If you want to cancel and discard operations in tx, then you must call Discard() func to prevent a memory leack
|
||||
// If you create a tx, but do not commit, than you have to call this
|
||||
tx.Discard()
|
||||
|
||||
Iterator
|
||||
|
||||
An iteractor provides a way to get all keys sequentially.
|
||||
// create an iterator that covers all range
|
||||
for iter := database.Iterator(nil, nil); iter.Valid(); iter.Next() {
|
||||
// print each key-value pair
|
||||
fmt.Printf("%s = %s", string(iter.Key()), string(iter.Value()))
|
||||
}
|
||||
|
||||
You can find more detail usages at a db_test.go file
|
||||
*/
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/aergoio/aergo-lib/log"
|
||||
)
|
||||
|
||||
var dbImpls = map[ImplType]dbConstructor{}
|
||||
var logger *extendedLog
|
||||
|
||||
func registorDBConstructor(dbimpl ImplType, constructor dbConstructor) {
|
||||
dbImpls[dbimpl] = constructor
|
||||
}
|
||||
|
||||
// NewDB creates new database or load existing database in the directory
|
||||
func NewDB(dbimpltype ImplType, dir string) DB {
|
||||
logger = &extendedLog{Logger: log.NewLogger("db")}
|
||||
db, err := dbImpls[dbimpltype](dir)
|
||||
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Fail to Create New DB: %v", err))
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
|
||||
func convNilToBytes(byteArray []byte) []byte {
|
||||
if byteArray == nil {
|
||||
return []byte{}
|
||||
}
|
||||
return byteArray
|
||||
}
|
||||
276
db/db_test.go
Normal file
276
db/db_test.go
Normal file
@@ -0,0 +1,276 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const (
|
||||
tmpDbTestKey1 = "tempkey1"
|
||||
tmpDbTestKey2 = "tempkey2"
|
||||
tmpDbTestStrVal1 = "val1"
|
||||
tmpDbTestStrVal2 = "val2"
|
||||
tmpDbTestIntVal1 = 1
|
||||
tmpDbTestIntVal2 = 2
|
||||
)
|
||||
|
||||
func createTmpDB(key ImplType) (dir string, db DB) {
|
||||
dir, err := ioutil.TempDir("", string(key))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
db = NewDB(key, dir)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func setInitData(db DB) {
|
||||
tx := db.NewTx()
|
||||
|
||||
tx.Set([]byte("1"), []byte("1"))
|
||||
tx.Set([]byte("2"), []byte("2"))
|
||||
tx.Set([]byte("3"), []byte("3"))
|
||||
tx.Set([]byte("4"), []byte("4"))
|
||||
tx.Set([]byte("5"), []byte("5"))
|
||||
tx.Set([]byte("6"), []byte("6"))
|
||||
tx.Set([]byte("7"), []byte("7"))
|
||||
|
||||
tx.Commit()
|
||||
}
|
||||
|
||||
func TestGetSetDeleteExist(t *testing.T) {
|
||||
// for each db implementation
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// initial value of empty key must be empty byte
|
||||
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
|
||||
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
// set value
|
||||
db.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
|
||||
|
||||
// check value set
|
||||
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
|
||||
assert.True(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
// delete value
|
||||
db.Delete([]byte(tmpDbTestKey1))
|
||||
|
||||
// value must be erased
|
||||
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
|
||||
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionSet(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
|
||||
// set the value in the tx
|
||||
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
|
||||
// the value will not visible at a db
|
||||
assert.Empty(t, db.Get([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
tx.Commit()
|
||||
|
||||
// after commit, the value visible from the db
|
||||
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(tmpDbTestKey1))), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionDiscard(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
// discard test
|
||||
tx = db.NewTx()
|
||||
// set the value in the tx
|
||||
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal2))
|
||||
|
||||
// discard tx
|
||||
tx.Discard()
|
||||
|
||||
assert.Panics(t, func() { tx.Commit() }, "commit after discard is not allowed")
|
||||
|
||||
// after discard, the value must be reset at the db
|
||||
assert.False(t, db.Exist([]byte(tmpDbTestKey1)), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionDelete(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
|
||||
// set the value in the tx
|
||||
tx.Set([]byte(tmpDbTestKey1), []byte(tmpDbTestStrVal1))
|
||||
|
||||
// delete the value in the tx
|
||||
tx.Delete([]byte(tmpDbTestKey1))
|
||||
|
||||
tx.Commit()
|
||||
|
||||
// after commit, chekc the value from the db
|
||||
assert.Equal(t, "", string(db.Get([]byte(tmpDbTestKey1))), db.Type())
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionCommitTwice(t *testing.T) {
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new writable tx
|
||||
tx := db.NewTx()
|
||||
|
||||
// a first commit will success
|
||||
tx.Commit()
|
||||
|
||||
// a second commit will cause panic
|
||||
assert.Panics(t, func() { tx.Commit() })
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBulk(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
// create a new Bulk instance
|
||||
bulk := db.NewBulk()
|
||||
|
||||
// set the huge number of value in the bulk
|
||||
for i := 0; i < 1000000; i++ {
|
||||
bulk.Set([]byte(fmt.Sprintf("key%d", i)),
|
||||
[]byte(tmpDbTestStrVal1))
|
||||
}
|
||||
|
||||
bulk.Flush()
|
||||
|
||||
// after commit, the value visible from the db
|
||||
|
||||
for i := 0; i < 1000000; i++ {
|
||||
assert.Equal(t, tmpDbTestStrVal1, string(db.Get([]byte(fmt.Sprintf("key%d", i)))), db.Type())
|
||||
}
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIter(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
setInitData(db)
|
||||
|
||||
i := 1
|
||||
|
||||
for iter := db.Iterator(nil, nil); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
i++
|
||||
}
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRangeIter(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
setInitData(db)
|
||||
|
||||
// test iteration 2 -> 5
|
||||
i := 2
|
||||
for iter := db.Iterator([]byte("2"), []byte("5")); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i++
|
||||
}
|
||||
assert.EqualValues(t, i, 5)
|
||||
|
||||
// nil sames with []byte("0")
|
||||
// test iteration 0 -> 5
|
||||
i = 1
|
||||
for iter := db.Iterator(nil, []byte("5")); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i++
|
||||
}
|
||||
assert.EqualValues(t, i, 5)
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReverseIter(t *testing.T) {
|
||||
|
||||
for key := range dbImpls {
|
||||
dir, db := createTmpDB(key)
|
||||
|
||||
setInitData(db)
|
||||
|
||||
// test reverse iteration 5 <- 2
|
||||
i := 5
|
||||
for iter := db.Iterator([]byte("5"), []byte("2")); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i--
|
||||
}
|
||||
assert.EqualValues(t, i, 2)
|
||||
|
||||
// nil sames with []byte("0")
|
||||
// test reverse iteration 5 -> 0
|
||||
i = 5
|
||||
for iter := db.Iterator([]byte("5"), nil); iter.Valid(); iter.Next() {
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Key()))
|
||||
assert.EqualValues(t, strconv.Itoa(i), string(iter.Value()))
|
||||
i--
|
||||
}
|
||||
assert.EqualValues(t, i, 0)
|
||||
|
||||
db.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
322
db/leveldb.go
Normal file
322
db/leveldb.go
Normal file
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
// This function is always called first
|
||||
func init() {
|
||||
dbConstructor := func(dir string) (DB, error) {
|
||||
return newLevelDB(dir)
|
||||
}
|
||||
registorDBConstructor(LevelImpl, dbConstructor)
|
||||
}
|
||||
|
||||
func newLevelDB(dir string) (DB, error) {
|
||||
dbPath := filepath.Join(dir, "data.db")
|
||||
|
||||
db, err := leveldb.OpenFile(dbPath, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
database := &levelDB{
|
||||
db: db,
|
||||
}
|
||||
return database, nil
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// DB Implementation
|
||||
//=========================================================
|
||||
|
||||
// Enforce database and transaction implements interfaces
|
||||
var _ DB = (*levelDB)(nil)
|
||||
|
||||
type levelDB struct {
|
||||
db *leveldb.DB
|
||||
}
|
||||
|
||||
func (db *levelDB) Type() string {
|
||||
return "leveldb"
|
||||
}
|
||||
|
||||
func (db *levelDB) Set(key, value []byte) {
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
|
||||
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
func (db *levelDB) Delete(key []byte) {
|
||||
key = convNilToBytes(key)
|
||||
|
||||
err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
|
||||
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
func (db *levelDB) Get(key []byte) []byte {
|
||||
key = convNilToBytes(key)
|
||||
res, err := db.db.Get(key, nil)
|
||||
if err != nil {
|
||||
if err == errors.ErrNotFound {
|
||||
return []byte{}
|
||||
}
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (db *levelDB) Exist(key []byte) bool {
|
||||
res, _ := db.db.Has(key, nil)
|
||||
return res
|
||||
}
|
||||
|
||||
func (db *levelDB) Close() {
|
||||
db.db.Close()
|
||||
}
|
||||
|
||||
func (db *levelDB) NewTx() Transaction {
|
||||
batch := new(leveldb.Batch)
|
||||
|
||||
return &levelTransaction{db, batch, false, false}
|
||||
}
|
||||
|
||||
func (db *levelDB) NewBulk() Bulk {
|
||||
batch := new(leveldb.Batch)
|
||||
|
||||
return &levelBulk{db, batch, false, false}
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Transaction Implementation
|
||||
//=========================================================
|
||||
|
||||
type levelTransaction struct {
|
||||
db *levelDB
|
||||
tx *leveldb.Batch
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
/*
|
||||
func (transaction *levelTransaction) Get(key []byte) []byte {
|
||||
panic(fmt.Sprintf("DO not support"))
|
||||
}
|
||||
*/
|
||||
|
||||
func (transaction *levelTransaction) Set(key, value []byte) {
|
||||
transaction.tx.Put(key, value)
|
||||
}
|
||||
|
||||
func (transaction *levelTransaction) Delete(key []byte) {
|
||||
transaction.tx.Delete(key)
|
||||
}
|
||||
|
||||
func (transaction *levelTransaction) Commit() {
|
||||
if transaction.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if transaction.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
err := transaction.db.db.Write(transaction.tx, &opt.WriteOptions{Sync: true})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
transaction.isCommit = true
|
||||
}
|
||||
|
||||
func (transaction *levelTransaction) Discard() {
|
||||
transaction.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Bulk Implementation
|
||||
//=========================================================
|
||||
|
||||
type levelBulk struct {
|
||||
db *levelDB
|
||||
tx *leveldb.Batch
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) Set(key, value []byte) {
|
||||
bulk.tx.Put(key, value)
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) Delete(key []byte) {
|
||||
bulk.tx.Delete(key)
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) Flush() {
|
||||
// do the same behavior that a transaction commit does
|
||||
// db.write internally will handle large transaction
|
||||
if bulk.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if bulk.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
|
||||
err := bulk.db.db.Write(bulk.tx, &opt.WriteOptions{Sync: true})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Database Error: %v", err))
|
||||
}
|
||||
bulk.isCommit = true
|
||||
}
|
||||
|
||||
func (bulk *levelBulk) DiscardLast() {
|
||||
bulk.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Iterator Implementation
|
||||
//=========================================================
|
||||
|
||||
type levelIterator struct {
|
||||
start []byte
|
||||
end []byte
|
||||
reverse bool
|
||||
iter iterator.Iterator
|
||||
isInvalid bool
|
||||
}
|
||||
|
||||
func (db *levelDB) Iterator(start, end []byte) Iterator {
|
||||
var reverse bool
|
||||
|
||||
// if end is bigger then start, then reverse order
|
||||
if bytes.Compare(start, end) == 1 {
|
||||
reverse = true
|
||||
} else {
|
||||
reverse = false
|
||||
}
|
||||
|
||||
iter := db.db.NewIterator(nil, nil)
|
||||
|
||||
if reverse {
|
||||
if start == nil {
|
||||
iter.Last()
|
||||
} else {
|
||||
valid := iter.Seek(start)
|
||||
if valid {
|
||||
soakey := iter.Key()
|
||||
if bytes.Compare(start, soakey) < 0 {
|
||||
iter.Prev()
|
||||
}
|
||||
} else {
|
||||
iter.Last()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if start == nil {
|
||||
iter.First()
|
||||
} else {
|
||||
iter.Seek(start)
|
||||
}
|
||||
}
|
||||
return &levelIterator{
|
||||
iter: iter,
|
||||
start: start,
|
||||
end: end,
|
||||
reverse: reverse,
|
||||
isInvalid: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Next() {
|
||||
if iter.Valid() {
|
||||
if iter.reverse {
|
||||
iter.iter.Prev()
|
||||
} else {
|
||||
iter.iter.Next()
|
||||
}
|
||||
} else {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Valid() bool {
|
||||
|
||||
// Once invalid, forever invalid.
|
||||
if iter.isInvalid {
|
||||
return false
|
||||
}
|
||||
|
||||
// Panic on DB error. No way to recover.
|
||||
if err := iter.iter.Error(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// If source is invalid, invalid.
|
||||
if !iter.iter.Valid() {
|
||||
iter.isInvalid = true
|
||||
return false
|
||||
}
|
||||
|
||||
// If key is end or past it, invalid.
|
||||
var end = iter.end
|
||||
var key = iter.iter.Key()
|
||||
|
||||
if iter.reverse {
|
||||
if end != nil && bytes.Compare(key, end) <= 0 {
|
||||
iter.isInvalid = true
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
if end != nil && bytes.Compare(end, key) <= 0 {
|
||||
iter.isInvalid = true
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Valid
|
||||
return true
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Key() (key []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is invalid")
|
||||
} else if err := iter.iter.Error(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
originalKey := iter.iter.Key()
|
||||
|
||||
key = make([]byte, len(originalKey))
|
||||
copy(key, originalKey)
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func (iter *levelIterator) Value() (value []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is invalid")
|
||||
} else if err := iter.iter.Error(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
originalValue := iter.iter.Value()
|
||||
|
||||
value = make([]byte, len(originalValue))
|
||||
copy(value, originalValue)
|
||||
|
||||
return value
|
||||
}
|
||||
26
db/logger.go
Normal file
26
db/logger.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"github.com/aergoio/aergo-lib/log"
|
||||
)
|
||||
|
||||
type extendedLog struct {
|
||||
*log.Logger
|
||||
}
|
||||
|
||||
func (l *extendedLog) Errorf(f string, v ...interface{}) {
|
||||
l.Error().Msgf(f, v...)
|
||||
}
|
||||
|
||||
func (l *extendedLog) Warningf(f string, v ...interface{}) {
|
||||
l.Warn().Msgf(f, v...)
|
||||
}
|
||||
|
||||
func (l *extendedLog) Infof(f string, v ...interface{}) {
|
||||
// reduce info to debug level because infos at badgerdb are too detail
|
||||
l.Debug().Msgf(f, v...) // INFO -> DEBUG
|
||||
}
|
||||
|
||||
func (l *extendedLog) Debugf(f string, v ...interface{}) {
|
||||
l.Debug().Msgf(f, v...)
|
||||
}
|
||||
384
db/memorydb.go
Normal file
384
db/memorydb.go
Normal file
@@ -0,0 +1,384 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"encoding/gob"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// This function is always called first
|
||||
func init() {
|
||||
dbConstructor := func(dir string) (DB, error) {
|
||||
return newMemoryDB(dir)
|
||||
}
|
||||
registorDBConstructor(MemoryImpl, dbConstructor)
|
||||
}
|
||||
|
||||
func newMemoryDB(dir string) (DB, error) {
|
||||
var db map[string][]byte
|
||||
|
||||
filePath := path.Join(dir, "database")
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err == nil {
|
||||
decoder := gob.NewDecoder(file) //
|
||||
err = decoder.Decode(&db)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
file.Close()
|
||||
|
||||
if db == nil {
|
||||
db = make(map[string][]byte)
|
||||
}
|
||||
|
||||
database := &memorydb{
|
||||
db: db,
|
||||
dir: filePath,
|
||||
}
|
||||
|
||||
return database, nil
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// DB Implementation
|
||||
//=========================================================
|
||||
|
||||
// Enforce database and transaction implements interfaces
|
||||
var _ DB = (*memorydb)(nil)
|
||||
|
||||
type memorydb struct {
|
||||
lock sync.Mutex
|
||||
db map[string][]byte
|
||||
dir string
|
||||
}
|
||||
|
||||
func (db *memorydb) Type() string {
|
||||
return "memorydb"
|
||||
}
|
||||
|
||||
func (db *memorydb) Set(key, value []byte) {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
db.db[string(key)] = value
|
||||
}
|
||||
|
||||
func (db *memorydb) Delete(key []byte) {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
delete(db.db, string(key))
|
||||
}
|
||||
|
||||
func (db *memorydb) Get(key []byte) []byte {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
return db.db[string(key)]
|
||||
}
|
||||
|
||||
func (db *memorydb) Exist(key []byte) bool {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
_, ok := db.db[string(key)]
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
func (db *memorydb) Close() {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
file, err := os.OpenFile(db.dir, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
if err == nil {
|
||||
encoder := gob.NewEncoder(file)
|
||||
encoder.Encode(db.db)
|
||||
}
|
||||
file.Close()
|
||||
}
|
||||
|
||||
func (db *memorydb) NewTx() Transaction {
|
||||
|
||||
return &memoryTransaction{
|
||||
db: db,
|
||||
opList: list.New(),
|
||||
isDiscard: false,
|
||||
isCommit: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (db *memorydb) NewBulk() Bulk {
|
||||
|
||||
return &memoryBulk{
|
||||
db: db,
|
||||
opList: list.New(),
|
||||
isDiscard: false,
|
||||
isCommit: false,
|
||||
}
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Transaction Implementation
|
||||
//=========================================================
|
||||
|
||||
type memoryTransaction struct {
|
||||
txLock sync.Mutex
|
||||
db *memorydb
|
||||
opList *list.List
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
type txOp struct {
|
||||
isSet bool
|
||||
key []byte
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Set(key, value []byte) {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
transaction.opList.PushBack(&txOp{true, key, value})
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Delete(key []byte) {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
transaction.opList.PushBack(&txOp{false, key, nil})
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Commit() {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
if transaction.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if transaction.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
|
||||
db := transaction.db
|
||||
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
for e := transaction.opList.Front(); e != nil; e = e.Next() {
|
||||
op := e.Value.(*txOp)
|
||||
if op.isSet {
|
||||
db.db[string(op.key)] = op.value
|
||||
} else {
|
||||
delete(db.db, string(op.key))
|
||||
}
|
||||
}
|
||||
|
||||
transaction.isCommit = true
|
||||
}
|
||||
|
||||
func (transaction *memoryTransaction) Discard() {
|
||||
transaction.txLock.Lock()
|
||||
defer transaction.txLock.Unlock()
|
||||
|
||||
transaction.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Bulk Implementation
|
||||
//=========================================================
|
||||
|
||||
type memoryBulk struct {
|
||||
txLock sync.Mutex
|
||||
db *memorydb
|
||||
opList *list.List
|
||||
isDiscard bool
|
||||
isCommit bool
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) Set(key, value []byte) {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
value = convNilToBytes(value)
|
||||
|
||||
bulk.opList.PushBack(&txOp{true, key, value})
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) Delete(key []byte) {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
key = convNilToBytes(key)
|
||||
|
||||
bulk.opList.PushBack(&txOp{false, key, nil})
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) Flush() {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
if bulk.isDiscard {
|
||||
panic("Commit after dicard tx is not allowed")
|
||||
} else if bulk.isCommit {
|
||||
panic("Commit occures two times")
|
||||
}
|
||||
|
||||
db := bulk.db
|
||||
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
for e := bulk.opList.Front(); e != nil; e = e.Next() {
|
||||
op := e.Value.(*txOp)
|
||||
if op.isSet {
|
||||
db.db[string(op.key)] = op.value
|
||||
} else {
|
||||
delete(db.db, string(op.key))
|
||||
}
|
||||
}
|
||||
|
||||
bulk.isCommit = true
|
||||
}
|
||||
|
||||
func (bulk *memoryBulk) DiscardLast() {
|
||||
bulk.txLock.Lock()
|
||||
defer bulk.txLock.Unlock()
|
||||
|
||||
bulk.isDiscard = true
|
||||
}
|
||||
|
||||
//=========================================================
|
||||
// Iterator Implementation
|
||||
//=========================================================
|
||||
|
||||
type memoryIterator struct {
|
||||
start []byte
|
||||
end []byte
|
||||
reverse bool
|
||||
keys []string
|
||||
isInvalid bool
|
||||
cursor int
|
||||
db *memorydb
|
||||
}
|
||||
|
||||
func isKeyInRange(key []byte, start []byte, end []byte, reverse bool) bool {
|
||||
if reverse {
|
||||
if start != nil && bytes.Compare(start, key) < 0 {
|
||||
return false
|
||||
}
|
||||
if end != nil && bytes.Compare(key, end) <= 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if bytes.Compare(key, start) < 0 {
|
||||
return false
|
||||
}
|
||||
if end != nil && bytes.Compare(end, key) <= 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
|
||||
}
|
||||
|
||||
func (db *memorydb) Iterator(start, end []byte) Iterator {
|
||||
db.lock.Lock()
|
||||
defer db.lock.Unlock()
|
||||
|
||||
var reverse bool
|
||||
|
||||
// if end is bigger then start, then reverse order
|
||||
if bytes.Compare(start, end) == 1 {
|
||||
reverse = true
|
||||
} else {
|
||||
reverse = false
|
||||
}
|
||||
|
||||
var keys sort.StringSlice
|
||||
|
||||
for key := range db.db {
|
||||
if isKeyInRange([]byte(key), start, end, reverse) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
if reverse {
|
||||
sort.Sort(sort.Reverse(keys))
|
||||
} else {
|
||||
sort.Strings(keys)
|
||||
}
|
||||
|
||||
return &memoryIterator{
|
||||
start: start,
|
||||
end: end,
|
||||
reverse: reverse,
|
||||
isInvalid: false,
|
||||
keys: keys,
|
||||
cursor: 0,
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Next() {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
|
||||
iter.cursor++
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Valid() bool {
|
||||
// Once invalid, forever invalid.
|
||||
if iter.isInvalid {
|
||||
return false
|
||||
}
|
||||
|
||||
return 0 <= iter.cursor && iter.cursor < len(iter.keys)
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Key() (key []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
|
||||
return []byte(iter.keys[iter.cursor])
|
||||
}
|
||||
|
||||
func (iter *memoryIterator) Value() (value []byte) {
|
||||
if !iter.Valid() {
|
||||
panic("Iterator is Invalid")
|
||||
}
|
||||
|
||||
key := []byte(iter.keys[iter.cursor])
|
||||
|
||||
return iter.db.Get(key)
|
||||
}
|
||||
443
db/perf_test.go
Normal file
443
db/perf_test.go
Normal file
@@ -0,0 +1,443 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/guptarohit/asciigraph"
|
||||
)
|
||||
|
||||
type testKeyType int64
|
||||
|
||||
// Simple execution cmd
|
||||
// go test -run=XXX -bench=. -benchmem -cpuprofile=cpu.out -memprofile=mem.out -timeout 20m -benchtime=1m
|
||||
|
||||
var valueLen = 256
|
||||
var testSetSize = int64(10000000)
|
||||
var batchSize = 100
|
||||
|
||||
// parameters for drawing graph
|
||||
var graphCountingPeriod = 10000
|
||||
var graphWidth = 120
|
||||
var graphHeigh = 20
|
||||
|
||||
var deviceType = "ssd"
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().Unix())
|
||||
}
|
||||
|
||||
func int64ToBytes(i testKeyType) []byte {
|
||||
buf := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(buf, uint64(i))
|
||||
return buf
|
||||
}
|
||||
|
||||
func printGraph(statics list.List) {
|
||||
if statics.Len() == 0 {
|
||||
return
|
||||
}
|
||||
// convert list to slice
|
||||
var staticsSlice = make([]float64, statics.Len())
|
||||
i := 0
|
||||
for e := statics.Front(); e != nil; e = e.Next() {
|
||||
staticsSlice[i] = e.Value.(float64)
|
||||
i++
|
||||
}
|
||||
|
||||
graph := asciigraph.Plot(staticsSlice, asciigraph.Width(graphWidth), asciigraph.Height(graphHeigh))
|
||||
fmt.Println(graph)
|
||||
}
|
||||
|
||||
func dirSizeKB(path string) (int64, error) {
|
||||
var size int64
|
||||
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
if !info.IsDir() {
|
||||
size += info.Size()
|
||||
}
|
||||
return err
|
||||
})
|
||||
return size / 1024, err
|
||||
}
|
||||
|
||||
func BenchmarkRandomWR(b *testing.B) {
|
||||
|
||||
// generate a common random data set
|
||||
internal := map[testKeyType][]byte{}
|
||||
for i := 0; i < int(testSetSize); i++ {
|
||||
// generate a random value
|
||||
token := make([]byte, valueLen)
|
||||
internal[testKeyType(i)] = token
|
||||
}
|
||||
|
||||
for dbType, dbConstructors := range dbImpls {
|
||||
|
||||
// create db
|
||||
dbName := string(dbType)
|
||||
tmpDir, _ := ioutil.TempDir("", dbName)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbInstance, _ := dbConstructors(tmpDir)
|
||||
var numOfWrite int64
|
||||
var idx testKeyType
|
||||
var statics list.List
|
||||
var startTime time.Time
|
||||
|
||||
fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
|
||||
dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
|
||||
fmt.Println("device type: ssd")
|
||||
|
||||
// write only
|
||||
b.Run(dbName+"-write", func(b *testing.B) {
|
||||
var i int
|
||||
for i = 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
|
||||
rand.Read(internal[idx]) // generate a random data
|
||||
|
||||
dbInstance.Set(
|
||||
int64ToBytes(testKeyType(idx)),
|
||||
internal[idx],
|
||||
)
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
numOfWrite += int64(i)
|
||||
})
|
||||
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// read only
|
||||
b.Run(dbName+"-read", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize))
|
||||
originalVal := internal[idx]
|
||||
|
||||
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
|
||||
|
||||
if len(retrievedVal) != 0 {
|
||||
if len(retrievedVal) != valueLen {
|
||||
b.Errorf("Expected length %X for %v, got %X",
|
||||
valueLen, idx, len(retrievedVal))
|
||||
break
|
||||
} else if !bytes.Equal(retrievedVal, originalVal) {
|
||||
b.Errorf("Expected %v for %v, got %v",
|
||||
originalVal, idx, retrievedVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// write and read
|
||||
b.Run(dbName+"-write-read", func(b *testing.B) {
|
||||
var i int
|
||||
for i = 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
idx = testKeyType(int64(rand.Int()) % testSetSize) // pick a random key
|
||||
rand.Read(internal[idx]) // generate a random data
|
||||
|
||||
dbInstance.Set(
|
||||
int64ToBytes(testKeyType(idx)),
|
||||
internal[idx],
|
||||
)
|
||||
|
||||
originalVal := internal[idx]
|
||||
|
||||
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
|
||||
|
||||
if len(retrievedVal) != 0 {
|
||||
if len(retrievedVal) != valueLen {
|
||||
b.Errorf("Expected length %X for %v, got %X",
|
||||
valueLen, idx, len(retrievedVal))
|
||||
break
|
||||
} else if !bytes.Equal(retrievedVal, originalVal) {
|
||||
b.Errorf("Expected %v for %v, got %v",
|
||||
originalVal, idx, retrievedVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
numOfWrite += int64(i)
|
||||
})
|
||||
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// close
|
||||
dbInstance.Close()
|
||||
|
||||
size, err := dirSizeKB(tmpDir)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
} else {
|
||||
fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRandomBatchWR(b *testing.B) {
|
||||
|
||||
// generate a common random data set
|
||||
internal := map[testKeyType][]byte{}
|
||||
for i := 0; i < int(testSetSize); i++ {
|
||||
// generate a random value
|
||||
token := make([]byte, valueLen)
|
||||
internal[testKeyType(i)] = token
|
||||
}
|
||||
|
||||
for dbType, dbConstructors := range dbImpls {
|
||||
|
||||
// create db
|
||||
dbName := string(dbType)
|
||||
tmpDir, _ := ioutil.TempDir("", dbName)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
dbInstance, _ := dbConstructors(tmpDir)
|
||||
var numOfWrite int64
|
||||
var idx testKeyType
|
||||
var statics list.List
|
||||
var startTime time.Time
|
||||
|
||||
fmt.Printf("[%s]\ntestset_size: %d\nkey_len: %d\nval_len: %d\n",
|
||||
dbName, testSetSize, reflect.TypeOf(idx).Size(), valueLen)
|
||||
fmt.Println("device type: " + deviceType)
|
||||
fmt.Printf("batch size: %d\n", batchSize)
|
||||
|
||||
// write only
|
||||
b.Run(dbName+"-batch-write", func(b *testing.B) {
|
||||
var i int
|
||||
|
||||
for i = 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
tx := dbInstance.NewTx()
|
||||
|
||||
for j := 0; j < batchSize; j++ {
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize)) // pick a random key
|
||||
rand.Read(internal[idx]) // generate a random data
|
||||
|
||||
tx.Set(
|
||||
int64ToBytes(testKeyType(idx)),
|
||||
internal[idx],
|
||||
)
|
||||
}
|
||||
tx.Commit()
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
numOfWrite += int64(i)
|
||||
})
|
||||
|
||||
// print a graph
|
||||
printGraph(statics)
|
||||
statics.Init()
|
||||
|
||||
// read only
|
||||
b.Run(dbName+"-read", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
startTime = time.Now()
|
||||
}
|
||||
|
||||
idx = testKeyType((int64(rand.Int()) % testSetSize))
|
||||
originalVal := internal[idx]
|
||||
|
||||
retrievedVal := dbInstance.Get(int64ToBytes(testKeyType(idx)))
|
||||
|
||||
if len(retrievedVal) != 0 {
|
||||
if len(retrievedVal) != valueLen {
|
||||
b.Errorf("Expected length %X for %v, got %X",
|
||||
valueLen, idx, len(retrievedVal))
|
||||
break
|
||||
} else if !bytes.Equal(retrievedVal, originalVal) {
|
||||
b.Errorf("Expected %v for %v, got %v",
|
||||
originalVal, idx, retrievedVal)
|
||||
break
|
||||
}
|
||||
}
|
||||
if i != 0 && i%graphCountingPeriod == 0 {
|
||||
endTime := time.Now().Sub(startTime).Seconds()
|
||||
statics.PushBack(endTime)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// close
|
||||
dbInstance.Close()
|
||||
|
||||
printGraph(statics)
|
||||
|
||||
size, err := dirSizeKB(tmpDir)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
} else {
|
||||
fmt.Printf("* Total size of %s db: %v kb, Size of 1 write: %v byte\n", dbName, size, size*1024/numOfWrite)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
[Refined Test Result]
|
||||
goos: windows
|
||||
goarch: amd64
|
||||
pkg: github.com/aergoio/aergo-lib/db
|
||||
|
||||
[badgerdb-ssd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
BenchmarkRandomWR/badgerdb-write-12 30000000 29536 ns/op 4983 B/op 84 allocs/op
|
||||
BenchmarkRandomWR/badgerdb-read-12 20000000 47739 ns/op 13154 B/op 60 allocs/op
|
||||
BenchmarkRandomWR/badgerdb-write-read-12 20000000 41990 ns/op 10317 B/op 140 allocs/op
|
||||
* Total size of badgerdb db: 14544473 kb, Size of 1 write: 286 byte
|
||||
|
||||
[leveldb-ssd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
BenchmarkRandomWR/leveldb-write-12 1000000 1006445 ns/op 532 B/op 5 allocs/op
|
||||
BenchmarkRandomWR/leveldb-read-12 50000000 18941 ns/op 1160 B/op 17 allocs/op
|
||||
BenchmarkRandomWR/leveldb-write-read-12 1000000 986440 ns/op 1272 B/op 11 allocs/op
|
||||
* Total size of leveldb db: 567386 kb, Size of 1 write: 250 byte
|
||||
|
||||
[badgerdb-ssd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 100
|
||||
BenchmarkRandomBatchWR/badgerdb-batch-write-12 2000000 612737 ns/op 383751 B/op 4576 allocs/op
|
||||
BenchmarkRandomBatchWR/badgerdb-read-12 20000000 53298 ns/op 22177 B/op 71 allocs/op
|
||||
* Total size of badgerdb db: 74546912 kb, Size of 1 write: 25359 byte
|
||||
|
||||
[leveldb-ssd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 100
|
||||
BenchmarkRandomBatchWR/leveldb-batch-write-12 300000 6778458 ns/op 127466 B/op 399 allocs/op
|
||||
BenchmarkRandomBatchWR/leveldb-read-12 5000000 141176 ns/op 4547 B/op 34 allocs/op
|
||||
* Total size of leveldb db: 2540271 kb, Size of 1 write: 8388 byte
|
||||
|
||||
[badgerdb-ssd-batch1000]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 1000
|
||||
BenchmarkRandomBatchWR1000/badgerdb-batch-write-12 200000 5507060 ns/op 3504344 B/op 42743 allocs/op
|
||||
BenchmarkRandomBatchWR1000/badgerdb-read-12 20000000 52636 ns/op 21867 B/op 73 allocs/op
|
||||
* Total size of badgerdb db: 51315516 kb, Size of 1 write: 250103 byte
|
||||
|
||||
[leveldb-ssd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: ssd
|
||||
batch size: 1000
|
||||
BenchmarkRandomBatchWR1000/leveldb-batch-write-12 20000 70431430 ns/op 1081956 B/op 3906 allocs/op
|
||||
BenchmarkRandomBatchWR1000/leveldb-read-12 5000000 138074 ns/op 4194 B/op 33 allocs/op
|
||||
* Total size of leveldb db: 2529393 kb, Size of 1 write: 86046 byte
|
||||
|
||||
[badgerdb-hdd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
BenchmarkHddRandomWR/badgerdb-write-12 30000000 27901 ns/op 4809 B/op 82 allocs/op
|
||||
BenchmarkHddRandomWR/badgerdb-read-12 20000000 45765 ns/op 14321 B/op 68 allocs/op
|
||||
BenchmarkHddRandomWR/badgerdb-write-read-12 20000000 39404 ns/op 8693 B/op 124 allocs/op
|
||||
* Total size of badgerdb db: 14697136 kb, Size of 1 write: 289 byte
|
||||
|
||||
[leveldb-hdd-single]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
BenchmarkHddRandomWR/leveldb-write-12 50000 20975160 ns/op 551 B/op 4 allocs/op
|
||||
BenchmarkHddRandomWR/leveldb-read-12 100000000 10375 ns/op 988 B/op 14 allocs/op
|
||||
BenchmarkHddRandomWR/leveldb-write-read-12 50000 21144257 ns/op 994 B/op 9 allocs/op
|
||||
* Total size of leveldb db: 31940 kb, Size of 1 write: 272 byte
|
||||
|
||||
[badgerdb-hdd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 100
|
||||
BenchmarkHddRandomBatchWR/badgerdb-batch-write-12 1000000 675351 ns/op 329398 B/op 4007 allocs/op
|
||||
BenchmarkHddRandomBatchWR/badgerdb-read-12 10000000 87933 ns/op 22713 B/op 85 allocs/op
|
||||
* Total size of badgerdb db: 24285947 kb, Size of 1 write: 24620 byte
|
||||
|
||||
[leveldb-hdd-batch100]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 100
|
||||
BenchmarkHddRandomBatchWR/leveldb-batch-write-12 30000 45769594 ns/op 136783 B/op 399 allocs/op
|
||||
BenchmarkHddRandomBatchWR/leveldb-read-12 20000000 30579 ns/op 3386 B/op 19 allocs/op
|
||||
* Total size of leveldb db: 878403 kb, Size of 1 write: 22430 byte
|
||||
|
||||
[badgerdb-hdd-batch1000]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 1000
|
||||
BenchmarkHddRandomBatchWR1000/badgerdb-batch-write-12 200000 6599637 ns/op 3724911 B/op 43392 allocs/op
|
||||
BenchmarkHddRandomBatchWR1000/badgerdb-read-12 10000000 99201 ns/op 31710 B/op 105 allocs/op
|
||||
* Total size of badgerdb db: 51157857 kb, Size of 1 write: 249335 byte
|
||||
|
||||
[leveldb-hdd-batch1000]
|
||||
testset_size: 10000000
|
||||
key_len: 8
|
||||
val_len: 256
|
||||
device type: hdd
|
||||
batch size: 1000
|
||||
BenchmarkHddRandomBatchWR1000/leveldb-batch-write-12 10000 209360868 ns/op 1027546 B/op 3363 allocs/op
|
||||
BenchmarkHddRandomBatchWR1000/leveldb-read-12 10000000 87929 ns/op 3971 B/op 29 allocs/op
|
||||
* Total size of leveldb db: 1689120 kb, Size of 1 write: 171236 byte
|
||||
|
||||
PASS
|
||||
ok github.com/aergoio/aergo-lib/db 33617.451s
|
||||
*/
|
||||
60
db/types.go
Normal file
60
db/types.go
Normal file
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* @file
|
||||
* @copyright defined in aergo/LICENSE.txt
|
||||
*/
|
||||
|
||||
package db
|
||||
|
||||
// ImplType represents implementators of a DB interface
|
||||
type ImplType string
|
||||
|
||||
const (
|
||||
// LevelImpl represents a name of DB interface implementation using leveldb
|
||||
LevelImpl ImplType = "leveldb"
|
||||
|
||||
// MemoryImpl represents a name of DB interface implementation in memory
|
||||
MemoryImpl ImplType = "memorydb"
|
||||
)
|
||||
|
||||
type dbConstructor func(dir string) (DB, error)
|
||||
|
||||
// DB is an general interface to access at storage data
|
||||
type DB interface {
|
||||
Type() string
|
||||
Set(key, value []byte)
|
||||
Delete(key []byte)
|
||||
Get(key []byte) []byte
|
||||
Exist(key []byte) bool
|
||||
Iterator(start, end []byte) Iterator
|
||||
NewTx() Transaction
|
||||
NewBulk() Bulk
|
||||
Close()
|
||||
//Print()
|
||||
//Stats() map[string]string
|
||||
}
|
||||
|
||||
// Transaction is used to batch multiple operations
|
||||
type Transaction interface {
|
||||
// Get(key []byte) []byte
|
||||
Set(key, value []byte)
|
||||
Delete(key []byte)
|
||||
Commit()
|
||||
Discard()
|
||||
}
|
||||
|
||||
// Bulk is used to batch multiple transactions
|
||||
// This will internally commit transactions when reach maximum tx size
|
||||
type Bulk interface {
|
||||
Set(key, value []byte)
|
||||
Delete(key []byte)
|
||||
Flush()
|
||||
DiscardLast()
|
||||
}
|
||||
|
||||
// Iterator is used to navigate specific key ranges
|
||||
type Iterator interface {
|
||||
Next()
|
||||
Valid() bool
|
||||
Key() []byte
|
||||
Value() []byte
|
||||
}
|
||||
Reference in New Issue
Block a user