arnaucube/arbo
1
1
Fork 0
mirror of https://github.com/arnaucube/arbo.git synced 2026-01-17 02:31:28 +01:00
Code Issues Projects Releases Wiki Activity

Migrate kv db to go.vocdoni.io/dvote/db interface

Browse Source 
Case tree empty, AddBatch was 10.95x times faster than without AddBatch
	nCPU: 4, nLeafs: 1024, hash: Poseidon, db: memory
	dbgStats(hash: 2.047k, dbGet: 1, dbPut: 2.049k)
Case tree not empty w/ few leafs, AddBatch was 7.28x times faster than without AddBatch
	nCPU: 4, nLeafs: 1024, hash: Poseidon, db: memory
	dbgStats(hash: 2.047k, dbGet: 198, dbPut: 2.049k)
Case tree not empty w/ enough leafs, AddBatch was 5.94x times faster than without AddBatch
	nCPU: 4, nLeafs: 1024, hash: Poseidon, db: memory
	dbgStats(hash: 2.047k, dbGet: 1.000k, dbPut: 2.049k)
Case tree not empty, AddBatch was 9.27x times faster than without AddBatch
	nCPU: 4, nLeafs: 4096, hash: Poseidon, db: memory
	dbgStats(hash: 8.191k, dbGet: 1.800k, dbPut: 8.193k)
Case tree not empty & unbalanced, AddBatch was 10.67x times faster than without AddBatch
	nCPU: 4, nLeafs: 4096, hash: Poseidon, db: memory
	dbgStats(hash: 10.409k, dbGet: 2.668k, dbPut: 10.861k)
TestAddBatchBench: nCPU: 4, nLeafs: 50000, hash: Blake2b, db: badgerdb
	Add loop:	10.10829114s
	AddBatch:	732.030263ms
		dbgStats(hash: 122.518k, dbGet: 1, dbPut: 122.520k)
TestDbgStats
	add in loop in emptyTree dbgStats(hash: 141.721k, dbGet: 134.596k, dbPut: 161.721k)
	addbatch caseEmptyTree dbgStats(hash: 24.402k, dbGet: 1, dbPut: 24.404k)
	addbatch caseNotEmptyTree dbgStats(hash: 26.868k, dbGet: 2.468k, dbPut: 26.872k)
This commit is contained in:
arnaucube
2021-06-03 14:16:06 +02:00
parent 467f063129
commit c6059fcb75
6 changed files with 1896 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
tree_test.go
View File

@@ -7,9 +7,19 @@ import (
"time"
qt "github.com/frankban/quicktest"
"github.com/iden3/go-merkletree/db/memory"
"go.vocdoni.io/dvote/db"
)
func TestDBBatch(t *testing.T) {
c := qt.New(t)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
b := database.NewBatch()
err = b.Put([]byte("a"), []byte("b"))
c.Assert(err, qt.IsNil)
}
func TestAddTestVectors(t *testing.T) {
c := qt.New(t)
@@ -32,9 +42,11 @@ func TestAddTestVectors(t *testing.T) {
}
func testAdd(c *qt.C, hashFunc HashFunction, testVectors []string) {
tree, err := NewTree(memory.NewMemoryStorage(), 10, hashFunc)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 10, hashFunc)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
c.Check(hex.EncodeToString(tree.Root()), qt.Equals, testVectors[0])
@@ -63,9 +75,11 @@ func testAdd(c *qt.C, hashFunc HashFunction, testVectors []string) {
func TestAddBatch(t *testing.T) {
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
bLen := tree.HashFunction().Len()
for i := 0; i < 1000; i++ {
@@ -80,9 +94,11 @@ func TestAddBatch(t *testing.T) {
c.Check(rootBI.String(), qt.Equals,
"296519252211642170490407814696803112091039265640052570497930797516015811235")
tree2, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err = db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree2.db.Close()
tree2, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree2.db.Close() //nolint:errcheck
var keys, values [][]byte
for i := 0; i < 1000; i++ {
@@ -102,9 +118,11 @@ func TestAddBatch(t *testing.T) {
func TestAddDifferentOrder(t *testing.T) {
c := qt.New(t)
tree1, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database1, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree1.db.Close()
tree1, err := NewTree(database1, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree1.db.Close() //nolint:errcheck
bLen := tree1.HashFunction().Len()
for i := 0; i < 16; i++ {
@@ -115,9 +133,11 @@ func TestAddDifferentOrder(t *testing.T) {
}
}
tree2, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database2, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree2.db.Close()
tree2, err := NewTree(database2, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree2.db.Close() //nolint:errcheck
for i := 16 - 1; i >= 0; i-- {
k := BigIntToBytes(bLen, big.NewInt(int64(i)))
@@ -134,9 +154,11 @@ func TestAddDifferentOrder(t *testing.T) {
func TestAddRepeatedIndex(t *testing.T) {
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
bLen := tree.HashFunction().Len()
k := BigIntToBytes(bLen, big.NewInt(int64(3)))
@@ -150,9 +172,11 @@ func TestAddRepeatedIndex(t *testing.T) {
func TestUpdate(t *testing.T) {
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
bLen := tree.HashFunction().Len()
k := BigIntToBytes(bLen, big.NewInt(int64(20)))
@@ -201,9 +225,11 @@ func TestUpdate(t *testing.T) {
func TestAux(t *testing.T) { // TODO split in proper tests
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
bLen := tree.HashFunction().Len()
k := BigIntToBytes(bLen, big.NewInt(int64(1)))
@@ -238,9 +264,11 @@ func TestAux(t *testing.T) { // TODO split in proper tests
func TestGet(t *testing.T) {
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
bLen := tree.HashFunction().Len()
for i := 0; i < 10; i++ {
@@ -260,9 +288,11 @@ func TestGet(t *testing.T) {
func TestGenProofAndVerify(t *testing.T) {
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
bLen := tree.HashFunction().Len()
for i := 0; i < 10; i++ {
@@ -286,9 +316,11 @@ func TestGenProofAndVerify(t *testing.T) {
func TestDumpAndImportDump(t *testing.T) {
c := qt.New(t)
tree1, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database1, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree1.db.Close()
tree1, err := NewTree(database1, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree1.db.Close() //nolint:errcheck
bLen := tree1.HashFunction().Len()
for i := 0; i < 16; i++ {
@@ -302,9 +334,11 @@ func TestDumpAndImportDump(t *testing.T) {
e, err := tree1.Dump(nil)
c.Assert(err, qt.IsNil)
tree2, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database2, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree2.db.Close()
tree2, err := NewTree(database2, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree2.db.Close() //nolint:errcheck
err = tree2.ImportDump(e)
c.Assert(err, qt.IsNil)
c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
@@ -314,9 +348,11 @@ func TestDumpAndImportDump(t *testing.T) {
func TestRWMutex(t *testing.T) {
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
bLen := tree.HashFunction().Len()
var keys, values [][]byte
@@ -343,17 +379,18 @@ func TestRWMutex(t *testing.T) {
func TestSetGetNLeafs(t *testing.T) {
c := qt.New(t)
tree, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionPoseidon)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
tree, err := NewTree(database, 100, HashFunctionPoseidon)
c.Assert(err, qt.IsNil)
// 0
tree.tx, err = tree.db.NewTx()
c.Assert(err, qt.IsNil)
tree.dbBatch = tree.db.NewBatch()
err = tree.setNLeafs(0)
c.Assert(err, qt.IsNil)
err = tree.tx.Commit()
err = tree.dbBatch.Write()
c.Assert(err, qt.IsNil)
n, err := tree.GetNLeafs()
@@ -361,13 +398,12 @@ func TestSetGetNLeafs(t *testing.T) {
c.Assert(n, qt.Equals, 0)
// 1024
tree.tx, err = tree.db.NewTx()
c.Assert(err, qt.IsNil)
tree.dbBatch = tree.db.NewBatch()
err = tree.setNLeafs(1024)
c.Assert(err, qt.IsNil)
err = tree.tx.Commit()
err = tree.dbBatch.Write()
c.Assert(err, qt.IsNil)
n, err = tree.GetNLeafs()
@@ -375,8 +411,7 @@ func TestSetGetNLeafs(t *testing.T) {
c.Assert(n, qt.Equals, 1024)
// 2**64 -1
tree.tx, err = tree.db.NewTx()
c.Assert(err, qt.IsNil)
tree.dbBatch = tree.db.NewBatch()
maxUint := ^uint(0)
maxInt := int(maxUint >> 1)
@@ -384,7 +419,7 @@ func TestSetGetNLeafs(t *testing.T) {
err = tree.setNLeafs(maxInt)
c.Assert(err, qt.IsNil)
err = tree.tx.Commit()
err = tree.dbBatch.Write()
c.Assert(err, qt.IsNil)
n, err = tree.GetNLeafs()
@@ -413,9 +448,11 @@ func BenchmarkAdd(b *testing.B) {
func benchmarkAdd(b *testing.B, hashFunc HashFunction, ks, vs [][]byte) {
c := qt.New(b)
tree, err := NewTree(memory.NewMemoryStorage(), 140, hashFunc)
database, err := db.NewBadgerDB(c.TempDir())
c.Assert(err, qt.IsNil)
defer tree.db.Close()
tree, err := NewTree(database, 140, hashFunc)
c.Assert(err, qt.IsNil)
defer tree.db.Close() //nolint:errcheck
for i := 0; i < len(ks); i++ {
if err := tree.Add(ks[i], vs[i]); err != nil {