Add TestAddBatchBench

TestAddBatchBench
	nCPU: 4, nLeafs: 50000, hash: Blake2b, db: leveldb
		Add loop:	9.304195479s
		AddBatch:	817.590526ms

.github/workflows/lint.yml

@@ -7,7 +7,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.14.x
+          go-version: 1.16.x
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Lint

.github/workflows/test.yml

@@ -5,7 +5,7 @@ jobs:
     # matrix strategy from: https://github.com/mvdan/github-actions-golang/blob/master/.github/workflows/test.yml
     strategy:
       matrix:
-        go-version: [1.14.x, 1.16.x]
+        go-version: [1.16.x]
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:

addbatch.go

@@ -727,6 +727,11 @@ func combineInKVSet(base, toAdd []kv) ([]kv, []int) {
 	return r, invalids
 }
 
+// TODO WIP
+// func loadDBTreeToVirtualTree() error {
+//         return nil
+// }
+
 // func computeSimpleAddCost(nLeafs int) int {
 //         // nLvls 2^nLvls
 //         nLvls := int(math.Log2(float64(nLeafs)))

addbatch_test.go

@@ -5,10 +5,12 @@ import (
 	"encoding/hex"
 	"fmt"
 	"math/big"
+	"runtime"
 	"testing"
 	"time"
 
 	qt "github.com/frankban/quicktest"
+	"github.com/iden3/go-merkletree/db/leveldb"
 	"github.com/iden3/go-merkletree/db/memory"
 )
 
@@ -221,52 +223,6 @@ func TestAddBatchCaseATestVector(t *testing.T) {
 	// check that both trees roots are equal
 	// fmt.Println(hex.EncodeToString(tree1.Root()))
 	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
-
-	//////
-
-	// tree1, err = NewTree(memory.NewMemoryStorage(), 100, HashFunctionBlake2b)
-	// c.Assert(err, qt.IsNil)
-	// defer tree1.db.Close()
-	//
-	// tree2, err = NewTree(memory.NewMemoryStorage(), 100, HashFunctionBlake2b)
-	// c.Assert(err, qt.IsNil)
-	// defer tree2.db.Close()
-	//
-	// // leafs in 2nd level subtrees: [ 6, 0, 1, 1]
-	// testvectorKeys = []string{
-	//         "1c7c2265e368314ca58ed2e1f33a326f1220e234a566d55c3605439dbe411642",
-	//         "2c9f0a578afff5bfa4e0992a43066460faaab9e8e500db0b16647c701cdb16bf",
-	//         "9cb87ec67e875c61390edcd1ab517f443591047709a4d4e45b0f9ed980857b8e",
-	//         "9b4e9e92e974a589f426ceeb4cb291dc24893513fecf8e8460992dcf52621d4d",
-	//         "1c45cb31f2fa39ec7b9ebf0fad40e0b8296016b5ce8844ae06ff77226379d9a5",
-	//         "d8af98bbbb585129798ae54d5eabbc9d0561d583faf1663b3a3724d15bda4ec7",
-	//         "3cd55dbfb8f975f20a0925dfbdabe79fa2d51dd0268afbb8ba6b01de9dfcdd3c",
-	//         "5d0a9d6d9f197c091bf054fac9cb60e11ec723d6610ed8578e617b4d46cb43d5",
-	// }
-	// keys = [][]byte{}
-	// values = [][]byte{}
-	// for i := 0; i < len(testvectorKeys); i++ {
-	//         key, err := hex.DecodeString(testvectorKeys[i])
-	//         c.Assert(err, qt.IsNil)
-	//         keys = append(keys, key)
-	//         values = append(values, []byte{0})
-	// }
-	//
-	// for i := 0; i < len(keys); i++ {
-	//         if err := tree1.Add(keys[i], values[i]); err != nil {
-	//                 t.Fatal(err)
-	//         }
-	// }
-	//
-	// indexes, err = tree2.AddBatch(keys, values)
-	// c.Assert(err, qt.IsNil)
-	// tree1.PrintGraphviz(nil)
-	// tree2.PrintGraphviz(nil)
-	//
-	// c.Check(len(indexes), qt.Equals, 0)
-	//
-	// // check that both trees roots are equal
-	// // c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
 }
 
 func TestAddBatchCaseARandomKeys(t *testing.T) {
@@ -678,6 +634,64 @@ func TestFlp2(t *testing.T) {
 	c.Assert(flp2(9000), qt.Equals, 8192)
 }
 
+func TestAddBatchBench(t *testing.T) {
+	nLeafs := 50_000
+	fmt.Printf("TestAddBatchBench\n	nCPU: %d, nLeafs: %d, hash: Blake2b, db: leveldb\n",
+		runtime.NumCPU(), nLeafs)
+
+	// prepare inputs
+	var ks, vs [][]byte
+	for i := 0; i < nLeafs; i++ {
+		k := randomBytes(32)
+		v := randomBytes(32)
+		ks = append(ks, k)
+		vs = append(vs, v)
+	}
+
+	benchAdd(t, ks, vs)
+
+	benchAddBatch(t, ks, vs)
+}
+
+func benchAdd(t *testing.T, ks, vs [][]byte) {
+	c := qt.New(t)
+
+	dbDir := t.TempDir()
+	storage, err := leveldb.NewLevelDbStorage(dbDir, false)
+	c.Assert(err, qt.IsNil)
+	tree, err := NewTree(storage, 140, HashFunctionBlake2b)
+	c.Assert(err, qt.IsNil)
+
+	start := time.Now()
+	for i := 0; i < len(ks); i++ {
+		err = tree.Add(ks[i], vs[i])
+		c.Assert(err, qt.IsNil)
+	}
+	printRes("	Add loop", time.Since(start))
+}
+
+func benchAddBatch(t *testing.T, ks, vs [][]byte) {
+	c := qt.New(t)
+
+	dbDir := t.TempDir()
+	storage, err := leveldb.NewLevelDbStorage(dbDir, false)
+	c.Assert(err, qt.IsNil)
+	tree, err := NewTree(storage, 140, HashFunctionBlake2b)
+	c.Assert(err, qt.IsNil)
+
+	start := time.Now()
+	invalids, err := tree.AddBatch(ks, vs)
+	printRes("	AddBatch", time.Since(start))
+	c.Assert(err, qt.IsNil)
+	c.Assert(len(invalids), qt.Equals, 0)
+}
+
+func printRes(name string, duration time.Duration) {
+	if debug {
+		fmt.Printf("	%s:	%s \n", name, duration)
+	}
+}
+
 // func printLeafs(name string, t *Tree) {
 //         w := bytes.NewBufferString("")
 //

go.sum

@@ -123,6 +123,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws=
 github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -263,6 +264,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=

tree_test.go

@@ -14,13 +14,13 @@ func TestAddTestVectors(t *testing.T) {
 	c := qt.New(t)
 
 	// Poseidon test vectors generated using https://github.com/iden3/circomlib smt.js
-	// testVectorsPoseidon := []string{
-	//         "0000000000000000000000000000000000000000000000000000000000000000",
-	//         "13578938674299138072471463694055224830892726234048532520316387704878000008795",
-	//         "5412393676474193513566895793055462193090331607895808993925969873307089394741",
-	//         "14204494359367183802864593755198662203838502594566452929175967972147978322084",
-	// }
-	// testAdd(c, HashFunctionPoseidon, testVectorsPoseidon)
+	testVectorsPoseidon := []string{
+		"0000000000000000000000000000000000000000000000000000000000000000",
+		"13578938674299138072471463694055224830892726234048532520316387704878000008795",
+		"5412393676474193513566895793055462193090331607895808993925969873307089394741",
+		"14204494359367183802864593755198662203838502594566452929175967972147978322084",
+	}
+	testAdd(c, HashFunctionPoseidon, testVectorsPoseidon)
 
 	testVectorsSha256 := []string{
 		"0000000000000000000000000000000000000000000000000000000000000000",