From 2a57e223effee0deebd5ea8c9fcd80f0de0be10e Mon Sep 17 00:00:00 2001 From: arnaucube Date: Thu, 20 May 2021 11:06:09 +0200 Subject: [PATCH] Add TestAddBatchBench TestAddBatchBench nCPU: 4, nLeafs: 50000, hash: Blake2b, db: leveldb Add loop: 9.304195479s AddBatch: 817.590526ms --- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 2 +- addbatch.go | 5 ++ addbatch_test.go | 106 +++++++++++++++++++++---------------- go.sum | 2 + tree_test.go | 14 ++--- 6 files changed, 76 insertions(+), 55 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cdedb10..74abf7e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ jobs: - name: Install Go uses: actions/setup-go@v1 with: - go-version: 1.14.x + go-version: 1.16.x - name: Checkout code uses: actions/checkout@v2 - name: Lint diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ad5500..610eea4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ jobs: # matrix strategy from: https://github.com/mvdan/github-actions-golang/blob/master/.github/workflows/test.yml strategy: matrix: - go-version: [1.14.x, 1.16.x] + go-version: [1.16.x] platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} steps: diff --git a/addbatch.go b/addbatch.go index 23aa96f..3d69f4f 100644 --- a/addbatch.go +++ b/addbatch.go @@ -727,6 +727,11 @@ func combineInKVSet(base, toAdd []kv) ([]kv, []int) { return r, invalids } +// TODO WIP +// func loadDBTreeToVirtualTree() error { +// return nil +// } + // func computeSimpleAddCost(nLeafs int) int { // // nLvls 2^nLvls // nLvls := int(math.Log2(float64(nLeafs))) diff --git a/addbatch_test.go b/addbatch_test.go index d35aa7d..d23337a 100644 --- a/addbatch_test.go +++ b/addbatch_test.go @@ -5,10 +5,12 @@ import ( "encoding/hex" "fmt" "math/big" + "runtime" "testing" "time" qt "github.com/frankban/quicktest" + "github.com/iden3/go-merkletree/db/leveldb" "github.com/iden3/go-merkletree/db/memory" ) @@ -221,52 +223,6 @@ func TestAddBatchCaseATestVector(t *testing.T) { // check that both trees roots are equal // fmt.Println(hex.EncodeToString(tree1.Root())) c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) - - ////// - - // tree1, err = NewTree(memory.NewMemoryStorage(), 100, HashFunctionBlake2b) - // c.Assert(err, qt.IsNil) - // defer tree1.db.Close() - // - // tree2, err = NewTree(memory.NewMemoryStorage(), 100, HashFunctionBlake2b) - // c.Assert(err, qt.IsNil) - // defer tree2.db.Close() - // - // // leafs in 2nd level subtrees: [ 6, 0, 1, 1] - // testvectorKeys = []string{ - // "1c7c2265e368314ca58ed2e1f33a326f1220e234a566d55c3605439dbe411642", - // "2c9f0a578afff5bfa4e0992a43066460faaab9e8e500db0b16647c701cdb16bf", - // "9cb87ec67e875c61390edcd1ab517f443591047709a4d4e45b0f9ed980857b8e", - // "9b4e9e92e974a589f426ceeb4cb291dc24893513fecf8e8460992dcf52621d4d", - // "1c45cb31f2fa39ec7b9ebf0fad40e0b8296016b5ce8844ae06ff77226379d9a5", - // "d8af98bbbb585129798ae54d5eabbc9d0561d583faf1663b3a3724d15bda4ec7", - // "3cd55dbfb8f975f20a0925dfbdabe79fa2d51dd0268afbb8ba6b01de9dfcdd3c", - // "5d0a9d6d9f197c091bf054fac9cb60e11ec723d6610ed8578e617b4d46cb43d5", - // } - // keys = [][]byte{} - // values = [][]byte{} - // for i := 0; i < len(testvectorKeys); i++ { - // key, err := hex.DecodeString(testvectorKeys[i]) - // c.Assert(err, qt.IsNil) - // keys = append(keys, key) - // values = append(values, []byte{0}) - // } - // - // for i := 0; i < len(keys); i++ { - // if err := tree1.Add(keys[i], values[i]); err != nil { - // t.Fatal(err) - // } - // } - // - // indexes, err = tree2.AddBatch(keys, values) - // c.Assert(err, qt.IsNil) - // tree1.PrintGraphviz(nil) - // tree2.PrintGraphviz(nil) - // - // c.Check(len(indexes), qt.Equals, 0) - // - // // check that both trees roots are equal - // // c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) } func TestAddBatchCaseARandomKeys(t *testing.T) { @@ -678,6 +634,64 @@ func TestFlp2(t *testing.T) { c.Assert(flp2(9000), qt.Equals, 8192) } +func TestAddBatchBench(t *testing.T) { + nLeafs := 50_000 + fmt.Printf("TestAddBatchBench\n nCPU: %d, nLeafs: %d, hash: Blake2b, db: leveldb\n", + runtime.NumCPU(), nLeafs) + + // prepare inputs + var ks, vs [][]byte + for i := 0; i < nLeafs; i++ { + k := randomBytes(32) + v := randomBytes(32) + ks = append(ks, k) + vs = append(vs, v) + } + + benchAdd(t, ks, vs) + + benchAddBatch(t, ks, vs) +} + +func benchAdd(t *testing.T, ks, vs [][]byte) { + c := qt.New(t) + + dbDir := t.TempDir() + storage, err := leveldb.NewLevelDbStorage(dbDir, false) + c.Assert(err, qt.IsNil) + tree, err := NewTree(storage, 140, HashFunctionBlake2b) + c.Assert(err, qt.IsNil) + + start := time.Now() + for i := 0; i < len(ks); i++ { + err = tree.Add(ks[i], vs[i]) + c.Assert(err, qt.IsNil) + } + printRes(" Add loop", time.Since(start)) +} + +func benchAddBatch(t *testing.T, ks, vs [][]byte) { + c := qt.New(t) + + dbDir := t.TempDir() + storage, err := leveldb.NewLevelDbStorage(dbDir, false) + c.Assert(err, qt.IsNil) + tree, err := NewTree(storage, 140, HashFunctionBlake2b) + c.Assert(err, qt.IsNil) + + start := time.Now() + invalids, err := tree.AddBatch(ks, vs) + printRes(" AddBatch", time.Since(start)) + c.Assert(err, qt.IsNil) + c.Assert(len(invalids), qt.Equals, 0) +} + +func printRes(name string, duration time.Duration) { + if debug { + fmt.Printf(" %s: %s \n", name, duration) + } +} + // func printLeafs(name string, t *Tree) { // w := bytes.NewBufferString("") // diff --git a/go.sum b/go.sum index 3201b00..0f2a6a3 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -263,6 +264,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/tree_test.go b/tree_test.go index 9cf88bd..3a7eb14 100644 --- a/tree_test.go +++ b/tree_test.go @@ -14,13 +14,13 @@ func TestAddTestVectors(t *testing.T) { c := qt.New(t) // Poseidon test vectors generated using https://github.com/iden3/circomlib smt.js - // testVectorsPoseidon := []string{ - // "0000000000000000000000000000000000000000000000000000000000000000", - // "13578938674299138072471463694055224830892726234048532520316387704878000008795", - // "5412393676474193513566895793055462193090331607895808993925969873307089394741", - // "14204494359367183802864593755198662203838502594566452929175967972147978322084", - // } - // testAdd(c, HashFunctionPoseidon, testVectorsPoseidon) + testVectorsPoseidon := []string{ + "0000000000000000000000000000000000000000000000000000000000000000", + "13578938674299138072471463694055224830892726234048532520316387704878000008795", + "5412393676474193513566895793055462193090331607895808993925969873307089394741", + "14204494359367183802864593755198662203838502594566452929175967972147978322084", + } + testAdd(c, HashFunctionPoseidon, testVectorsPoseidon) testVectorsSha256 := []string{ "0000000000000000000000000000000000000000000000000000000000000000",