diff --git a/addbatch_test.go b/addbatch_test.go index 529278b..467b233 100644 --- a/addbatch_test.go +++ b/addbatch_test.go @@ -61,7 +61,7 @@ func testInit(c *qt.C, n int) (*Tree, *Tree) { return tree1, tree2 } -func TestAddBatchCaseA(t *testing.T) { +func TestAddBatchTreeEmpty(t *testing.T) { c := qt.New(t) nLeafs := 1024 @@ -97,7 +97,7 @@ func TestAddBatchCaseA(t *testing.T) { c.Assert(err, qt.IsNil) time2 := time.Since(start) if debug { - debugTime("CASE A, AddBatch", time1, time2) + debugTime("Case tree empty, AddBatch", time1, time2) printTestContext(" ", nLeafs, "Poseidon", "memory") tree2.dbg.print(" ") } @@ -107,7 +107,7 @@ func TestAddBatchCaseA(t *testing.T) { c.Check(tree2.Root(), qt.DeepEquals, tree.Root()) } -func TestAddBatchCaseANotPowerOf2(t *testing.T) { +func TestAddBatchTreeEmptyNotPowerOf2(t *testing.T) { c := qt.New(t) nLeafs := 1027 @@ -152,7 +152,7 @@ func randomBytes(n int) []byte { return b } -func TestAddBatchCaseATestVector(t *testing.T) { +func TestAddBatchTreeEmptyTestVector(t *testing.T) { c := qt.New(t) tree1, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionBlake2b) c.Assert(err, qt.IsNil) @@ -230,7 +230,7 @@ func TestAddBatchCaseATestVector(t *testing.T) { c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) } -func TestAddBatchCaseARandomKeys(t *testing.T) { +func TestAddBatchTreeEmptyRandomKeys(t *testing.T) { c := qt.New(t) nLeafs := 8 @@ -246,21 +246,9 @@ func TestAddBatchCaseARandomKeys(t *testing.T) { var keys, values [][]byte for i := 0; i < nLeafs; i++ { keys = append(keys, randomBytes(32)) - // values = append(values, randomBytes(32)) - values = append(values, []byte{0}) - // fmt.Println("K", hex.EncodeToString(keys[i])) + values = append(values, randomBytes(32)) } - // TODO delete - keys[0], _ = hex.DecodeString("1c7c2265e368314ca58ed2e1f33a326f1220e234a566d55c3605439dbe411642") - keys[1], _ = hex.DecodeString("2c9f0a578afff5bfa4e0992a43066460faaab9e8e500db0b16647c701cdb16bf") - keys[2], _ = hex.DecodeString("9cb87ec67e875c61390edcd1ab517f443591047709a4d4e45b0f9ed980857b8e") - keys[3], _ = hex.DecodeString("9b4e9e92e974a589f426ceeb4cb291dc24893513fecf8e8460992dcf52621d4d") - keys[4], _ = hex.DecodeString("1c45cb31f2fa39ec7b9ebf0fad40e0b8296016b5ce8844ae06ff77226379d9a5") - keys[5], _ = hex.DecodeString("d8af98bbbb585129798ae54d5eabbc9d0561d583faf1663b3a3724d15bda4ec7") - keys[6], _ = hex.DecodeString("3cd55dbfb8f975f20a0925dfbdabe79fa2d51dd0268afbb8ba6b01de9dfcdd3c") - keys[7], _ = hex.DecodeString("5d0a9d6d9f197c091bf054fac9cb60e11ec723d6610ed8578e617b4d46cb43d5") - for i := 0; i < len(keys); i++ { if err := tree1.Add(keys[i], values[i]); err != nil { t.Fatal(err) @@ -269,20 +257,16 @@ func TestAddBatchCaseARandomKeys(t *testing.T) { indexes, err := tree2.AddBatch(keys, values) c.Assert(err, qt.IsNil) - // tree1.PrintGraphviz(nil) - // tree2.PrintGraphviz(nil) - c.Check(len(indexes), qt.Equals, 0) - // check that both trees roots are equal c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) } -func TestAddBatchCaseB(t *testing.T) { +func TestAddBatchTreeNotEmptyFewLeafs(t *testing.T) { c := qt.New(t) nLeafs := 1024 - initialNLeafs := 99 // TMP TODO use const minLeafsThreshold-1 once ready + initialNLeafs := 99 tree1, tree2 := testInit(c, initialNLeafs) tree2.dbgInit() @@ -310,7 +294,7 @@ func TestAddBatchCaseB(t *testing.T) { c.Assert(err, qt.IsNil) time2 := time.Since(start) if debug { - debugTime("CASE B, AddBatch", time1, time2) + debugTime("Case tree not empty w/ few leafs, AddBatch", time1, time2) printTestContext(" ", nLeafs, "Poseidon", "memory") tree2.dbg.print(" ") } @@ -320,14 +304,16 @@ func TestAddBatchCaseB(t *testing.T) { c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) } -func TestAddBatchCaseBRepeatedLeafs(t *testing.T) { +func TestAddBatchTreeNotEmptyEnoughLeafs(t *testing.T) { c := qt.New(t) nLeafs := 1024 - initialNLeafs := 99 // TMP TODO use const minLeafsThreshold-1 once ready + initialNLeafs := 500 tree1, tree2 := testInit(c, initialNLeafs) + tree2.dbgInit() + start := time.Now() for i := initialNLeafs; i < nLeafs; i++ { k := BigIntToBytes(big.NewInt(int64(i))) v := BigIntToBytes(big.NewInt(int64(i * 2))) @@ -335,8 +321,39 @@ func TestAddBatchCaseBRepeatedLeafs(t *testing.T) { t.Fatal(err) } } + time1 := time.Since(start) - // prepare the key-values to be added, including already existing keys + // prepare the key-values to be added + var keys, values [][]byte + for i := initialNLeafs; i < nLeafs; i++ { + k := BigIntToBytes(big.NewInt(int64(i))) + v := BigIntToBytes(big.NewInt(int64(i * 2))) + keys = append(keys, k) + values = append(values, v) + } + start = time.Now() + indexes, err := tree2.AddBatch(keys, values) + c.Assert(err, qt.IsNil) + time2 := time.Since(start) + if debug { + debugTime("Case tree not empty w/ enough leafs, AddBatch", time1, time2) + printTestContext(" ", nLeafs, "Poseidon", "memory") + tree2.dbg.print(" ") + } + c.Check(len(indexes), qt.Equals, 0) + // check that both trees roots are equal + c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) +} + +func TestAddBatchTreeEmptyRepeatedLeafs(t *testing.T) { + c := qt.New(t) + + nLeafs := 1024 + nRepeatedKeys := 99 + + tree1, tree2 := testInit(c, 0) + + // prepare the key-values to be added var keys, values [][]byte for i := 0; i < nLeafs; i++ { k := BigIntToBytes(big.NewInt(int64(i))) @@ -344,10 +361,55 @@ func TestAddBatchCaseBRepeatedLeafs(t *testing.T) { keys = append(keys, k) values = append(values, v) } + // add repeated key-values + for i := 0; i < nRepeatedKeys; i++ { + k := BigIntToBytes(big.NewInt(int64(i))) + v := BigIntToBytes(big.NewInt(int64(i * 2))) + keys = append(keys, k) + values = append(values, v) + } + + // add the non-repeated key-values in tree1 with .Add loop + for i := 0; i < nLeafs; i++ { + if err := tree1.Add(keys[i], values[i]); err != nil { + t.Fatal(err) + } + } + indexes, err := tree2.AddBatch(keys, values) c.Assert(err, qt.IsNil) - c.Check(len(indexes), qt.Equals, initialNLeafs) + c.Check(len(indexes), qt.Equals, nRepeatedKeys) + // check that both trees roots are equal + c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) +} + +func TestAddBatchTreeNotEmptyFewLeafsRepeatedLeafs(t *testing.T) { + c := qt.New(t) + + nLeafs := 1024 + initialNLeafs := 99 + + tree1, tree2 := testInit(c, initialNLeafs) + + // prepare the key-values to be added + var keys, values [][]byte + for i := 0; i < nLeafs; i++ { + k := BigIntToBytes(big.NewInt(int64(i))) + v := BigIntToBytes(big.NewInt(int64(i * 2))) + keys = append(keys, k) + values = append(values, v) + } + + // add the keys that will be existing when AddBatch is called + for i := initialNLeafs; i < nLeafs; i++ { + if err := tree1.Add(keys[i], values[i]); err != nil { + t.Fatal(err) + } + } + indexes, err := tree2.AddBatch(keys, values) + c.Assert(err, qt.IsNil) + c.Check(len(indexes), qt.Equals, initialNLeafs) // check that both trees roots are equal c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) } @@ -452,49 +514,7 @@ func sortKvs(kvs []kv) { }) } -func TestAddBatchCaseC(t *testing.T) { - c := qt.New(t) - - nLeafs := 1024 - initialNLeafs := 101 // TMP TODO use const minLeafsThreshold+1 once ready - - tree1, tree2 := testInit(c, initialNLeafs) - tree2.dbgInit() - - start := time.Now() - for i := initialNLeafs; i < nLeafs; i++ { - k := BigIntToBytes(big.NewInt(int64(i))) - v := BigIntToBytes(big.NewInt(int64(i * 2))) - if err := tree1.Add(k, v); err != nil { - t.Fatal(err) - } - } - time1 := time.Since(start) - - // prepare the key-values to be added - var keys, values [][]byte - for i := initialNLeafs; i < nLeafs; i++ { - k := BigIntToBytes(big.NewInt(int64(i))) - v := BigIntToBytes(big.NewInt(int64(i * 2))) - keys = append(keys, k) - values = append(values, v) - } - start = time.Now() - indexes, err := tree2.AddBatch(keys, values) - c.Assert(err, qt.IsNil) - time2 := time.Since(start) - if debug { - debugTime("CASE C, AddBatch", time1, time2) - printTestContext(" ", nLeafs, "Poseidon", "memory") - tree2.dbg.print(" ") - } - c.Check(len(indexes), qt.Equals, 0) - - // check that both trees roots are equal - c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) -} - -func TestAddBatchCaseD(t *testing.T) { +func TestAddBatchTreeNotEmpty(t *testing.T) { c := qt.New(t) nLeafs := 4096 @@ -526,7 +546,7 @@ func TestAddBatchCaseD(t *testing.T) { c.Assert(err, qt.IsNil) time2 := time.Since(start) if debug { - debugTime("CASE D, AddBatch", time1, time2) + debugTime("Case tree not empty, AddBatch", time1, time2) printTestContext(" ", nLeafs, "Poseidon", "memory") tree2.dbg.print(" ") } @@ -536,7 +556,7 @@ func TestAddBatchCaseD(t *testing.T) { c.Check(tree2.Root(), qt.DeepEquals, tree1.Root()) } -func TestAddBatchCaseE(t *testing.T) { +func TestAddBatchNotEmptyUnbalanced(t *testing.T) { c := qt.New(t) nLeafs := 4096 @@ -588,7 +608,7 @@ func TestAddBatchCaseE(t *testing.T) { c.Assert(err, qt.IsNil) time2 := time.Since(start) if debug { - debugTime("CASE E, AddBatch", time1, time2) + debugTime("Case tree not empty & unbalanced, AddBatch", time1, time2) printTestContext(" ", nLeafs, "Poseidon", "memory") tree2.dbg.print(" ") } @@ -630,21 +650,21 @@ func benchAdd(t *testing.T, ks, vs [][]byte) { c := qt.New(t) dbDir := t.TempDir() + // storage, err := pebble.NewPebbleStorage(dbDir, false) storage, err := leveldb.NewLevelDbStorage(dbDir, false) c.Assert(err, qt.IsNil) tree, err := NewTree(storage, 140, HashFunctionBlake2b) c.Assert(err, qt.IsNil) - if debug { - tree.dbgInit() - } start := time.Now() for i := 0; i < len(ks); i++ { err = tree.Add(ks[i], vs[i]) c.Assert(err, qt.IsNil) } - printRes(" Add loop", time.Since(start)) - tree.dbg.print(" ") + if debug { + printRes(" Add loop", time.Since(start)) + tree.dbg.print(" ") + } } func benchAddBatch(t *testing.T, ks, vs [][]byte) { @@ -656,15 +676,16 @@ func benchAddBatch(t *testing.T, ks, vs [][]byte) { tree, err := NewTree(storage, 140, HashFunctionBlake2b) c.Assert(err, qt.IsNil) - if debug { - tree.dbgInit() - } + tree.dbgInit() + start := time.Now() invalids, err := tree.AddBatch(ks, vs) - printRes(" AddBatch", time.Since(start)) + if debug { + printRes(" AddBatch", time.Since(start)) + tree.dbg.print(" ") + } c.Assert(err, qt.IsNil) c.Assert(len(invalids), qt.Equals, 0) - tree.dbg.print(" ") } func TestDbgStats(t *testing.T) { @@ -727,9 +748,9 @@ func TestDbgStats(t *testing.T) { if debug { fmt.Println("TestDbgStats") - tree1.dbg.print(" add in loop ") - tree2.dbg.print(" addbatch caseA ") - tree3.dbg.print(" addbatch caseD ") + tree1.dbg.print(" add in loop in emptyTree ") + tree2.dbg.print(" addbatch caseEmptyTree ") + tree3.dbg.print(" addbatch caseNotEmptyTree ") } } @@ -762,5 +783,4 @@ func TestLoadVT(t *testing.T) { c.Check(tree.Root(), qt.DeepEquals, vt.root.h) } -// TODO test adding batch with repeated keys in the batch // TODO test adding batch with multiple invalid keys diff --git a/go.sum b/go.sum index 0f2a6a3..6a40894 100644 --- a/go.sum +++ b/go.sum @@ -45,10 +45,15 @@ github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20201229190758-9e27ae169fdd h1:skK6H2F9tTmNCDxePTMtJbEZ+hqqCtlzNK4BEcVxsvY= github.com/cockroachdb/pebble v0.0.0-20201229190758-9e27ae169fdd/go.mod h1:c3G8ud5zF3+nYHCWmVmtsA8eEtjrDSa6qeLtcRZyevE= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -107,6 +112,7 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -227,6 +233,7 @@ github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssy github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -293,6 +300,7 @@ golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 h1:gzMM0EjIYiRmJI3+jBdFuo golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/tree.go b/tree.go index f5dd91e..14083ad 100644 --- a/tree.go +++ b/tree.go @@ -39,6 +39,9 @@ const ( // PrefixValueIntermediate is used for the first byte of a Value to // indicate that is a Intermediate value PrefixValueIntermediate = 2 + + // nChars is used to crop the Graphviz nodes labels + nChars = 4 ) var ( @@ -852,7 +855,6 @@ func (t *Tree) GraphvizFirstNLevels(w io.Writer, rootKey []byte, untilLvl int) e fmt.Fprintf(w, `digraph hierarchy { node [fontname=Monospace,fontsize=10,shape=box] `) - nChars := 4 // TODO move to global constant nEmpties := 0 err := t.iterWithStop(t.root, 0, func(currLvl int, k, v []byte) bool { if currLvl == untilLvl { diff --git a/vt.go b/vt.go index 422468c..5f9f0f8 100644 --- a/vt.go +++ b/vt.go @@ -593,7 +593,6 @@ node [fontname=Monospace,fontsize=10,shape=box] //nolint:unused func (n *node) graphviz(w io.Writer, p *params, nEmpties int) (int, error) { - nChars := 4 // TODO move to global constant if n == nil { return nEmpties, nil }