M addbatch_test.go → addbatch_test.go

@@ -61,7 +61,7 @@ }
 	return tree1, tree2
 }
 
-func TestAddBatchCaseA(t *testing.T) {
+func TestAddBatchTreeEmpty(t *testing.T) {
 	c := qt.New(t)
 
 	nLeafs := 1024
@@ -97,7 +97,7 @@ indexes, err := tree2.AddBatch(keys, values)
 	c.Assert(err, qt.IsNil)
 	time2 := time.Since(start)
 	if debug {
-		debugTime("CASE A, AddBatch", time1, time2)
+		debugTime("Case tree empty, AddBatch", time1, time2)
 		printTestContext("	", nLeafs, "Poseidon", "memory")
 		tree2.dbg.print("	")
 	}
@@ -107,7 +107,7 @@ // check that both trees roots are equal
 	c.Check(tree2.Root(), qt.DeepEquals, tree.Root())
 }
 
-func TestAddBatchCaseANotPowerOf2(t *testing.T) {
+func TestAddBatchTreeEmptyNotPowerOf2(t *testing.T) {
 	c := qt.New(t)
 
 	nLeafs := 1027
@@ -152,7 +152,7 @@ }
 	return b
 }
 
-func TestAddBatchCaseATestVector(t *testing.T) {
+func TestAddBatchTreeEmptyTestVector(t *testing.T) {
 	c := qt.New(t)
 	tree1, err := NewTree(memory.NewMemoryStorage(), 100, HashFunctionBlake2b)
 	c.Assert(err, qt.IsNil)
@@ -230,7 +230,7 @@ // check that both trees roots are equal
 	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
 }
 
-func TestAddBatchCaseARandomKeys(t *testing.T) {
+func TestAddBatchTreeEmptyRandomKeys(t *testing.T) {
 	c := qt.New(t)
 
 	nLeafs := 8
@@ -246,21 +246,9 @@
 	var keys, values [][]byte
 	for i := 0; i < nLeafs; i++ {
 		keys = append(keys, randomBytes(32))
-		// values = append(values, randomBytes(32))
-		values = append(values, []byte{0})
-		// fmt.Println("K", hex.EncodeToString(keys[i]))
+		values = append(values, randomBytes(32))
 	}
 
-	// TODO delete
-	keys[0], _ = hex.DecodeString("1c7c2265e368314ca58ed2e1f33a326f1220e234a566d55c3605439dbe411642")
-	keys[1], _ = hex.DecodeString("2c9f0a578afff5bfa4e0992a43066460faaab9e8e500db0b16647c701cdb16bf")
-	keys[2], _ = hex.DecodeString("9cb87ec67e875c61390edcd1ab517f443591047709a4d4e45b0f9ed980857b8e")
-	keys[3], _ = hex.DecodeString("9b4e9e92e974a589f426ceeb4cb291dc24893513fecf8e8460992dcf52621d4d")
-	keys[4], _ = hex.DecodeString("1c45cb31f2fa39ec7b9ebf0fad40e0b8296016b5ce8844ae06ff77226379d9a5")
-	keys[5], _ = hex.DecodeString("d8af98bbbb585129798ae54d5eabbc9d0561d583faf1663b3a3724d15bda4ec7")
-	keys[6], _ = hex.DecodeString("3cd55dbfb8f975f20a0925dfbdabe79fa2d51dd0268afbb8ba6b01de9dfcdd3c")
-	keys[7], _ = hex.DecodeString("5d0a9d6d9f197c091bf054fac9cb60e11ec723d6610ed8578e617b4d46cb43d5")
-
 	for i := 0; i < len(keys); i++ {
 		if err := tree1.Add(keys[i], values[i]); err != nil {
 			t.Fatal(err)
@@ -269,20 +257,16 @@ }
 
 	indexes, err := tree2.AddBatch(keys, values)
 	c.Assert(err, qt.IsNil)
-	// tree1.PrintGraphviz(nil)
-	// tree2.PrintGraphviz(nil)
-
 	c.Check(len(indexes), qt.Equals, 0)
-
 	// check that both trees roots are equal
 	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
 }
 
-func TestAddBatchCaseB(t *testing.T) {
+func TestAddBatchTreeNotEmptyFewLeafs(t *testing.T) {
 	c := qt.New(t)
 
 	nLeafs := 1024
-	initialNLeafs := 99 // TMP TODO use const minLeafsThreshold-1 once ready
+	initialNLeafs := 99
 
 	tree1, tree2 := testInit(c, initialNLeafs)
 	tree2.dbgInit()
@@ -310,7 +294,7 @@ indexes, err := tree2.AddBatch(keys, values)
 	c.Assert(err, qt.IsNil)
 	time2 := time.Since(start)
 	if debug {
-		debugTime("CASE B, AddBatch", time1, time2)
+		debugTime("Case tree not empty w/ few leafs, AddBatch", time1, time2)
 		printTestContext("	", nLeafs, "Poseidon", "memory")
 		tree2.dbg.print("	")
 	}
@@ -320,14 +304,16 @@ // check that both trees roots are equal
 	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
 }
 
-func TestAddBatchCaseBRepeatedLeafs(t *testing.T) {
+func TestAddBatchTreeNotEmptyEnoughLeafs(t *testing.T) {
 	c := qt.New(t)
 
 	nLeafs := 1024
-	initialNLeafs := 99 // TMP TODO use const minLeafsThreshold-1 once ready
+	initialNLeafs := 500
 
 	tree1, tree2 := testInit(c, initialNLeafs)
+	tree2.dbgInit()
 
+	start := time.Now()
 	for i := initialNLeafs; i < nLeafs; i++ {
 		k := BigIntToBytes(big.NewInt(int64(i)))
 		v := BigIntToBytes(big.NewInt(int64(i * 2)))
@@ -335,8 +321,39 @@ if err := tree1.Add(k, v); err != nil {
 			t.Fatal(err)
 		}
 	}
+	time1 := time.Since(start)
 
-	// prepare the key-values to be added, including already existing keys
+	// prepare the key-values to be added
+	var keys, values [][]byte
+	for i := initialNLeafs; i < nLeafs; i++ {
+		k := BigIntToBytes(big.NewInt(int64(i)))
+		v := BigIntToBytes(big.NewInt(int64(i * 2)))
+		keys = append(keys, k)
+		values = append(values, v)
+	}
+	start = time.Now()
+	indexes, err := tree2.AddBatch(keys, values)
+	c.Assert(err, qt.IsNil)
+	time2 := time.Since(start)
+	if debug {
+		debugTime("Case tree not empty w/ enough leafs, AddBatch", time1, time2)
+		printTestContext("	", nLeafs, "Poseidon", "memory")
+		tree2.dbg.print("	")
+	}
+	c.Check(len(indexes), qt.Equals, 0)
+	// check that both trees roots are equal
+	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
+}
+
+func TestAddBatchTreeEmptyRepeatedLeafs(t *testing.T) {
+	c := qt.New(t)
+
+	nLeafs := 1024
+	nRepeatedKeys := 99
+
+	tree1, tree2 := testInit(c, 0)
+
+	// prepare the key-values to be added
 	var keys, values [][]byte
 	for i := 0; i < nLeafs; i++ {
 		k := BigIntToBytes(big.NewInt(int64(i)))
@@ -344,10 +361,55 @@ v := BigIntToBytes(big.NewInt(int64(i * 2)))
 		keys = append(keys, k)
 		values = append(values, v)
 	}
+	// add repeated key-values
+	for i := 0; i < nRepeatedKeys; i++ {
+		k := BigIntToBytes(big.NewInt(int64(i)))
+		v := BigIntToBytes(big.NewInt(int64(i * 2)))
+		keys = append(keys, k)
+		values = append(values, v)
+	}
+
+	// add the non-repeated key-values in tree1 with .Add loop
+	for i := 0; i < nLeafs; i++ {
+		if err := tree1.Add(keys[i], values[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
 	indexes, err := tree2.AddBatch(keys, values)
 	c.Assert(err, qt.IsNil)
-	c.Check(len(indexes), qt.Equals, initialNLeafs)
+	c.Check(len(indexes), qt.Equals, nRepeatedKeys)
+	// check that both trees roots are equal
+	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
+}
+
+func TestAddBatchTreeNotEmptyFewLeafsRepeatedLeafs(t *testing.T) {
+	c := qt.New(t)
+
+	nLeafs := 1024
+	initialNLeafs := 99
 
+	tree1, tree2 := testInit(c, initialNLeafs)
+
+	// prepare the key-values to be added
+	var keys, values [][]byte
+	for i := 0; i < nLeafs; i++ {
+		k := BigIntToBytes(big.NewInt(int64(i)))
+		v := BigIntToBytes(big.NewInt(int64(i * 2)))
+		keys = append(keys, k)
+		values = append(values, v)
+	}
+
+	// add the keys that will be existing when AddBatch is called
+	for i := initialNLeafs; i < nLeafs; i++ {
+		if err := tree1.Add(keys[i], values[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	indexes, err := tree2.AddBatch(keys, values)
+	c.Assert(err, qt.IsNil)
+	c.Check(len(indexes), qt.Equals, initialNLeafs)
 	// check that both trees roots are equal
 	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
 }
@@ -452,49 +514,7 @@ return compareBytes(kvs[i].keyPath, kvs[j].keyPath)
 	})
 }
 
-func TestAddBatchCaseC(t *testing.T) {
-	c := qt.New(t)
-
-	nLeafs := 1024
-	initialNLeafs := 101 // TMP TODO use const minLeafsThreshold+1 once ready
-
-	tree1, tree2 := testInit(c, initialNLeafs)
-	tree2.dbgInit()
-
-	start := time.Now()
-	for i := initialNLeafs; i < nLeafs; i++ {
-		k := BigIntToBytes(big.NewInt(int64(i)))
-		v := BigIntToBytes(big.NewInt(int64(i * 2)))
-		if err := tree1.Add(k, v); err != nil {
-			t.Fatal(err)
-		}
-	}
-	time1 := time.Since(start)
-
-	// prepare the key-values to be added
-	var keys, values [][]byte
-	for i := initialNLeafs; i < nLeafs; i++ {
-		k := BigIntToBytes(big.NewInt(int64(i)))
-		v := BigIntToBytes(big.NewInt(int64(i * 2)))
-		keys = append(keys, k)
-		values = append(values, v)
-	}
-	start = time.Now()
-	indexes, err := tree2.AddBatch(keys, values)
-	c.Assert(err, qt.IsNil)
-	time2 := time.Since(start)
-	if debug {
-		debugTime("CASE C, AddBatch", time1, time2)
-		printTestContext("	", nLeafs, "Poseidon", "memory")
-		tree2.dbg.print("	")
-	}
-	c.Check(len(indexes), qt.Equals, 0)
-
-	// check that both trees roots are equal
-	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
-}
-
-func TestAddBatchCaseD(t *testing.T) {
+func TestAddBatchTreeNotEmpty(t *testing.T) {
 	c := qt.New(t)
 
 	nLeafs := 4096
@@ -526,7 +546,7 @@ indexes, err := tree2.AddBatch(keys, values)
 	c.Assert(err, qt.IsNil)
 	time2 := time.Since(start)
 	if debug {
-		debugTime("CASE D, AddBatch", time1, time2)
+		debugTime("Case tree not empty, AddBatch", time1, time2)
 		printTestContext("	", nLeafs, "Poseidon", "memory")
 		tree2.dbg.print("	")
 	}
@@ -536,7 +556,7 @@ // check that both trees roots are equal
 	c.Check(tree2.Root(), qt.DeepEquals, tree1.Root())
 }
 
-func TestAddBatchCaseE(t *testing.T) {
+func TestAddBatchNotEmptyUnbalanced(t *testing.T) {
 	c := qt.New(t)
 
 	nLeafs := 4096
@@ -588,7 +608,7 @@ indexes, err := tree2.AddBatch(keys, values)
 	c.Assert(err, qt.IsNil)
 	time2 := time.Since(start)
 	if debug {
-		debugTime("CASE E, AddBatch", time1, time2)
+		debugTime("Case tree not empty & unbalanced, AddBatch", time1, time2)
 		printTestContext("	", nLeafs, "Poseidon", "memory")
 		tree2.dbg.print("	")
 	}
@@ -630,21 +650,21 @@ func benchAdd(t *testing.T, ks, vs [][]byte) {
 	c := qt.New(t)
 
 	dbDir := t.TempDir()
+	// storage, err := pebble.NewPebbleStorage(dbDir, false)
 	storage, err := leveldb.NewLevelDbStorage(dbDir, false)
 	c.Assert(err, qt.IsNil)
 	tree, err := NewTree(storage, 140, HashFunctionBlake2b)
 	c.Assert(err, qt.IsNil)
 
-	if debug {
-		tree.dbgInit()
-	}
 	start := time.Now()
 	for i := 0; i < len(ks); i++ {
 		err = tree.Add(ks[i], vs[i])
 		c.Assert(err, qt.IsNil)
 	}
-	printRes("	Add loop", time.Since(start))
-	tree.dbg.print("		")
+	if debug {
+		printRes("	Add loop", time.Since(start))
+		tree.dbg.print("		")
+	}
 }
 
 func benchAddBatch(t *testing.T, ks, vs [][]byte) {
@@ -656,15 +676,16 @@ c.Assert(err, qt.IsNil)
 	tree, err := NewTree(storage, 140, HashFunctionBlake2b)
 	c.Assert(err, qt.IsNil)
 
+	tree.dbgInit()
+
+	start := time.Now()
+	invalids, err := tree.AddBatch(ks, vs)
 	if debug {
-		tree.dbgInit()
+		printRes("	AddBatch", time.Since(start))
+		tree.dbg.print("		")
 	}
-	start := time.Now()
-	invalids, err := tree.AddBatch(ks, vs)
-	printRes("	AddBatch", time.Since(start))
 	c.Assert(err, qt.IsNil)
 	c.Assert(len(invalids), qt.Equals, 0)
-	tree.dbg.print("		")
 }
 
 func TestDbgStats(t *testing.T) {
@@ -727,9 +748,9 @@ c.Check(tree3.Root(), qt.DeepEquals, tree1.Root())
 
 	if debug {
 		fmt.Println("TestDbgStats")
-		tree1.dbg.print("	add in loop    ")
-		tree2.dbg.print("	addbatch caseA ")
-		tree3.dbg.print("	addbatch caseD ")
+		tree1.dbg.print("	add in loop in emptyTree ")
+		tree2.dbg.print("	addbatch caseEmptyTree ")
+		tree3.dbg.print("	addbatch caseNotEmptyTree ")
 	}
 }
 
@@ -762,5 +783,4 @@ // check that tree & vt roots are equal
 	c.Check(tree.Root(), qt.DeepEquals, vt.root.h)
 }
 
-// TODO test adding batch with repeated keys in the batch
 // TODO test adding batch with multiple invalid keys

M go.sum → go.sum

@@ -45,10 +45,15 @@ github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
 github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
+github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y=
 github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/cockroachdb/pebble v0.0.0-20201229190758-9e27ae169fdd h1:skK6H2F9tTmNCDxePTMtJbEZ+hqqCtlzNK4BEcVxsvY=
 github.com/cockroachdb/pebble v0.0.0-20201229190758-9e27ae169fdd/go.mod h1:c3G8ud5zF3+nYHCWmVmtsA8eEtjrDSa6qeLtcRZyevE=
+github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw=
 github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
@@ -107,6 +112,7 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -227,6 +233,7 @@ github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -293,6 +300,7 @@ golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 h1:gzMM0EjIYiRmJI3+jBdFuoynZlpxa2JQZsolKu09BXo=
 golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc=
 golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

M tree.go → tree.go

@@ -39,6 +39,9 @@ PrefixValueLeaf = 1
 	// PrefixValueIntermediate is used for the first byte of a Value to
 	// indicate that is a Intermediate value
 	PrefixValueIntermediate = 2
+
+	// nChars is used to crop the Graphviz nodes labels
+	nChars = 4
 )
 
 var (
@@ -852,7 +855,6 @@ func (t *Tree) GraphvizFirstNLevels(w io.Writer, rootKey []byte, untilLvl int) error {
 	fmt.Fprintf(w, `digraph hierarchy {
 node [fontname=Monospace,fontsize=10,shape=box]
 `)
-	nChars := 4 // TODO move to global constant
 	nEmpties := 0
 	err := t.iterWithStop(t.root, 0, func(currLvl int, k, v []byte) bool {
 		if currLvl == untilLvl {

M vt.go → vt.go

@@ -593,7 +593,6 @@ }
 
 //nolint:unused
 func (n *node) graphviz(w io.Writer, p *params, nEmpties int) (int, error) {
-	nChars := 4 // TODO move to global constant
 	if n == nil {
 		return nEmpties, nil
 	}