arnaucube
arnaucube/asmtPublic Self-hosted repository
apply golang-ci suggestions

Signed-off-by: p4u <pau@dabax.net>
p4u pau@dabax.net
Fri, 26 Mar 2021 19:47:07 +0100
commit

d69f5d6618b2c4bea9afae116b1bf24db2ea22ee

parent

d855f4fb146d2c538ab79e098119c88ecbab0ab6

6 files changed, 130 insertions(+), 66 deletions(-)

jump to
M censustree.go → censustree.go

@@ -19,7 +19,7 @@ // We use go-bare for export/import the trie. In order to support

// big census (up to 8 Million entries) we need to increase the maximums. const bareMaxArrayLength uint64 = 1024 * 1014 * 8 // 8 Million -const bareMaxUnmarshalBytes uint64 = 1024 * 1024 * 200 // 200 MiB +const bareMaxUnmarshalBytes uint64 = bareMaxArrayLength * 32 * 2 // 512 MiB type Tree struct { Tree *asmt.Trie

@@ -48,8 +48,8 @@ Elements []exportElement `bare:"elements"`

} const ( - MaxKeySize = 256 - MaxValueSize = 256 + MaxKeySize = 32 + MaxValueSize = 64 dbRootPrefix = "this is the last root for the SMT tree" )
M go.mod → go.mod

@@ -8,6 +8,6 @@ github.com/aergoio/aergo-lib v1.0.2

github.com/guptarohit/asciigraph v0.4.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca - go.vocdoni.io/dvote v1.0.0 + go.vocdoni.io/dvote v0.6.1-0.20210326182730-ba86106de602 golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 )
M go.sum → go.sum

@@ -1547,11 +1547,11 @@ go.vocdoni.io/dvote v0.6.1-0.20210125120603-df82783bb500/go.mod h1:Kf3LwprU/2MhEXLmgL34N12mn3TmhJZ0EnPPL7LoNXs=

go.vocdoni.io/dvote v0.6.1-0.20210129144810-e6ce68464ed2/go.mod h1:7yTzMiJiZY3E9djQAuEsjEuDhflLcVJKqd1chYC1GVI= go.vocdoni.io/dvote v0.6.1-0.20210130094936-75dbb92de3f0/go.mod h1:NJyERGs2LmEvlevZUtOl36MI0HlOUh0d2I0vJcYh5RE= go.vocdoni.io/dvote v0.6.1-0.20210206210936-a0407e833753/go.mod h1:HhMMtdnkLI3ujk+zcL1frzFg5S7SbvxWtis7RZTq1dA= -go.vocdoni.io/dvote v1.0.0 h1:a6sSDmeyLfwBZ9b++2UI+BI9KPLglpp7KRuVOSEDfgo= -go.vocdoni.io/dvote v1.0.0/go.mod h1:C9wjSWCzy33Bl2sNiF/2ie/oxcERR3uRD2G2c/NDeH0= +go.vocdoni.io/dvote v0.6.1-0.20210326182730-ba86106de602 h1:qVDdnaU0vaWST5g/pt2P5Pg/gzqTuZfWQFAx7ChD/HQ= +go.vocdoni.io/dvote v0.6.1-0.20210326182730-ba86106de602/go.mod h1:TXzXfwGzoRhINzmiCtsuVnMya62WclVeFMUcWP29UwY= go.vocdoni.io/proto v0.1.7/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M= go.vocdoni.io/proto v0.1.8/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M= -go.vocdoni.io/proto v0.1.9-0.20210304214308-6f7363b52750/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M= +go.vocdoni.io/proto v1.0.2/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/arch v0.0.0-20190919213554-7fe50f7625bd/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
M smt/trie.go → smt/trie.go

@@ -137,7 +137,7 @@ } else {

// create a new shortcut batch. // simply storing the value will make it hard to move up the // shortcut in case of sibling deletion - batch = make([][]byte, 31, 31) + batch = make([][]byte, 31) node := s.leafHash(keys[0], values[0], root, batch, 0, height) ch <- mresult{node, false, nil} }

@@ -413,7 +413,7 @@ isShortcut := false

if height%4 == 0 { if len(root) == 0 { // create a new default batch - batch = make([][]byte, 31, 31) + batch = make([][]byte, 31) batch[0] = []byte{0} } else { var err error

@@ -452,7 +452,7 @@ if s.atomicUpdate {

// Return a copy so that Commit() doesnt have to be called at // each block and still commit every state transition. // Before Commit, the same batch is in liveCache and in updatedNodes - newVal := make([][]byte, 31, 31) + newVal := make([][]byte, 31) copy(newVal, val) return newVal, nil }

@@ -466,7 +466,7 @@ if exists {

if s.atomicUpdate { // Return a copy so that Commit() doesnt have to be called at // each block and still commit every state transition. - newVal := make([][]byte, 31, 31) + newVal := make([][]byte, 31) copy(newVal, val) return newVal, nil }

@@ -493,7 +493,7 @@ }

// parseBatch decodes the byte data into a slice of nodes and bitmap func (s *Trie) parseBatch(val []byte) [][]byte { - batch := make([][]byte, 31, 31) + batch := make([][]byte, 31) bitmap := val[:4] // check if the batch root is a shortcut if bitIsSet(val, 31) {
M smt/trie_test.go → smt/trie_test.go

@@ -77,7 +77,9 @@ root, _ := smt.AtomicUpdate(keys, values)

updatedNb := len(smt.db.updatedNodes) cacheNb := len(smt.db.liveCache) newvalues := getFreshData(1, 32) - smt.AtomicUpdate(keys, newvalues) + if _, err := smt.AtomicUpdate(keys, newvalues); err != nil { + t.Fatal(err) + } if len(smt.db.updatedNodes) != 2*updatedNb { t.Fatal("Atomic update doesnt store all tries") }

@@ -118,7 +120,9 @@ t.Fatal("Root not stored")

} newValues := getFreshData(20, 32) - smt.Update(keys, newValues) + if _, err := smt.Update(keys, newValues); err != nil { + t.Fatal(err) + } if len(smt.db.updatedNodes) != updatedNb { t.Fatal("multiple updates don't actualise updated nodes")

@@ -354,9 +358,11 @@ smt = NewTrie(nil, Hasher, nil)

keys = getFreshData(2, 32) values = getFreshData(2, 32) root, _ = smt.Update(keys, values) - key0 := make([]byte, 32, 32) - key1 := make([]byte, 32, 32) - smt.Update([][]byte{key0, key1}, [][]byte{DefaultLeaf, DefaultLeaf}) + key0 := make([]byte, 32) + key1 := make([]byte, 32) + if _, err := smt.Update([][]byte{key0, key1}, [][]byte{DefaultLeaf, DefaultLeaf}); err != nil { + t.Fatal(err) + } if !bytes.Equal(root, smt.Root) { t.Fatal("deleting a default key shouldnt' modify the tree") }

@@ -366,7 +372,7 @@ // test updating and deleting at the same time

func TestTrieUpdateAndDelete(t *testing.T) { smt := NewTrie(nil, Hasher, nil) smt.CacheHeightLimit = 0 - key0 := make([]byte, 32, 32) + key0 := make([]byte, 32) values := getFreshData(1, 32) root, _ := smt.Update([][]byte{key0}, values) cacheNb := len(smt.db.liveCache)

@@ -377,7 +383,7 @@ if !isShortcut || !bytes.Equal(k[:HashLength], key0) || !bytes.Equal(v[:HashLength], values[0]) {

t.Fatal("leaf shortcut didn't move up to root") } - key1 := make([]byte, 32, 32) + key1 := make([]byte, 32) // set the last bit bitSet(key1, 255) keys := [][]byte{key0, key1}

@@ -403,7 +409,9 @@ smt := NewTrie(nil, Hasher, nil)

// Add data to empty trie keys := getFreshData(10, 32) values := getFreshData(10, 32) - smt.Update(keys, values) + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } for i, key := range keys { ap, _, k, v, _ := smt.MerkleProof(key)

@@ -429,7 +437,9 @@ smt := NewTrie(nil, Hasher, nil)

// Add data to empty trie keys := getFreshData(10, 32) values := getFreshData(10, 32) - smt.Update(keys, values) + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } for i, key := range keys { bitmap, ap, length, _, k, v, _ := smt.MerkleProofCompressed(key)

@@ -460,8 +470,12 @@

smt := NewTrie(nil, Hasher, st) keys := getFreshData(10, 32) values := getFreshData(10, 32) - smt.Update(keys, values) - smt.Commit() + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } + if err := smt.Commit(); err != nil { + t.Fatal(err) + } // liveCache is deleted so the key is fetched in badger db smt.db.liveCache = make(map[Hash][][]byte) for i, key := range keys {

@@ -484,7 +498,9 @@

smt := NewTrie(nil, Hasher, st) keys := getFreshData(10, 32) values := getFreshData(10, 32) - smt.Update(keys, values) + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } txn := st.NewTx() smt.StageUpdates(txn.(DbTx)) txn.Commit()

@@ -511,52 +527,75 @@ smt := NewTrie(nil, Hasher, st)

// Edge case : Test that revert doesnt delete shortcut nodes // when moved to a different position in tree - key0 := make([]byte, 32, 32) - key1 := make([]byte, 32, 32) + key0 := make([]byte, 32) + key1 := make([]byte, 32) // setting the bit at 251 creates 2 shortcut batches at height 252 bitSet(key1, 251) values := getFreshData(2, 32) - keys := [][]byte{key0, key1} root, _ := smt.Update([][]byte{key0}, [][]byte{values[0]}) - smt.Commit() + if err := smt.Commit(); err != nil { + t.Fatal(err) + } root2, _ := smt.Update([][]byte{key1}, [][]byte{values[1]}) - smt.Commit() - smt.Revert(root) + if err := smt.Commit(); err != nil { + t.Fatal(err) + } + if err := smt.Revert(root); err != nil { + t.Fatal(err) + } if len(smt.db.Store.Get(root)) == 0 { t.Fatal("shortcut node shouldnt be deleted by revert") } if len(smt.db.Store.Get(root2)) != 0 { t.Fatal("reverted root should have been deleted") } - key1 = make([]byte, 32, 32) + key1 = make([]byte, 32) // setting the bit at 255 stores the keys as the tip bitSet(key1, 255) - smt.Update([][]byte{key1}, [][]byte{values[1]}) - smt.Commit() - smt.Revert(root) + if _, err := smt.Update([][]byte{key1}, [][]byte{values[1]}); err != nil { + t.Fatal(err) + } + if err := smt.Commit(); err != nil { + t.Fatal(err) + } + if err := smt.Revert(root); err != nil { + t.Fatal(err) + } if len(smt.db.Store.Get(root)) == 0 { t.Fatal("shortcut node shouldnt be deleted by revert") } // Test all nodes are reverted in the usual case // Add data to empty trie - keys = getFreshData(10, 32) + keys := getFreshData(10, 32) values = getFreshData(10, 32) root, _ = smt.Update(keys, values) - smt.Commit() + if err := smt.Commit(); err != nil { + t.Fatal(err) + } // Update the values newValues := getFreshData(10, 32) - smt.Update(keys, newValues) + if _, err := smt.Update(keys, newValues); err != nil { + t.Fatal(err) + } updatedNodes1 := smt.db.updatedNodes - smt.Commit() + if err := smt.Commit(); err != nil { + t.Fatal(err) + } newKeys := getFreshData(10, 32) newValues = getFreshData(10, 32) - smt.Update(newKeys, newValues) + if _, err := smt.Update(newKeys, newValues); err != nil { + t.Fatal(err) + } updatedNodes2 := smt.db.updatedNodes - smt.Commit() + if err := smt.Commit(); err != nil { + t.Fatal(err) + } - smt.Revert(root) + if err := smt.Revert(root); err != nil { + t.Fatal(err) + } if !bytes.Equal(smt.Root, root) { t.Fatal("revert failed")

@@ -575,12 +614,12 @@ if len(smt.db.liveCache) != 0 {

t.Fatal("live cache not reset after revert") } // Check all reverted nodes have been deleted - for node, _ := range updatedNodes2 { + for node := range updatedNodes2 { if len(smt.db.Store.Get(node[:])) != 0 { t.Fatal("nodes not deleted from database", node) } } - for node, _ := range updatedNodes1 { + for node := range updatedNodes1 { if len(smt.db.Store.Get(node[:])) != 0 { t.Fatal("nodes not deleted from database", node) }

@@ -600,7 +639,9 @@ smt := NewTrie(nil, Hasher, st)

// Add data to empty trie keys := getFreshData(10, 32) values := getFreshData(10, 32) - smt.Update(keys, values) + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } smt.db.liveCache = make(map[Hash][][]byte) smt.db.updatedNodes = make(map[Hash][][]byte)

@@ -629,7 +670,7 @@ t.Fatal("Error not created if database not connected")

} smt.db.liveCache = make(map[Hash][][]byte) smt.atomicUpdate = false - _, _, _, _, _, err = smt.loadChildren(make([]byte, 32, 32), smt.TrieHeight, 0, nil) + _, _, _, _, _, err = smt.loadChildren(make([]byte, 32), smt.TrieHeight, 0, nil) if err == nil { t.Fatal("Error not created if database not connected") }

@@ -649,11 +690,13 @@

smt := NewTrie(nil, Hasher, st) // Test size of cache smt.CacheHeightLimit = 0 - key0 := make([]byte, 32, 32) - key1 := make([]byte, 32, 32) + key0 := make([]byte, 32) + key1 := make([]byte, 32) bitSet(key1, 255) values := getFreshData(2, 32) - smt.Update([][]byte{key0, key1}, values) + if _, err := smt.Update([][]byte{key0, key1}, values); err != nil { + t.Fatal(err) + } if len(smt.db.liveCache) != 66 { // the nodes are at the tip, so 64 + 2 = 66 t.Fatal("cache size incorrect")

@@ -662,8 +705,12 @@

// Add data to empty trie keys := getFreshData(10, 32) values = getFreshData(10, 32) - smt.Update(keys, values) - smt.Commit() + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } + if err := smt.Commit(); err != nil { + t.Fatal(err) + } // Simulate node restart by deleting and loading cache cacheSize := len(smt.db.liveCache)

@@ -685,12 +732,14 @@ func TestHeight0LeafShortcut(t *testing.T) {

keySize := 32 smt := NewTrie(nil, Hasher, nil) // Add 2 sibling keys that will be stored at height 0 - key0 := make([]byte, keySize, keySize) - key1 := make([]byte, keySize, keySize) + key0 := make([]byte, keySize) + key1 := make([]byte, keySize) bitSet(key1, keySize*8-1) keys := [][]byte{key0, key1} values := getFreshData(2, 32) - smt.Update(keys, values) + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } updatedNb := len(smt.db.updatedNodes) // Check all keys have been stored

@@ -752,13 +801,19 @@ keys := getFreshData(20, 32)

values := getFreshData(20, 32) root, _ := smt.Update(keys, values) cacheSize := len(smt.db.liveCache) - smt.Commit() + if err := smt.Commit(); err != nil { + t.Fatal(err) + } if len(smt.pastTries) != 1 { t.Fatal("Past tries not updated after commit") } values = getFreshData(20, 32) - smt.Update(keys, values) - smt.Stash(true) + if _, err := smt.Update(keys, values); err != nil { + t.Fatal(err) + } + if err := smt.Stash(true); err != nil { + t.Fatal(err) + } if len(smt.pastTries) != 1 { t.Fatal("Past tries not updated after commit") }

@@ -773,13 +828,19 @@ t.Fatal("Trie not rolled back")

} keys = getFreshData(20, 32) values = getFreshData(20, 32) - smt.AtomicUpdate(keys, values) + if _, err := smt.AtomicUpdate(keys, values); err != nil { + t.Fatal(err) + } values = getFreshData(20, 32) - smt.AtomicUpdate(keys, values) + if _, err := smt.AtomicUpdate(keys, values); err != nil { + t.Fatal(err) + } if len(smt.pastTries) != 3 { t.Fatal("Past tries not updated after commit") } - smt.Stash(true) + if err := smt.Stash(true); err != nil { + t.Fatal(err) + } if !bytes.Equal(smt.Root, root) { t.Fatal("Trie not rolled back") }

@@ -800,15 +861,21 @@ func benchmark10MAccounts10Ktps(smt *Trie, b *testing.B) {

//b.ReportAllocs() keys := getFreshData(100, 32) values := getFreshData(100, 32) - smt.Update(keys, values) + if _, err := smt.Update(keys, values); err != nil { + b.Fatal(err) + } fmt.Println("\nLoading b.N x 1000 accounts") for i := 0; i < b.N; i++ { newkeys := getFreshData(1000, 32) newvalues := getFreshData(1000, 32) start := time.Now() - smt.Update(newkeys, newvalues) + if _, err := smt.Update(newkeys, newvalues); err != nil { + b.Fatal(err) + } end := time.Now() - smt.Commit() + if err := smt.Commit(); err != nil { + b.Fatal(err) + } end2 := time.Now() for j, key := range newkeys { val, _ := smt.Get(key)
M smt/trie_tools.go → smt/trie_tools.go

@@ -201,10 +201,7 @@ func (s *Trie) TrieRootExists(root []byte) bool {

s.db.lock.RLock() dbval := s.db.Store.Get(root) s.db.lock.RUnlock() - if len(dbval) != 0 { - return true - } - return false + return len(dbval) != 0 } // Commit stores the updated nodes to disk.