arnaucube/asmt
1
0
Fork 0
mirror of https://github.com/arnaucube/asmt.git synced 2026-02-06 19:06:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

apply golang-ci suggestions

Browse Source 
Signed-off-by: p4u <pau@dabax.net>
This commit is contained in:
p4u
2021-03-26 19:47:07 +01:00
parent d855f4fb14
commit d69f5d6618
6 changed files with 130 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
censustree.go
View File

@@ -19,7 +19,7 @@ import (
// big census (up to 8 Million entries) we need to increase the maximums.
const bareMaxArrayLength uint64 = 1024 * 1014 * 8 // 8 Million
const bareMaxUnmarshalBytes uint64 = 1024 * 1024 * 200 // 200 MiB
const bareMaxUnmarshalBytes uint64 = bareMaxArrayLength * 32 * 2 // 512 MiB
type Tree struct {
Tree *asmt.Trie
@@ -48,8 +48,8 @@ type exportData struct {
}
const (
MaxKeySize = 256
MaxValueSize = 256
MaxKeySize = 32
MaxValueSize = 64
dbRootPrefix = "this is the last root for the SMT tree"
)

2
go.mod
View File

@@ -8,6 +8,6 @@ require (
github.com/guptarohit/asciigraph v0.4.1
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
go.vocdoni.io/dvote v1.0.0
go.vocdoni.io/dvote v0.6.1-0.20210326182730-ba86106de602
golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
)

6
go.sum
View File

@@ -1547,11 +1547,11 @@ go.vocdoni.io/dvote v0.6.1-0.20210125120603-df82783bb500/go.mod h1:Kf3LwprU/2MhE
go.vocdoni.io/dvote v0.6.1-0.20210129144810-e6ce68464ed2/go.mod h1:7yTzMiJiZY3E9djQAuEsjEuDhflLcVJKqd1chYC1GVI=
go.vocdoni.io/dvote v0.6.1-0.20210130094936-75dbb92de3f0/go.mod h1:NJyERGs2LmEvlevZUtOl36MI0HlOUh0d2I0vJcYh5RE=
go.vocdoni.io/dvote v0.6.1-0.20210206210936-a0407e833753/go.mod h1:HhMMtdnkLI3ujk+zcL1frzFg5S7SbvxWtis7RZTq1dA=
go.vocdoni.io/dvote v1.0.0 h1:a6sSDmeyLfwBZ9b++2UI+BI9KPLglpp7KRuVOSEDfgo=
go.vocdoni.io/dvote v1.0.0/go.mod h1:C9wjSWCzy33Bl2sNiF/2ie/oxcERR3uRD2G2c/NDeH0=
go.vocdoni.io/dvote v0.6.1-0.20210326182730-ba86106de602 h1:qVDdnaU0vaWST5g/pt2P5Pg/gzqTuZfWQFAx7ChD/HQ=
go.vocdoni.io/dvote v0.6.1-0.20210326182730-ba86106de602/go.mod h1:TXzXfwGzoRhINzmiCtsuVnMya62WclVeFMUcWP29UwY=
go.vocdoni.io/proto v0.1.7/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M=
go.vocdoni.io/proto v0.1.8/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M=
go.vocdoni.io/proto v0.1.9-0.20210304214308-6f7363b52750/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M=
go.vocdoni.io/proto v1.0.2/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
golang.org/x/arch v0.0.0-20190919213554-7fe50f7625bd/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=

10
smt/trie.go
View File

@@ -137,7 +137,7 @@ func (s *Trie) update(root []byte, keys, values, batch [][]byte, iBatch, height
// create a new shortcut batch.
// simply storing the value will make it hard to move up the
// shortcut in case of sibling deletion
batch = make([][]byte, 31, 31)
batch = make([][]byte, 31)
node := s.leafHash(keys[0], values[0], root, batch, 0, height)
ch <- mresult{node, false, nil}
}
@@ -413,7 +413,7 @@ func (s *Trie) loadChildren(root []byte, height, iBatch int, batch [][]byte) ([]
if height%4 == 0 {
if len(root) == 0 {
// create a new default batch
batch = make([][]byte, 31, 31)
batch = make([][]byte, 31)
batch[0] = []byte{0}
} else {
var err error
@@ -452,7 +452,7 @@ func (s *Trie) loadBatch(root []byte) ([][]byte, error) {
// Return a copy so that Commit() doesnt have to be called at
// each block and still commit every state transition.
// Before Commit, the same batch is in liveCache and in updatedNodes
newVal := make([][]byte, 31, 31)
newVal := make([][]byte, 31)
copy(newVal, val)
return newVal, nil
}
@@ -466,7 +466,7 @@ func (s *Trie) loadBatch(root []byte) ([][]byte, error) {
if s.atomicUpdate {
// Return a copy so that Commit() doesnt have to be called at
// each block and still commit every state transition.
newVal := make([][]byte, 31, 31)
newVal := make([][]byte, 31)
copy(newVal, val)
return newVal, nil
}
@@ -493,7 +493,7 @@ func (s *Trie) loadBatch(root []byte) ([][]byte, error) {
// parseBatch decodes the byte data into a slice of nodes and bitmap
func (s *Trie) parseBatch(val []byte) [][]byte {
batch := make([][]byte, 31, 31)
batch := make([][]byte, 31)
bitmap := val[:4]
// check if the batch root is a shortcut
if bitIsSet(val, 31) {

167
smt/trie_test.go
View File

@@ -77,7 +77,9 @@ func TestTrieAtomicUpdate(t *testing.T) {
updatedNb := len(smt.db.updatedNodes)
cacheNb := len(smt.db.liveCache)
newvalues := getFreshData(1, 32)
smt.AtomicUpdate(keys, newvalues)
if _, err := smt.AtomicUpdate(keys, newvalues); err != nil {
t.Fatal(err)
}
if len(smt.db.updatedNodes) != 2*updatedNb {
t.Fatal("Atomic update doesnt store all tries")
}
@@ -118,7 +120,9 @@ func TestTriePublicUpdateAndGet(t *testing.T) {
}
newValues := getFreshData(20, 32)
smt.Update(keys, newValues)
if _, err := smt.Update(keys, newValues); err != nil {
t.Fatal(err)
}
if len(smt.db.updatedNodes) != updatedNb {
t.Fatal("multiple updates don't actualise updated nodes")
@@ -354,9 +358,11 @@ func TestTrieDelete(t *testing.T) {
keys = getFreshData(2, 32)
values = getFreshData(2, 32)
root, _ = smt.Update(keys, values)
key0 := make([]byte, 32, 32)
key1 := make([]byte, 32, 32)
smt.Update([][]byte{key0, key1}, [][]byte{DefaultLeaf, DefaultLeaf})
key0 := make([]byte, 32)
key1 := make([]byte, 32)
if _, err := smt.Update([][]byte{key0, key1}, [][]byte{DefaultLeaf, DefaultLeaf}); err != nil {
t.Fatal(err)
}
if !bytes.Equal(root, smt.Root) {
t.Fatal("deleting a default key shouldnt' modify the tree")
}
@@ -366,7 +372,7 @@ func TestTrieDelete(t *testing.T) {
func TestTrieUpdateAndDelete(t *testing.T) {
smt := NewTrie(nil, Hasher, nil)
smt.CacheHeightLimit = 0
key0 := make([]byte, 32, 32)
key0 := make([]byte, 32)
values := getFreshData(1, 32)
root, _ := smt.Update([][]byte{key0}, values)
cacheNb := len(smt.db.liveCache)
@@ -377,7 +383,7 @@ func TestTrieUpdateAndDelete(t *testing.T) {
t.Fatal("leaf shortcut didn't move up to root")
}
key1 := make([]byte, 32, 32)
key1 := make([]byte, 32)
// set the last bit
bitSet(key1, 255)
keys := [][]byte{key0, key1}
@@ -403,7 +409,9 @@ func TestTrieMerkleProof(t *testing.T) {
// Add data to empty trie
keys := getFreshData(10, 32)
values := getFreshData(10, 32)
smt.Update(keys, values)
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
for i, key := range keys {
ap, _, k, v, _ := smt.MerkleProof(key)
@@ -429,7 +437,9 @@ func TestTrieMerkleProofCompressed(t *testing.T) {
// Add data to empty trie
keys := getFreshData(10, 32)
values := getFreshData(10, 32)
smt.Update(keys, values)
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
for i, key := range keys {
bitmap, ap, length, _, k, v, _ := smt.MerkleProofCompressed(key)
@@ -460,8 +470,12 @@ func TestTrieCommit(t *testing.T) {
smt := NewTrie(nil, Hasher, st)
keys := getFreshData(10, 32)
values := getFreshData(10, 32)
smt.Update(keys, values)
smt.Commit()
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
// liveCache is deleted so the key is fetched in badger db
smt.db.liveCache = make(map[Hash][][]byte)
for i, key := range keys {
@@ -484,7 +498,9 @@ func TestTrieStageUpdates(t *testing.T) {
smt := NewTrie(nil, Hasher, st)
keys := getFreshData(10, 32)
values := getFreshData(10, 32)
smt.Update(keys, values)
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
txn := st.NewTx()
smt.StageUpdates(txn.(DbTx))
txn.Commit()
@@ -511,52 +527,75 @@ func TestTrieRevert(t *testing.T) {
// Edge case : Test that revert doesnt delete shortcut nodes
// when moved to a different position in tree
key0 := make([]byte, 32, 32)
key1 := make([]byte, 32, 32)
key0 := make([]byte, 32)
key1 := make([]byte, 32)
// setting the bit at 251 creates 2 shortcut batches at height 252
bitSet(key1, 251)
values := getFreshData(2, 32)
keys := [][]byte{key0, key1}
root, _ := smt.Update([][]byte{key0}, [][]byte{values[0]})
smt.Commit()
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
root2, _ := smt.Update([][]byte{key1}, [][]byte{values[1]})
smt.Commit()
smt.Revert(root)
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
if err := smt.Revert(root); err != nil {
t.Fatal(err)
}
if len(smt.db.Store.Get(root)) == 0 {
t.Fatal("shortcut node shouldnt be deleted by revert")
}
if len(smt.db.Store.Get(root2)) != 0 {
t.Fatal("reverted root should have been deleted")
}
key1 = make([]byte, 32, 32)
key1 = make([]byte, 32)
// setting the bit at 255 stores the keys as the tip
bitSet(key1, 255)
smt.Update([][]byte{key1}, [][]byte{values[1]})
smt.Commit()
smt.Revert(root)
if _, err := smt.Update([][]byte{key1}, [][]byte{values[1]}); err != nil {
t.Fatal(err)
}
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
if err := smt.Revert(root); err != nil {
t.Fatal(err)
}
if len(smt.db.Store.Get(root)) == 0 {
t.Fatal("shortcut node shouldnt be deleted by revert")
}
// Test all nodes are reverted in the usual case
// Add data to empty trie
keys = getFreshData(10, 32)
keys := getFreshData(10, 32)
values = getFreshData(10, 32)
root, _ = smt.Update(keys, values)
smt.Commit()
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
// Update the values
newValues := getFreshData(10, 32)
smt.Update(keys, newValues)
if _, err := smt.Update(keys, newValues); err != nil {
t.Fatal(err)
}
updatedNodes1 := smt.db.updatedNodes
smt.Commit()
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
newKeys := getFreshData(10, 32)
newValues = getFreshData(10, 32)
smt.Update(newKeys, newValues)
if _, err := smt.Update(newKeys, newValues); err != nil {
t.Fatal(err)
}
updatedNodes2 := smt.db.updatedNodes
smt.Commit()
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
smt.Revert(root)
if err := smt.Revert(root); err != nil {
t.Fatal(err)
}
if !bytes.Equal(smt.Root, root) {
t.Fatal("revert failed")
@@ -575,12 +614,12 @@ func TestTrieRevert(t *testing.T) {
t.Fatal("live cache not reset after revert")
}
// Check all reverted nodes have been deleted
for node, _ := range updatedNodes2 {
for node := range updatedNodes2 {
if len(smt.db.Store.Get(node[:])) != 0 {
t.Fatal("nodes not deleted from database", node)
}
}
for node, _ := range updatedNodes1 {
for node := range updatedNodes1 {
if len(smt.db.Store.Get(node[:])) != 0 {
t.Fatal("nodes not deleted from database", node)
}
@@ -600,7 +639,9 @@ func TestTrieRaisesError(t *testing.T) {
// Add data to empty trie
keys := getFreshData(10, 32)
values := getFreshData(10, 32)
smt.Update(keys, values)
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
smt.db.liveCache = make(map[Hash][][]byte)
smt.db.updatedNodes = make(map[Hash][][]byte)
@@ -629,7 +670,7 @@ func TestTrieRaisesError(t *testing.T) {
}
smt.db.liveCache = make(map[Hash][][]byte)
smt.atomicUpdate = false
_, _, _, _, _, err = smt.loadChildren(make([]byte, 32, 32), smt.TrieHeight, 0, nil)
_, _, _, _, _, err = smt.loadChildren(make([]byte, 32), smt.TrieHeight, 0, nil)
if err == nil {
t.Fatal("Error not created if database not connected")
}
@@ -649,11 +690,13 @@ func TestTrieLoadCache(t *testing.T) {
smt := NewTrie(nil, Hasher, st)
// Test size of cache
smt.CacheHeightLimit = 0
key0 := make([]byte, 32, 32)
key1 := make([]byte, 32, 32)
key0 := make([]byte, 32)
key1 := make([]byte, 32)
bitSet(key1, 255)
values := getFreshData(2, 32)
smt.Update([][]byte{key0, key1}, values)
if _, err := smt.Update([][]byte{key0, key1}, values); err != nil {
t.Fatal(err)
}
if len(smt.db.liveCache) != 66 {
// the nodes are at the tip, so 64 + 2 = 66
t.Fatal("cache size incorrect")
@@ -662,8 +705,12 @@ func TestTrieLoadCache(t *testing.T) {
// Add data to empty trie
keys := getFreshData(10, 32)
values = getFreshData(10, 32)
smt.Update(keys, values)
smt.Commit()
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
// Simulate node restart by deleting and loading cache
cacheSize := len(smt.db.liveCache)
@@ -685,12 +732,14 @@ func TestHeight0LeafShortcut(t *testing.T) {
keySize := 32
smt := NewTrie(nil, Hasher, nil)
// Add 2 sibling keys that will be stored at height 0
key0 := make([]byte, keySize, keySize)
key1 := make([]byte, keySize, keySize)
key0 := make([]byte, keySize)
key1 := make([]byte, keySize)
bitSet(key1, keySize*8-1)
keys := [][]byte{key0, key1}
values := getFreshData(2, 32)
smt.Update(keys, values)
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
updatedNb := len(smt.db.updatedNodes)
// Check all keys have been stored
@@ -752,13 +801,19 @@ func TestStash(t *testing.T) {
values := getFreshData(20, 32)
root, _ := smt.Update(keys, values)
cacheSize := len(smt.db.liveCache)
smt.Commit()
if err := smt.Commit(); err != nil {
t.Fatal(err)
}
if len(smt.pastTries) != 1 {
t.Fatal("Past tries not updated after commit")
}
values = getFreshData(20, 32)
smt.Update(keys, values)
smt.Stash(true)
if _, err := smt.Update(keys, values); err != nil {
t.Fatal(err)
}
if err := smt.Stash(true); err != nil {
t.Fatal(err)
}
if len(smt.pastTries) != 1 {
t.Fatal("Past tries not updated after commit")
}
@@ -773,13 +828,19 @@ func TestStash(t *testing.T) {
}
keys = getFreshData(20, 32)
values = getFreshData(20, 32)
smt.AtomicUpdate(keys, values)
if _, err := smt.AtomicUpdate(keys, values); err != nil {
t.Fatal(err)
}
values = getFreshData(20, 32)
smt.AtomicUpdate(keys, values)
if _, err := smt.AtomicUpdate(keys, values); err != nil {
t.Fatal(err)
}
if len(smt.pastTries) != 3 {
t.Fatal("Past tries not updated after commit")
}
smt.Stash(true)
if err := smt.Stash(true); err != nil {
t.Fatal(err)
}
if !bytes.Equal(smt.Root, root) {
t.Fatal("Trie not rolled back")
}
@@ -800,15 +861,21 @@ func benchmark10MAccounts10Ktps(smt *Trie, b *testing.B) {
//b.ReportAllocs()
keys := getFreshData(100, 32)
values := getFreshData(100, 32)
smt.Update(keys, values)
if _, err := smt.Update(keys, values); err != nil {
b.Fatal(err)
}
fmt.Println("\nLoading b.N x 1000 accounts")
for i := 0; i < b.N; i++ {
newkeys := getFreshData(1000, 32)
newvalues := getFreshData(1000, 32)
start := time.Now()
smt.Update(newkeys, newvalues)
if _, err := smt.Update(newkeys, newvalues); err != nil {
b.Fatal(err)
}
end := time.Now()
smt.Commit()
if err := smt.Commit(); err != nil {
b.Fatal(err)
}
end2 := time.Now()
for j, key := range newkeys {
val, _ := smt.Get(key)

5
smt/trie_tools.go
View File

@@ -201,10 +201,7 @@ func (s *Trie) TrieRootExists(root []byte) bool {
s.db.lock.RLock()
dbval := s.db.Store.Get(root)
s.db.lock.RUnlock()
if len(dbval) != 0 {
return true
}
return false
return len(dbval) != 0
}
// Commit stores the updated nodes to disk.