arnaucube
arnaucube/hermez-nodePublic Self-hosted repository
Merge pull request #107 from hermeznetwork/feature/statedb-intermediatestate

StateDB intermediate state reset when opening DB
Eduard S eduard@iden3.io
Mon, 07 Sep 2020 13:46:20 +0200
commit

7701af194fd8b9da116795d6302af2136b781a76

parent

9db9508b445883fe90d96359159453d1bc8503f3

5 files changed, 133 insertions(+), 17 deletions(-)

jump to
M db/statedb/statedb.go → db/statedb/statedb.go

@@ -75,6 +75,12 @@ if err != nil {

return nil, err } + // make reset (get checkpoint) at currentBatch + err = sdb.Reset(sdb.currentBatch) + if err != nil { + return nil, err + } + return sdb, nil }

@@ -101,7 +107,10 @@ tx, err := s.db.NewTx()

if err != nil { return err } - tx.Put(KeyCurrentBatch, s.currentBatch.Bytes()) + err = tx.Put(KeyCurrentBatch, s.currentBatch.Bytes()) + if err != nil { + return err + } if err := tx.Commit(); err != nil { return err }

@@ -153,11 +162,6 @@ // does not delete the checkpoints between old current and the new current,

// those checkpoints will remain in the storage, and eventually will be // deleted when MakeCheckpoint overwrites them. func (s *StateDB) Reset(batchNum common.BatchNum) error { - if batchNum == 0 { - s.idx = 0 - return nil - } - checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum)) currentPath := s.path + PathCurrent

@@ -166,6 +170,18 @@ err := os.RemoveAll(currentPath)

if err != nil { return err } + if batchNum == 0 { + // if batchNum == 0, open the new fresh 'current' + sto, err := pebble.NewPebbleStorage(currentPath, false) + if err != nil { + return err + } + s.db = sto + s.idx = 0 + s.currentBatch = batchNum + return nil + } + // copy 'BatchNumX' to 'current' cmd := exec.Command("cp", "-r", checkpointPath, currentPath) //nolint:gosec err = cmd.Run()

@@ -191,12 +207,14 @@ if err != nil {

return err } - // open the MT for the current s.db - mt, err := merkletree.NewMerkleTree(s.db, s.mt.MaxLevels()) - if err != nil { - return err + if s.mt != nil { + // open the MT for the current s.db + mt, err := merkletree.NewMerkleTree(s.db, s.mt.MaxLevels()) + if err != nil { + return err + } + s.mt = mt } - s.mt = mt return nil }

@@ -255,8 +273,14 @@ if err != db.ErrNotFound {

return nil, ErrAccountAlreadyExists } - tx.Put(v.Bytes(), accountBytes[:]) - tx.Put(idx.Bytes(), v.Bytes()) + err = tx.Put(v.Bytes(), accountBytes[:]) + if err != nil { + return nil, err + } + err = tx.Put(idx.Bytes(), v.Bytes()) + if err != nil { + return nil, err + } if err := tx.Commit(); err != nil { return nil, err

@@ -295,8 +319,14 @@ tx, err := sto.NewTx()

if err != nil { return nil, err } - tx.Put(v.Bytes(), accountBytes[:]) - tx.Put(idx.Bytes(), v.Bytes()) + err = tx.Put(v.Bytes(), accountBytes[:]) + if err != nil { + return nil, err + } + err = tx.Put(idx.Bytes(), v.Bytes()) + if err != nil { + return nil, err + } if err := tx.Commit(); err != nil { return nil, err
M db/statedb/statedb_test.go → db/statedb/statedb_test.go

@@ -34,6 +34,87 @@ EthAddr: address,

} } +func TestNewStateDBIntermediateState(t *testing.T) { + dir, err := ioutil.TempDir("", "tmpdb") + require.Nil(t, err) + + sdb, err := NewStateDB(dir, false, 0) + assert.Nil(t, err) + + // test values + k0 := []byte("testkey0") + k1 := []byte("testkey1") + v0 := []byte("testvalue0") + v1 := []byte("testvalue1") + + // store some data + tx, err := sdb.db.NewTx() + assert.Nil(t, err) + err = tx.Put(k0, v0) + assert.Nil(t, err) + err = tx.Commit() + assert.Nil(t, err) + v, err := sdb.db.Get(k0) + assert.Nil(t, err) + assert.Equal(t, v0, v) + + // call NewStateDB which should get the db at the last checkpoint state + // executing a Reset (discarding the last 'testkey0'&'testvalue0' data) + sdb, err = NewStateDB(dir, false, 0) + assert.Nil(t, err) + v, err = sdb.db.Get(k0) + assert.NotNil(t, err) + assert.Equal(t, db.ErrNotFound, err) + assert.Nil(t, v) + + // store the same data from the beginning that has ben lost since last NewStateDB + tx, err = sdb.db.NewTx() + assert.Nil(t, err) + err = tx.Put(k0, v0) + assert.Nil(t, err) + err = tx.Commit() + assert.Nil(t, err) + v, err = sdb.db.Get(k0) + assert.Nil(t, err) + assert.Equal(t, v0, v) + + // make checkpoints with the current state + bn, err := sdb.GetCurrentBatch() + assert.Nil(t, err) + assert.Equal(t, common.BatchNum(0), bn) + err = sdb.MakeCheckpoint() + assert.Nil(t, err) + bn, err = sdb.GetCurrentBatch() + assert.Nil(t, err) + assert.Equal(t, common.BatchNum(1), bn) + + // write more data + tx, err = sdb.db.NewTx() + assert.Nil(t, err) + err = tx.Put(k1, v1) + assert.Nil(t, err) + err = tx.Commit() + assert.Nil(t, err) + + v, err = sdb.db.Get(k1) + assert.Nil(t, err) + assert.Equal(t, v1, v) + + // call NewStateDB which should get the db at the last checkpoint state + // executing a Reset (discarding the last 'testkey1'&'testvalue1' data) + sdb, err = NewStateDB(dir, false, 0) + assert.Nil(t, err) + + v, err = sdb.db.Get(k0) + assert.Nil(t, err) + assert.Equal(t, v0, v) + + v, err = sdb.db.Get(k1) + assert.NotNil(t, err) + assert.Equal(t, db.ErrNotFound, err) + assert.Nil(t, v) +} + func TestStateDBWithoutMT(t *testing.T) { dir, err := ioutil.TempDir("", "tmpdb") require.Nil(t, err)
M db/statedb/txprocessors.go → db/statedb/txprocessors.go

@@ -322,7 +322,10 @@ tx, err := s.DB().NewTx()

if err != nil { return err } - tx.Put(keyidx, idx.Bytes()) + err = tx.Put(keyidx, idx.Bytes()) + if err != nil { + return err + } if err := tx.Commit(); err != nil { return err }
M go.mod → go.mod

@@ -9,7 +9,7 @@ github.com/go-sql-driver/mysql v1.5.0 // indirect

github.com/gobuffalo/packr/v2 v2.8.0 github.com/iden3/go-iden3-core v0.0.8 github.com/iden3/go-iden3-crypto v0.0.6-0.20200823174058-e04ca5764a15 - github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41 + github.com/iden3/go-merkletree v0.0.0-20200902123354-eeb949f8c334 github.com/jinzhu/gorm v1.9.15 github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.8.0
M go.sum → go.sum

@@ -309,6 +309,8 @@ github.com/iden3/go-merkletree v0.0.0-20200819092443-dc656fdd32fc h1:VnRP7JCp5TJHnTC+r8NrXGkRBvp62tRGqeA3FcdEq+Q=

github.com/iden3/go-merkletree v0.0.0-20200819092443-dc656fdd32fc/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU= github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41 h1:mCOMMQ/YmL9ST9kk7ifT961chESkB2GFFEp8osF0Jw8= github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU= +github.com/iden3/go-merkletree v0.0.0-20200902123354-eeb949f8c334 h1:FQngDJKiwM6i4kHlVFvSpJa9sO+QvZ7C+GqoPWe+5BI= +github.com/iden3/go-merkletree v0.0.0-20200902123354-eeb949f8c334/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU= github.com/iden3/go-wasm3 v0.0.1/go.mod h1:j+TcAB94Dfrjlu5kJt83h2OqAU+oyNUTwNZnQyII1sI= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=