Browse Source

Merge pull request #107 from hermeznetwork/feature/statedb-intermediatestate

StateDB intermediate state reset when opening DB
feature/sql-semaphore1
Eduard S 4 years ago
committed by GitHub
parent
commit
7701af194f
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 133 additions and 17 deletions
  1. +45
    -15
      db/statedb/statedb.go
  2. +81
    -0
      db/statedb/statedb_test.go
  3. +4
    -1
      db/statedb/txprocessors.go
  4. +1
    -1
      go.mod
  5. +2
    -0
      go.sum

+ 45
- 15
db/statedb/statedb.go

@ -75,6 +75,12 @@ func NewStateDB(path string, withMT bool, nLevels int) (*StateDB, error) {
return nil, err return nil, err
} }
// make reset (get checkpoint) at currentBatch
err = sdb.Reset(sdb.currentBatch)
if err != nil {
return nil, err
}
return sdb, nil return sdb, nil
} }
@ -101,7 +107,10 @@ func (s *StateDB) setCurrentBatch() error {
if err != nil { if err != nil {
return err return err
} }
tx.Put(KeyCurrentBatch, s.currentBatch.Bytes())
err = tx.Put(KeyCurrentBatch, s.currentBatch.Bytes())
if err != nil {
return err
}
if err := tx.Commit(); err != nil { if err := tx.Commit(); err != nil {
return err return err
} }
@ -153,11 +162,6 @@ func (s *StateDB) DeleteCheckpoint(batchNum common.BatchNum) error {
// those checkpoints will remain in the storage, and eventually will be // those checkpoints will remain in the storage, and eventually will be
// deleted when MakeCheckpoint overwrites them. // deleted when MakeCheckpoint overwrites them.
func (s *StateDB) Reset(batchNum common.BatchNum) error { func (s *StateDB) Reset(batchNum common.BatchNum) error {
if batchNum == 0 {
s.idx = 0
return nil
}
checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum)) checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum))
currentPath := s.path + PathCurrent currentPath := s.path + PathCurrent
@ -166,6 +170,18 @@ func (s *StateDB) Reset(batchNum common.BatchNum) error {
if err != nil { if err != nil {
return err return err
} }
if batchNum == 0 {
// if batchNum == 0, open the new fresh 'current'
sto, err := pebble.NewPebbleStorage(currentPath, false)
if err != nil {
return err
}
s.db = sto
s.idx = 0
s.currentBatch = batchNum
return nil
}
// copy 'BatchNumX' to 'current' // copy 'BatchNumX' to 'current'
cmd := exec.Command("cp", "-r", checkpointPath, currentPath) //nolint:gosec cmd := exec.Command("cp", "-r", checkpointPath, currentPath) //nolint:gosec
err = cmd.Run() err = cmd.Run()
@ -191,12 +207,14 @@ func (s *StateDB) Reset(batchNum common.BatchNum) error {
return err return err
} }
// open the MT for the current s.db
mt, err := merkletree.NewMerkleTree(s.db, s.mt.MaxLevels())
if err != nil {
return err
if s.mt != nil {
// open the MT for the current s.db
mt, err := merkletree.NewMerkleTree(s.db, s.mt.MaxLevels())
if err != nil {
return err
}
s.mt = mt
} }
s.mt = mt
return nil return nil
} }
@ -255,8 +273,14 @@ func createAccountInTreeDB(sto db.Storage, mt *merkletree.MerkleTree, idx common
return nil, ErrAccountAlreadyExists return nil, ErrAccountAlreadyExists
} }
tx.Put(v.Bytes(), accountBytes[:])
tx.Put(idx.Bytes(), v.Bytes())
err = tx.Put(v.Bytes(), accountBytes[:])
if err != nil {
return nil, err
}
err = tx.Put(idx.Bytes(), v.Bytes())
if err != nil {
return nil, err
}
if err := tx.Commit(); err != nil { if err := tx.Commit(); err != nil {
return nil, err return nil, err
@ -295,8 +319,14 @@ func updateAccountInTreeDB(sto db.Storage, mt *merkletree.MerkleTree, idx common
if err != nil { if err != nil {
return nil, err return nil, err
} }
tx.Put(v.Bytes(), accountBytes[:])
tx.Put(idx.Bytes(), v.Bytes())
err = tx.Put(v.Bytes(), accountBytes[:])
if err != nil {
return nil, err
}
err = tx.Put(idx.Bytes(), v.Bytes())
if err != nil {
return nil, err
}
if err := tx.Commit(); err != nil { if err := tx.Commit(); err != nil {
return nil, err return nil, err

+ 81
- 0
db/statedb/statedb_test.go

@ -34,6 +34,87 @@ func newAccount(t *testing.T, i int) *common.Account {
} }
} }
func TestNewStateDBIntermediateState(t *testing.T) {
dir, err := ioutil.TempDir("", "tmpdb")
require.Nil(t, err)
sdb, err := NewStateDB(dir, false, 0)
assert.Nil(t, err)
// test values
k0 := []byte("testkey0")
k1 := []byte("testkey1")
v0 := []byte("testvalue0")
v1 := []byte("testvalue1")
// store some data
tx, err := sdb.db.NewTx()
assert.Nil(t, err)
err = tx.Put(k0, v0)
assert.Nil(t, err)
err = tx.Commit()
assert.Nil(t, err)
v, err := sdb.db.Get(k0)
assert.Nil(t, err)
assert.Equal(t, v0, v)
// call NewStateDB which should get the db at the last checkpoint state
// executing a Reset (discarding the last 'testkey0'&'testvalue0' data)
sdb, err = NewStateDB(dir, false, 0)
assert.Nil(t, err)
v, err = sdb.db.Get(k0)
assert.NotNil(t, err)
assert.Equal(t, db.ErrNotFound, err)
assert.Nil(t, v)
// store the same data from the beginning that has ben lost since last NewStateDB
tx, err = sdb.db.NewTx()
assert.Nil(t, err)
err = tx.Put(k0, v0)
assert.Nil(t, err)
err = tx.Commit()
assert.Nil(t, err)
v, err = sdb.db.Get(k0)
assert.Nil(t, err)
assert.Equal(t, v0, v)
// make checkpoints with the current state
bn, err := sdb.GetCurrentBatch()
assert.Nil(t, err)
assert.Equal(t, common.BatchNum(0), bn)
err = sdb.MakeCheckpoint()
assert.Nil(t, err)
bn, err = sdb.GetCurrentBatch()
assert.Nil(t, err)
assert.Equal(t, common.BatchNum(1), bn)
// write more data
tx, err = sdb.db.NewTx()
assert.Nil(t, err)
err = tx.Put(k1, v1)
assert.Nil(t, err)
err = tx.Commit()
assert.Nil(t, err)
v, err = sdb.db.Get(k1)
assert.Nil(t, err)
assert.Equal(t, v1, v)
// call NewStateDB which should get the db at the last checkpoint state
// executing a Reset (discarding the last 'testkey1'&'testvalue1' data)
sdb, err = NewStateDB(dir, false, 0)
assert.Nil(t, err)
v, err = sdb.db.Get(k0)
assert.Nil(t, err)
assert.Equal(t, v0, v)
v, err = sdb.db.Get(k1)
assert.NotNil(t, err)
assert.Equal(t, db.ErrNotFound, err)
assert.Nil(t, v)
}
func TestStateDBWithoutMT(t *testing.T) { func TestStateDBWithoutMT(t *testing.T) {
dir, err := ioutil.TempDir("", "tmpdb") dir, err := ioutil.TempDir("", "tmpdb")
require.Nil(t, err) require.Nil(t, err)

+ 4
- 1
db/statedb/txprocessors.go

@ -322,7 +322,10 @@ func (s *StateDB) setIdx(idx common.Idx) error {
if err != nil { if err != nil {
return err return err
} }
tx.Put(keyidx, idx.Bytes())
err = tx.Put(keyidx, idx.Bytes())
if err != nil {
return err
}
if err := tx.Commit(); err != nil { if err := tx.Commit(); err != nil {
return err return err
} }

+ 1
- 1
go.mod

@ -9,7 +9,7 @@ require (
github.com/gobuffalo/packr/v2 v2.8.0 github.com/gobuffalo/packr/v2 v2.8.0
github.com/iden3/go-iden3-core v0.0.8 github.com/iden3/go-iden3-core v0.0.8
github.com/iden3/go-iden3-crypto v0.0.6-0.20200823174058-e04ca5764a15 github.com/iden3/go-iden3-crypto v0.0.6-0.20200823174058-e04ca5764a15
github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41
github.com/iden3/go-merkletree v0.0.0-20200902123354-eeb949f8c334
github.com/jinzhu/gorm v1.9.15 github.com/jinzhu/gorm v1.9.15
github.com/jmoiron/sqlx v1.2.0 github.com/jmoiron/sqlx v1.2.0
github.com/lib/pq v1.8.0 github.com/lib/pq v1.8.0

+ 2
- 0
go.sum

@ -309,6 +309,8 @@ github.com/iden3/go-merkletree v0.0.0-20200819092443-dc656fdd32fc h1:VnRP7JCp5TJ
github.com/iden3/go-merkletree v0.0.0-20200819092443-dc656fdd32fc/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU= github.com/iden3/go-merkletree v0.0.0-20200819092443-dc656fdd32fc/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU=
github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41 h1:mCOMMQ/YmL9ST9kk7ifT961chESkB2GFFEp8osF0Jw8= github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41 h1:mCOMMQ/YmL9ST9kk7ifT961chESkB2GFFEp8osF0Jw8=
github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU= github.com/iden3/go-merkletree v0.0.0-20200825093552-a4b68208bb41/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU=
github.com/iden3/go-merkletree v0.0.0-20200902123354-eeb949f8c334 h1:FQngDJKiwM6i4kHlVFvSpJa9sO+QvZ7C+GqoPWe+5BI=
github.com/iden3/go-merkletree v0.0.0-20200902123354-eeb949f8c334/go.mod h1:MRe6i0mi2oDVUzgBIHsNRE6XAg8EBuqIQZMsd+do+dU=
github.com/iden3/go-wasm3 v0.0.1/go.mod h1:j+TcAB94Dfrjlu5kJt83h2OqAU+oyNUTwNZnQyII1sI= github.com/iden3/go-wasm3 v0.0.1/go.mod h1:j+TcAB94Dfrjlu5kJt83h2OqAU+oyNUTwNZnQyII1sI=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=

Loading…
Cancel
Save