hermez-node/db/statedb/statedb.go
db/statedb/statedb.goRaw
1package statedb
2
3import (
4 "errors"
5 "fmt"
6 "math/big"
7 "os"
8 "strconv"
9
10 "github.com/hermeznetwork/hermez-node/common"
11 "github.com/hermeznetwork/hermez-node/log"
12 "github.com/iden3/go-merkletree"
13 "github.com/iden3/go-merkletree/db"
14 "github.com/iden3/go-merkletree/db/pebble"
15)
16
17// TODO(Edu): Document here how StateDB is kept consistent
18
19var (
20 // ErrStateDBWithoutMT is used when a method that requires a MerkleTree
21 // is called in a StateDB that does not have a MerkleTree defined
22 ErrStateDBWithoutMT = errors.New("Can not call method to use MerkleTree in a StateDB without MerkleTree")
23
24 // ErrAccountAlreadyExists is used when CreateAccount is called and the
25 // Account already exists
26 ErrAccountAlreadyExists = errors.New("Can not CreateAccount because Account already exists")
27
28 // ErrToIdxNotFound is used when trying to get the ToIdx from ToEthAddr
29 // or ToEthAddr&ToBJJ
30 ErrToIdxNotFound = errors.New("ToIdx can not be found")
31 // ErrGetIdxNoCase is used when trying to get the Idx from EthAddr &
32 // BJJ with not compatible combination
33 ErrGetIdxNoCase = errors.New("Can not get Idx due unexpected combination of ethereum Address & BabyJubJub PublicKey")
34
35 // KeyCurrentBatch is used as key in the db to store the current BatchNum
36 KeyCurrentBatch = []byte("k:currentbatch")
37
38 // PrefixKeyIdx is the key prefix for idx in the db
39 PrefixKeyIdx = []byte("i:")
40 // PrefixKeyAccHash is the key prefix for account hash in the db
41 PrefixKeyAccHash = []byte("h:")
42 // PrefixKeyMT is the key prefix for merkle tree in the db
43 PrefixKeyMT = []byte("m:")
44 // PrefixKeyAddr is the key prefix for address in the db
45 PrefixKeyAddr = []byte("a:")
46 // PrefixKeyAddrBJJ is the key prefix for address-babyjubjub in the db
47 PrefixKeyAddrBJJ = []byte("ab:")
48)
49
50const (
51 // PathStateDB defines the subpath of the StateDB
52 PathStateDB = "/statedb"
53 // PathBatchNum defines the subpath of the Batch Checkpoint in the
54 // subpath of the StateDB
55 PathBatchNum = "/BatchNum"
56 // PathCurrent defines the subpath of the current Batch in the subpath
57 // of the StateDB
58 PathCurrent = "/current"
59 // TypeSynchronizer defines a StateDB used by the Synchronizer, that
60 // generates the ExitTree when processing the txs
61 TypeSynchronizer = "synchronizer"
62 // TypeTxSelector defines a StateDB used by the TxSelector, without
63 // computing ExitTree neither the ZKInputs
64 TypeTxSelector = "txselector"
65 // TypeBatchBuilder defines a StateDB used by the BatchBuilder, that
66 // generates the ExitTree and the ZKInput when processing the txs
67 TypeBatchBuilder = "batchbuilder"
68)
69
70// TypeStateDB determines the type of StateDB
71type TypeStateDB string
72
73// StateDB represents the StateDB object
74type StateDB struct {
75 path string
76 currentBatch common.BatchNum
77 db *pebble.PebbleStorage
78 mt *merkletree.MerkleTree
79 typ TypeStateDB
80 // idx holds the current Idx that the BatchBuilder is using
81 idx common.Idx
82 zki *common.ZKInputs
83 i int // i is the current transaction index in the ZKInputs generation (zki)
84}
85
86// NewStateDB creates a new StateDB, allowing to use an in-memory or in-disk
87// storage
88func NewStateDB(path string, typ TypeStateDB, nLevels int) (*StateDB, error) {
89 var sto *pebble.PebbleStorage
90 var err error
91 sto, err = pebble.NewPebbleStorage(path+PathStateDB+PathCurrent, false)
92 if err != nil {
93 return nil, err
94 }
95
96 var mt *merkletree.MerkleTree = nil
97 if typ == TypeSynchronizer || typ == TypeBatchBuilder {
98 mt, err = merkletree.NewMerkleTree(sto.WithPrefix(PrefixKeyMT), nLevels)
99 if err != nil {
100 return nil, err
101 }
102 }
103 if typ == TypeTxSelector && nLevels != 0 {
104 return nil, fmt.Errorf("invalid StateDB parameters: StateDB type==TypeStateDB can not have nLevels!=0")
105 }
106
107 sdb := &StateDB{
108 path: path + PathStateDB,
109 db: sto,
110 mt: mt,
111 typ: typ,
112 }
113
114 // load currentBatch
115 sdb.currentBatch, err = sdb.GetCurrentBatch()
116 if err != nil {
117 return nil, err
118 }
119
120 // make reset (get checkpoint) at currentBatch
121 err = sdb.reset(sdb.currentBatch, false)
122 if err != nil {
123 return nil, err
124 }
125
126 return sdb, nil
127}
128
129// DB returns the *pebble.PebbleStorage from the StateDB
130func (s *StateDB) DB() *pebble.PebbleStorage {
131 return s.db
132}
133
134// GetCurrentBatch returns the current BatchNum stored in the StateDB
135func (s *StateDB) GetCurrentBatch() (common.BatchNum, error) {
136 cbBytes, err := s.db.Get(KeyCurrentBatch)
137 if err == db.ErrNotFound {
138 return 0, nil
139 }
140 if err != nil {
141 return 0, err
142 }
143 return common.BatchNumFromBytes(cbBytes)
144}
145
146// setCurrentBatch stores the current BatchNum in the StateDB
147func (s *StateDB) setCurrentBatch() error {
148 tx, err := s.db.NewTx()
149 if err != nil {
150 return err
151 }
152 err = tx.Put(KeyCurrentBatch, s.currentBatch.Bytes())
153 if err != nil {
154 return err
155 }
156 if err := tx.Commit(); err != nil {
157 return err
158 }
159 return nil
160}
161
162// MakeCheckpoint does a checkpoint at the given batchNum in the defined path. Internally this advances & stores the current BatchNum, and then stores a Checkpoint of the current state of the StateDB.
163func (s *StateDB) MakeCheckpoint() error {
164 // advance currentBatch
165 s.currentBatch++
166
167 checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(s.currentBatch))
168
169 err := s.setCurrentBatch()
170 if err != nil {
171 return err
172 }
173
174 // if checkpoint BatchNum already exist in disk, delete it
175 if _, err := os.Stat(checkpointPath); !os.IsNotExist(err) {
176 err := os.RemoveAll(checkpointPath)
177 if err != nil {
178 return err
179 }
180 } else if err != nil && !os.IsNotExist(err) {
181 return err
182 }
183
184 // execute Checkpoint
185 err = s.db.Pebble().Checkpoint(checkpointPath)
186 if err != nil {
187 return err
188 }
189
190 return nil
191}
192
193// DeleteCheckpoint removes if exist the checkpoint of the given batchNum
194func (s *StateDB) DeleteCheckpoint(batchNum common.BatchNum) error {
195 checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum))
196
197 if _, err := os.Stat(checkpointPath); os.IsNotExist(err) {
198 return fmt.Errorf("Checkpoint with batchNum %d does not exist in DB", batchNum)
199 }
200
201 return os.RemoveAll(checkpointPath)
202}
203
204func pebbleMakeCheckpoint(source, dest string) error {
205 // Remove dest folder (if it exists) before doing the checkpoint
206 if _, err := os.Stat(dest); !os.IsNotExist(err) {
207 err := os.RemoveAll(dest)
208 if err != nil {
209 return err
210 }
211 } else if err != nil && !os.IsNotExist(err) {
212 return err
213 }
214
215 sto, err := pebble.NewPebbleStorage(source, false)
216 if err != nil {
217 return err
218 }
219 defer func() {
220 errClose := sto.Pebble().Close()
221 if errClose != nil {
222 log.Errorw("Pebble.Close", "err", errClose)
223 }
224 }()
225
226 // execute Checkpoint
227 err = sto.Pebble().Checkpoint(dest)
228 if err != nil {
229 return err
230 }
231
232 return nil
233}
234
235// Reset resets the StateDB to the checkpoint at the given batchNum. Reset
236// does not delete the checkpoints between old current and the new current,
237// those checkpoints will remain in the storage, and eventually will be
238// deleted when MakeCheckpoint overwrites them.
239func (s *StateDB) Reset(batchNum common.BatchNum) error {
240 return s.reset(batchNum, true)
241}
242
243// reset resets the StateDB to the checkpoint at the given batchNum. Reset
244// does not delete the checkpoints between old current and the new current,
245// those checkpoints will remain in the storage, and eventually will be
246// deleted when MakeCheckpoint overwrites them. `closeCurrent` will close the
247// currently opened db before doing the reset.
248func (s *StateDB) reset(batchNum common.BatchNum, closeCurrent bool) error {
249 checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum))
250 currentPath := s.path + PathCurrent
251
252 if closeCurrent {
253 if err := s.db.Pebble().Close(); err != nil {
254 return err
255 }
256 }
257 // remove 'current'
258 err := os.RemoveAll(currentPath)
259 if err != nil {
260 return err
261 }
262 if batchNum == 0 {
263 // if batchNum == 0, open the new fresh 'current'
264 sto, err := pebble.NewPebbleStorage(currentPath, false)
265 if err != nil {
266 return err
267 }
268 s.db = sto
269 s.idx = 255
270 s.currentBatch = batchNum
271 return nil
272 }
273
274 // copy 'BatchNumX' to 'current'
275 err = pebbleMakeCheckpoint(checkpointPath, currentPath)
276 if err != nil {
277 return err
278 }
279
280 // open the new 'current'
281 sto, err := pebble.NewPebbleStorage(currentPath, false)
282 if err != nil {
283 return err
284 }
285 s.db = sto
286
287 // get currentBatch num
288 s.currentBatch, err = s.GetCurrentBatch()
289 if err != nil {
290 return err
291 }
292 // idx is obtained from the statedb reset
293 s.idx, err = s.getIdx()
294 if err != nil {
295 return err
296 }
297
298 if s.mt != nil {
299 // open the MT for the current s.db
300 mt, err := merkletree.NewMerkleTree(s.db.WithPrefix(PrefixKeyMT), s.mt.MaxLevels())
301 if err != nil {
302 return err
303 }
304 s.mt = mt
305 }
306
307 return nil
308}
309
310// GetAccount returns the account for the given Idx
311func (s *StateDB) GetAccount(idx common.Idx) (*common.Account, error) {
312 return getAccountInTreeDB(s.db, idx)
313}
314
315// GetAccounts returns all the accounts in the db. Use for debugging pruposes
316// only.
317func (s *StateDB) GetAccounts() ([]common.Account, error) {
318 idxDB := s.db.WithPrefix(PrefixKeyIdx)
319 idxs := []common.Idx{}
320 // NOTE: Current implementation of Iterate in the pebble interface is
321 // not efficient, as it iterates over all keys. Improve it following
322 // this example: https://github.com/cockroachdb/pebble/pull/923/files
323 if err := idxDB.Iterate(func(k []byte, v []byte) (bool, error) {
324 idx, err := common.IdxFromBytes(k)
325 if err != nil {
326 return false, err
327 }
328 idxs = append(idxs, idx)
329 return true, nil
330 }); err != nil {
331 return nil, err
332 }
333 accs := []common.Account{}
334 for i := range idxs {
335 acc, err := s.GetAccount(idxs[i])
336 if err != nil {
337 return nil, err
338 }
339 accs = append(accs, *acc)
340 }
341 return accs, nil
342}
343
344// getAccountInTreeDB is abstracted from StateDB to be used from StateDB and
345// from ExitTree. GetAccount returns the account for the given Idx
346func getAccountInTreeDB(sto db.Storage, idx common.Idx) (*common.Account, error) {
347 idxBytes, err := idx.Bytes()
348 if err != nil {
349 return nil, err
350 }
351 vBytes, err := sto.Get(append(PrefixKeyIdx, idxBytes[:]...))
352 if err != nil {
353 return nil, err
354 }
355 accBytes, err := sto.Get(append(PrefixKeyAccHash, vBytes...))
356 if err != nil {
357 return nil, err
358 }
359 var b [32 * common.NLeafElems]byte
360 copy(b[:], accBytes)
361 account, err := common.AccountFromBytes(b)
362 if err != nil {
363 return nil, err
364 }
365 account.Idx = idx
366 return account, nil
367}
368
369// CreateAccount creates a new Account in the StateDB for the given Idx. If
370// StateDB.mt==nil, MerkleTree is not affected, otherwise updates the
371// MerkleTree, returning a CircomProcessorProof.
372func (s *StateDB) CreateAccount(idx common.Idx, account *common.Account) (*merkletree.CircomProcessorProof, error) {
373 cpp, err := createAccountInTreeDB(s.db, s.mt, idx, account)
374 if err != nil {
375 return cpp, err
376 }
377 // store idx by EthAddr & BJJ
378 err = s.setIdxByEthAddrBJJ(idx, account.EthAddr, account.PublicKey, account.TokenID)
379 return cpp, err
380}
381
382// createAccountInTreeDB is abstracted from StateDB to be used from StateDB and
383// from ExitTree. Creates a new Account in the StateDB for the given Idx. If
384// StateDB.mt==nil, MerkleTree is not affected, otherwise updates the
385// MerkleTree, returning a CircomProcessorProof.
386func createAccountInTreeDB(sto db.Storage, mt *merkletree.MerkleTree, idx common.Idx, account *common.Account) (*merkletree.CircomProcessorProof, error) {
387 // store at the DB the key: v, and value: leaf.Bytes()
388 v, err := account.HashValue()
389 if err != nil {
390 return nil, err
391 }
392 accountBytes, err := account.Bytes()
393 if err != nil {
394 return nil, err
395 }
396
397 // store the Leaf value
398 tx, err := sto.NewTx()
399 if err != nil {
400 return nil, err
401 }
402
403 idxBytes, err := idx.Bytes()
404 if err != nil {
405 return nil, err
406 }
407 _, err = tx.Get(append(PrefixKeyIdx, idxBytes[:]...))
408 if err != db.ErrNotFound {
409 return nil, ErrAccountAlreadyExists
410 }
411
412 err = tx.Put(append(PrefixKeyAccHash, v.Bytes()...), accountBytes[:])
413 if err != nil {
414 return nil, err
415 }
416 err = tx.Put(append(PrefixKeyIdx, idxBytes[:]...), v.Bytes())
417 if err != nil {
418 return nil, err
419 }
420
421 if err := tx.Commit(); err != nil {
422 return nil, err
423 }
424
425 if mt != nil {
426 return mt.AddAndGetCircomProof(idx.BigInt(), v)
427 }
428
429 return nil, nil
430}
431
432// UpdateAccount updates the Account in the StateDB for the given Idx. If
433// StateDB.mt==nil, MerkleTree is not affected, otherwise updates the
434// MerkleTree, returning a CircomProcessorProof.
435func (s *StateDB) UpdateAccount(idx common.Idx, account *common.Account) (*merkletree.CircomProcessorProof, error) {
436 return updateAccountInTreeDB(s.db, s.mt, idx, account)
437}
438
439// updateAccountInTreeDB is abstracted from StateDB to be used from StateDB and
440// from ExitTree. Updates the Account in the StateDB for the given Idx. If
441// StateDB.mt==nil, MerkleTree is not affected, otherwise updates the
442// MerkleTree, returning a CircomProcessorProof.
443func updateAccountInTreeDB(sto db.Storage, mt *merkletree.MerkleTree, idx common.Idx, account *common.Account) (*merkletree.CircomProcessorProof, error) {
444 // store at the DB the key: v, and value: account.Bytes()
445 v, err := account.HashValue()
446 if err != nil {
447 return nil, err
448 }
449 accountBytes, err := account.Bytes()
450 if err != nil {
451 return nil, err
452 }
453
454 tx, err := sto.NewTx()
455 if err != nil {
456 return nil, err
457 }
458 err = tx.Put(append(PrefixKeyAccHash, v.Bytes()...), accountBytes[:])
459 if err != nil {
460 return nil, err
461 }
462 idxBytes, err := idx.Bytes()
463 if err != nil {
464 return nil, err
465 }
466 err = tx.Put(append(PrefixKeyIdx, idxBytes[:]...), v.Bytes())
467 if err != nil {
468 return nil, err
469 }
470
471 if err := tx.Commit(); err != nil {
472 return nil, err
473 }
474
475 if mt != nil {
476 return mt.Update(idx.BigInt(), v)
477 }
478 return nil, nil
479}
480
481// MTGetProof returns the CircomVerifierProof for a given Idx
482func (s *StateDB) MTGetProof(idx common.Idx) (*merkletree.CircomVerifierProof, error) {
483 if s.mt == nil {
484 return nil, ErrStateDBWithoutMT
485 }
486 return s.mt.GenerateCircomVerifierProof(idx.BigInt(), s.mt.Root())
487}
488
489// MTGetRoot returns the current root of the underlying Merkle Tree
490func (s *StateDB) MTGetRoot() *big.Int {
491 return s.mt.Root().BigInt()
492}
493
494// LocalStateDB represents the local StateDB which allows to make copies from
495// the synchronizer StateDB, and is used by the tx-selector and the
496// batch-builder. LocalStateDB is an in-memory storage.
497type LocalStateDB struct {
498 *StateDB
499 synchronizerStateDB *StateDB
500}
501
502// NewLocalStateDB returns a new LocalStateDB connected to the given
503// synchronizerDB
504func NewLocalStateDB(path string, synchronizerDB *StateDB, typ TypeStateDB, nLevels int) (*LocalStateDB, error) {
505 s, err := NewStateDB(path, typ, nLevels)
506 if err != nil {
507 return nil, err
508 }
509 return &LocalStateDB{
510 s,
511 synchronizerDB,
512 }, nil
513}
514
515// Reset performs a reset in the LocaStateDB. If fromSynchronizer is true, it
516// gets the state from LocalStateDB.synchronizerStateDB for the given batchNum. If fromSynchronizer is false, get the state from LocalStateDB checkpoints.
517func (l *LocalStateDB) Reset(batchNum common.BatchNum, fromSynchronizer bool) error {
518 if batchNum == 0 {
519 l.idx = 0
520 return nil
521 }
522
523 synchronizerCheckpointPath := l.synchronizerStateDB.path + PathBatchNum + strconv.Itoa(int(batchNum))
524 checkpointPath := l.path + PathBatchNum + strconv.Itoa(int(batchNum))
525 currentPath := l.path + PathCurrent
526
527 if fromSynchronizer {
528 // use checkpoint from SynchronizerStateDB
529 if _, err := os.Stat(synchronizerCheckpointPath); os.IsNotExist(err) {
530 // if synchronizerStateDB does not have checkpoint at batchNum, return err
531 return fmt.Errorf("Checkpoint not exist in Synchronizer")
532 }
533
534 if err := l.db.Pebble().Close(); err != nil {
535 return err
536 }
537 // remove 'current'
538 err := os.RemoveAll(currentPath)
539 if err != nil {
540 return err
541 }
542 // copy synchronizer'BatchNumX' to 'current'
543 err = pebbleMakeCheckpoint(synchronizerCheckpointPath, currentPath)
544 if err != nil {
545 return err
546 }
547 // copy synchronizer'BatchNumX' to 'BatchNumX'
548 err = pebbleMakeCheckpoint(synchronizerCheckpointPath, checkpointPath)
549 if err != nil {
550 return err
551 }
552
553 // open the new 'current'
554 sto, err := pebble.NewPebbleStorage(currentPath, false)
555 if err != nil {
556 return err
557 }
558 l.db = sto
559
560 // get currentBatch num
561 l.currentBatch, err = l.GetCurrentBatch()
562 if err != nil {
563 return err
564 }
565 // open the MT for the current s.db
566 mt, err := merkletree.NewMerkleTree(l.db.WithPrefix(PrefixKeyMT), l.mt.MaxLevels())
567 if err != nil {
568 return err
569 }
570 l.mt = mt
571
572 return nil
573 }
574 // use checkpoint from LocalStateDB
575 return l.StateDB.reset(batchNum, true)
576}