Extend statedb and use prefixes, add debugapi

In statedb:
- Store all values using prefixes for keys to allow iteration
- Add methods MTGetRoot, GetAccounts

Implement debugapi, an http server with debugging endpoints:
- debugAPI.GET("sdb/batchnum", a.handleCurrentBatch)
- debugAPI.GET("sdb/mtroot", a.handleMTRoot)
- debugAPI.GET("sdb/accounts", a.handleAccounts)
- debugAPI.GET("sdb/accounts/:Idx", a.handleAccount)

db/statedb/statedb.go

@@ -3,6 +3,7 @@ package statedb
 import (
 	"errors"
 	"fmt"
+	"math/big"
 	"os"
 	"os/exec"
 	"strconv"
@@ -29,7 +30,18 @@ var (
 	ErrToIdxNotFound = errors.New("ToIdx can not be found")
 
 	// KeyCurrentBatch is used as key in the db to store the current BatchNum
-	KeyCurrentBatch = []byte("currentbatch")
+	KeyCurrentBatch = []byte("k:currentbatch")
+
+	// PrefixKeyIdx is the key prefix for idx in the db
+	PrefixKeyIdx = []byte("i:")
+	// PrefixKeyAccHash is the key prefix for account hash in the db
+	PrefixKeyAccHash = []byte("h:")
+	// PrefixKeyMT is the key prefix for merkle tree in the db
+	PrefixKeyMT = []byte("m:")
+	// PrefixKeyAddr is the key prefix for address in the db
+	PrefixKeyAddr = []byte("a:")
+	// PrefixKeyAddrBJJ is the key prefix for address-babyjubjub in the db
+	PrefixKeyAddrBJJ = []byte("ab:")
 )
 
 const (
@@ -80,7 +92,7 @@ func NewStateDB(path string, typ TypeStateDB, nLevels int) (*StateDB, error) {
 
 	var mt *merkletree.MerkleTree = nil
 	if typ == TypeSynchronizer || typ == TypeBatchBuilder {
-		mt, err = merkletree.NewMerkleTree(sto, nLevels)
+		mt, err = merkletree.NewMerkleTree(sto.WithPrefix(PrefixKeyMT), nLevels)
 		if err != nil {
 			return nil, err
 		}
@@ -236,7 +248,7 @@ func (s *StateDB) Reset(batchNum common.BatchNum) error {
 
 	if s.mt != nil {
 		// open the MT for the current s.db
-		mt, err := merkletree.NewMerkleTree(s.db, s.mt.MaxLevels())
+		mt, err := merkletree.NewMerkleTree(s.db.WithPrefix(PrefixKeyMT), s.mt.MaxLevels())
 		if err != nil {
 			return err
 		}
@@ -251,6 +263,35 @@ func (s *StateDB) GetAccount(idx common.Idx) (*common.Account, error) {
 	return getAccountInTreeDB(s.db, idx)
 }
 
+// GetAccounts returns all the accounts in the db.  Use for debugging pruposes
+// only.
+func (s *StateDB) GetAccounts() ([]common.Account, error) {
+	idxDB := s.db.WithPrefix(PrefixKeyIdx)
+	idxs := []common.Idx{}
+	// NOTE: Current implementation of Iterate in the pebble interface is
+	// not efficient, as it iterates over all keys.  Improve it following
+	// this example: https://github.com/cockroachdb/pebble/pull/923/files
+	if err := idxDB.Iterate(func(k []byte, v []byte) (bool, error) {
+		idx, err := common.IdxFromBytes(k)
+		if err != nil {
+			return false, err
+		}
+		idxs = append(idxs, idx)
+		return true, nil
+	}); err != nil {
+		return nil, err
+	}
+	accs := []common.Account{}
+	for i := range idxs {
+		acc, err := s.GetAccount(idxs[i])
+		if err != nil {
+			return nil, err
+		}
+		accs = append(accs, *acc)
+	}
+	return accs, nil
+}
+
 // getAccountInTreeDB is abstracted from StateDB to be used from StateDB and
 // from ExitTree.  GetAccount returns the account for the given Idx
 func getAccountInTreeDB(sto db.Storage, idx common.Idx) (*common.Account, error) {
@@ -258,17 +299,22 @@ func getAccountInTreeDB(sto db.Storage, idx common.Idx) (*common.Account, error)
 	if err != nil {
 		return nil, err
 	}
-	vBytes, err := sto.Get(idxBytes[:])
+	vBytes, err := sto.Get(append(PrefixKeyIdx, idxBytes[:]...))
 	if err != nil {
 		return nil, err
 	}
-	accBytes, err := sto.Get(vBytes)
+	accBytes, err := sto.Get(append(PrefixKeyAccHash, vBytes...))
 	if err != nil {
 		return nil, err
 	}
 	var b [32 * common.NLeafElems]byte
 	copy(b[:], accBytes)
-	return common.AccountFromBytes(b)
+	account, err := common.AccountFromBytes(b)
+	if err != nil {
+		return nil, err
+	}
+	account.Idx = idx
+	return account, nil
 }
 
 // CreateAccount creates a new Account in the StateDB for the given Idx.  If
@@ -309,16 +355,16 @@ func createAccountInTreeDB(sto db.Storage, mt *merkletree.MerkleTree, idx common
 	if err != nil {
 		return nil, err
 	}
-	_, err = tx.Get(idxBytes[:])
+	_, err = tx.Get(append(PrefixKeyIdx, idxBytes[:]...))
 	if err != db.ErrNotFound {
 		return nil, ErrAccountAlreadyExists
 	}
 
-	err = tx.Put(v.Bytes(), accountBytes[:])
+	err = tx.Put(append(PrefixKeyAccHash, v.Bytes()...), accountBytes[:])
 	if err != nil {
 		return nil, err
 	}
-	err = tx.Put(idxBytes[:], v.Bytes())
+	err = tx.Put(append(PrefixKeyIdx, idxBytes[:]...), v.Bytes())
 	if err != nil {
 		return nil, err
 	}
@@ -360,7 +406,7 @@ func updateAccountInTreeDB(sto db.Storage, mt *merkletree.MerkleTree, idx common
 	if err != nil {
 		return nil, err
 	}
-	err = tx.Put(v.Bytes(), accountBytes[:])
+	err = tx.Put(append(PrefixKeyAccHash, v.Bytes()...), accountBytes[:])
 	if err != nil {
 		return nil, err
 	}
@@ -368,7 +414,7 @@ func updateAccountInTreeDB(sto db.Storage, mt *merkletree.MerkleTree, idx common
 	if err != nil {
 		return nil, err
 	}
-	err = tx.Put(idxBytes[:], v.Bytes())
+	err = tx.Put(append(PrefixKeyIdx, idxBytes[:]...), v.Bytes())
 	if err != nil {
 		return nil, err
 	}
@@ -391,6 +437,11 @@ func (s *StateDB) MTGetProof(idx common.Idx) (*merkletree.CircomVerifierProof, e
 	return s.mt.GenerateCircomVerifierProof(idx.BigInt(), s.mt.Root())
 }
 
+// MTGetRoot returns the current root of the underlying Merkle Tree
+func (s *StateDB) MTGetRoot() *big.Int {
+	return s.mt.Root().BigInt()
+}
+
 // LocalStateDB represents the local StateDB which allows to make copies from
 // the synchronizer StateDB, and is used by the tx-selector and the
 // batch-builder. LocalStateDB is an in-memory storage.
@@ -462,7 +513,7 @@ func (l *LocalStateDB) Reset(batchNum common.BatchNum, fromSynchronizer bool) er
 			return err
 		}
 		// open the MT for the current s.db
-		mt, err := merkletree.NewMerkleTree(l.db, l.mt.MaxLevels())
+		mt, err := merkletree.NewMerkleTree(l.db.WithPrefix(PrefixKeyMT), l.mt.MaxLevels())
 		if err != nil {
 			return err
 		}

db/statedb/statedb_test.go

@@ -26,6 +26,7 @@ func newAccount(t *testing.T, i int) *common.Account {
 	address := ethCrypto.PubkeyToAddress(key.PublicKey)
 
 	return &common.Account{
+		Idx:       common.Idx(256 + i),
 		TokenID:   common.TokenID(i),
 		Nonce:     common.Nonce(i),
 		Balance:   big.NewInt(1000),
@@ -124,7 +125,7 @@ func TestStateDBWithoutMT(t *testing.T) {
 
 	// create test accounts
 	var accounts []*common.Account
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 4; i++ {
 		accounts = append(accounts, newAccount(t, i))
 	}
 
@@ -136,22 +137,22 @@ func TestStateDBWithoutMT(t *testing.T) {
 
 	// add test accounts
 	for i := 0; i < len(accounts); i++ {
-		_, err = sdb.CreateAccount(common.Idx(i), accounts[i])
+		_, err = sdb.CreateAccount(accounts[i].Idx, accounts[i])
 		assert.Nil(t, err)
 	}
 
 	for i := 0; i < len(accounts); i++ {
-		existingAccount := common.Idx(i)
+		existingAccount := accounts[i].Idx
 		accGetted, err := sdb.GetAccount(existingAccount)
 		assert.Nil(t, err)
 		assert.Equal(t, accounts[i], accGetted)
 	}
 
 	// try already existing idx and get error
-	existingAccount := common.Idx(1)
+	existingAccount := common.Idx(256)
 	_, err = sdb.GetAccount(existingAccount) // check that exist
 	assert.Nil(t, err)
-	_, err = sdb.CreateAccount(common.Idx(1), accounts[1]) // check that can not be created twice
+	_, err = sdb.CreateAccount(common.Idx(256), accounts[1]) // check that can not be created twice
 	assert.NotNil(t, err)
 	assert.Equal(t, ErrAccountAlreadyExists, err)
 
@@ -188,35 +189,35 @@ func TestStateDBWithMT(t *testing.T) {
 
 	// add test accounts
 	for i := 0; i < len(accounts); i++ {
-		_, err = sdb.CreateAccount(common.Idx(i), accounts[i])
+		_, err = sdb.CreateAccount(accounts[i].Idx, accounts[i])
 		assert.Nil(t, err)
 	}
 
 	for i := 0; i < len(accounts); i++ {
-		accGetted, err := sdb.GetAccount(common.Idx(i))
+		accGetted, err := sdb.GetAccount(accounts[i].Idx)
 		assert.Nil(t, err)
 		assert.Equal(t, accounts[i], accGetted)
 	}
 
 	// try already existing idx and get error
-	_, err = sdb.GetAccount(common.Idx(1)) // check that exist
+	_, err = sdb.GetAccount(common.Idx(256)) // check that exist
 	assert.Nil(t, err)
-	_, err = sdb.CreateAccount(common.Idx(1), accounts[1]) // check that can not be created twice
+	_, err = sdb.CreateAccount(common.Idx(256), accounts[1]) // check that can not be created twice
 	assert.NotNil(t, err)
 	assert.Equal(t, ErrAccountAlreadyExists, err)
 
-	_, err = sdb.MTGetProof(common.Idx(1))
+	_, err = sdb.MTGetProof(common.Idx(256))
 	assert.Nil(t, err)
 
 	// update accounts
 	for i := 0; i < len(accounts); i++ {
 		accounts[i].Nonce = accounts[i].Nonce + 1
-		_, err = sdb.UpdateAccount(common.Idx(i), accounts[i])
+		_, err = sdb.UpdateAccount(accounts[i].Idx, accounts[i])
 		assert.Nil(t, err)
 	}
-	a, err := sdb.GetAccount(common.Idx(1)) // check that account value has been updated
+	a, err := sdb.GetAccount(common.Idx(256)) // check that account value has been updated
 	assert.Nil(t, err)
-	assert.Equal(t, accounts[1].Nonce, a.Nonce)
+	assert.Equal(t, accounts[0].Nonce, a.Nonce)
 }
 
 func TestCheckpoints(t *testing.T) {
@@ -234,7 +235,7 @@ func TestCheckpoints(t *testing.T) {
 
 	// add test accounts
 	for i := 0; i < len(accounts); i++ {
-		_, err = sdb.CreateAccount(common.Idx(i), accounts[i])
+		_, err = sdb.CreateAccount(accounts[i].Idx, accounts[i])
 		assert.Nil(t, err)
 	}
 
@@ -334,6 +335,31 @@ func TestCheckpoints(t *testing.T) {
 	}
 }
 
+func TestStateDBGetAccounts(t *testing.T) {
+	dir, err := ioutil.TempDir("", "tmpdb")
+	require.Nil(t, err)
+
+	sdb, err := NewStateDB(dir, TypeTxSelector, 0)
+	assert.Nil(t, err)
+
+	// create test accounts
+	var accounts []common.Account
+	for i := 0; i < 16; i++ {
+		account := newAccount(t, i)
+		accounts = append(accounts, *account)
+	}
+
+	// add test accounts
+	for i := range accounts {
+		_, err = sdb.CreateAccount(accounts[i].Idx, &accounts[i])
+		require.Nil(t, err)
+	}
+
+	dbAccounts, err := sdb.GetAccounts()
+	require.Nil(t, err)
+	assert.Equal(t, accounts, dbAccounts)
+}
+
 func printCheckpoints(t *testing.T, path string) {
 	files, err := ioutil.ReadDir(path)
 	assert.Nil(t, err)

db/statedb/txprocessors.go

@@ -14,7 +14,7 @@ import (
 
 var (
 	// keyidx is used as key in the db to store the current Idx
-	keyidx = []byte("idx")
+	keyidx = []byte("k:idx")
 )
 
 func (s *StateDB) resetZKInputs() {

db/statedb/utils.go

@@ -51,12 +51,12 @@ func (s *StateDB) setIdxByEthAddrBJJ(idx common.Idx, addr ethCommon.Address, pk
 	if err != nil {
 		return err
 	}
-	err = tx.Put(k, idxBytes[:])
+	err = tx.Put(append(PrefixKeyAddrBJJ, k...), idxBytes[:])
 	if err != nil {
 		return err
 	}
 	// store Addr-idx
-	err = tx.Put(addr.Bytes(), idxBytes[:])
+	err = tx.Put(append(PrefixKeyAddr, addr.Bytes()...), idxBytes[:])
 	if err != nil {
 		return err
 	}
@@ -71,7 +71,7 @@ func (s *StateDB) setIdxByEthAddrBJJ(idx common.Idx, addr ethCommon.Address, pk
 // Ethereum Address. Will return common.Idx(0) and error in case that Idx is
 // not found in the StateDB.
 func (s *StateDB) GetIdxByEthAddr(addr ethCommon.Address) (common.Idx, error) {
-	b, err := s.db.Get(addr.Bytes())
+	b, err := s.db.Get(append(PrefixKeyAddr, addr.Bytes()...))
 	if err != nil {
 		return common.Idx(0), ErrToIdxNotFound
 	}
@@ -94,7 +94,7 @@ func (s *StateDB) GetIdxByEthAddrBJJ(addr ethCommon.Address, pk *babyjub.PublicK
 	} else if !bytes.Equal(addr.Bytes(), common.EmptyAddr.Bytes()) && pk != nil {
 		// case ToEthAddr!=0 && ToBJJ!=0
 		k := concatEthAddrBJJ(addr, pk)
-		b, err := s.db.Get(k)
+		b, err := s.db.Get(append(PrefixKeyAddrBJJ, k...))
 		if err != nil {
 			return common.Idx(0), ErrToIdxNotFound
 		}