Merge pull request #564 from hermeznetwork/feature/api-without-statedb

Stop using stateDB in API

api/account.go

@@ -4,10 +4,7 @@ import (
 	"net/http"
 
 	"github.com/gin-gonic/gin"
-	"github.com/hermeznetwork/hermez-node/apitypes"
 	"github.com/hermeznetwork/hermez-node/db/historydb"
-	"github.com/hermeznetwork/hermez-node/db/statedb"
-	"github.com/hermeznetwork/tracerr"
 )
 
 func (a *API) getAccount(c *gin.Context) {
@@ -23,16 +20,6 @@ func (a *API) getAccount(c *gin.Context) {
 		return
 	}
 
-	// Get balance from stateDB
-	account, err := a.s.LastGetAccount(*idx)
-	if err != nil {
-		retSQLErr(err, c)
-		return
-	}
-
-	apiAccount.Balance = apitypes.NewBigIntStr(account.Balance)
-	apiAccount.Nonce = account.Nonce
-
 	c.JSON(http.StatusOK, apiAccount)
 }
 
@@ -57,26 +44,6 @@ func (a *API) getAccounts(c *gin.Context) {
 		return
 	}
 
-	// Get balances from stateDB
-	if err := a.s.LastRead(func(sdb *statedb.Last) error {
-		for x, apiAccount := range apiAccounts {
-			idx, err := stringToIdx(string(apiAccount.Idx), "Account Idx")
-			if err != nil {
-				return tracerr.Wrap(err)
-			}
-			account, err := sdb.GetAccount(*idx)
-			if err != nil {
-				return tracerr.Wrap(err)
-			}
-			apiAccounts[x].Balance = apitypes.NewBigIntStr(account.Balance)
-			apiAccounts[x].Nonce = account.Nonce
-		}
-		return nil
-	}); err != nil {
-		retSQLErr(err, c)
-		return
-	}
-
 	// Build succesfull response
 	type accountResponse struct {
 		Accounts     []historydb.AccountAPI `json:"accounts"`

api/api.go

@@ -9,7 +9,6 @@ import (
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db/historydb"
 	"github.com/hermeznetwork/hermez-node/db/l2db"
-	"github.com/hermeznetwork/hermez-node/db/statedb"
 	"github.com/hermeznetwork/tracerr"
 )
 
@@ -34,7 +33,6 @@ type Status struct {
 type API struct {
 	h             *historydb.HistoryDB
 	cg            *configAPI
-	s             *statedb.StateDB
 	l2            *l2db.L2DB
 	status        Status
 	chainID       uint16
@@ -46,7 +44,6 @@ func NewAPI(
 	coordinatorEndpoints, explorerEndpoints bool,
 	server *gin.Engine,
 	hdb *historydb.HistoryDB,
-	sdb *statedb.StateDB,
 	l2db *l2db.L2DB,
 	config *Config,
 ) (*API, error) {
@@ -66,7 +63,6 @@ func NewAPI(
 			AuctionConstants:  config.AuctionConstants,
 			WDelayerConstants: config.WDelayerConstants,
 		},
-		s:             sdb,
 		l2:            l2db,
 		status:        Status{},
 		chainID:       config.ChainID,

api/api_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/hermeznetwork/hermez-node/db"
 	"github.com/hermeznetwork/hermez-node/db/historydb"
 	"github.com/hermeznetwork/hermez-node/db/l2db"
-	"github.com/hermeznetwork/hermez-node/db/statedb"
 	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/hermez-node/test"
 	"github.com/hermeznetwork/hermez-node/test/til"
@@ -216,10 +215,6 @@ func TestMain(m *testing.M) {
 			panic(err)
 		}
 	}()
-	sdb, err := statedb.NewStateDB(statedb.Config{Path: dir, Keep: 128, Type: statedb.TypeTxSelector, NLevels: 0})
-	if err != nil {
-		panic(err)
-	}
 	// L2DB
 	l2DB := l2db.NewL2DB(database, 10, 1000, 0.0, 24*time.Hour, apiConnCon)
 	test.WipeDB(l2DB.DB()) // this will clean HistoryDB and L2DB
@@ -239,7 +234,6 @@ func TestMain(m *testing.M) {
 		true,
 		apiGin,
 		hdb,
-		sdb,
 		l2DB,
 		&_config,
 	)
@@ -350,19 +344,6 @@ func TestMain(m *testing.M) {
 		}
 	}
 
-	// lastBlockNum2 := blocksData[len(blocksData)-1].Block.EthBlockNum
-
-	// Add accounts to StateDB
-	for i := 0; i < len(commonAccounts); i++ {
-		if _, err := api.s.CreateAccount(commonAccounts[i].Idx, &commonAccounts[i]); err != nil {
-			panic(err)
-		}
-	}
-	// Make a checkpoint to make the accounts available in Last
-	if err := api.s.MakeCheckpoint(); err != nil {
-		panic(err)
-	}
-
 	// Generate Coordinators and add them to HistoryDB
 	const nCoords = 10
 	commonCoords := test.GenCoordinators(nCoords, commonBlocks)
@@ -529,13 +510,41 @@ func TestMain(m *testing.M) {
 	testTxs := genTestTxs(commonL1Txs, commonL2Txs, commonAccounts, testTokens, commonBlocks)
 	testBatches, testFullBatches := genTestBatches(commonBlocks, commonBatches, testTxs)
 	poolTxsToSend, poolTxsToReceive := genTestPoolTxs(commonPoolTxs, testTokens, commonAccounts)
+	// Add balance and nonce to historyDB
+	accounts := genTestAccounts(commonAccounts, testTokens)
+	accUpdates := []common.AccountUpdate{}
+	for i := 0; i < len(accounts); i++ {
+		balance := new(big.Int)
+		balance.SetString(string(*accounts[i].Balance), 10)
+		idx, err := stringToIdx(string(accounts[i].Idx), "foo")
+		if err != nil {
+			panic(err)
+		}
+		accUpdates = append(accUpdates, common.AccountUpdate{
+			EthBlockNum: 0,
+			BatchNum:    1,
+			Idx:         *idx,
+			Nonce:       0,
+			Balance:     balance,
+		})
+		accUpdates = append(accUpdates, common.AccountUpdate{
+			EthBlockNum: 0,
+			BatchNum:    1,
+			Idx:         *idx,
+			Nonce:       accounts[i].Nonce,
+			Balance:     balance,
+		})
+	}
+	if err := api.h.AddAccountUpdates(accUpdates); err != nil {
+		panic(err)
+	}
 	tc = testCommon{
 		blocks:           commonBlocks,
 		tokens:           testTokens,
 		batches:          testBatches,
 		fullBatches:      testFullBatches,
 		coordinators:     testCoords,
-		accounts:         genTestAccounts(commonAccounts, testTokens),
+		accounts:         accounts,
 		txs:              testTxs,
 		exits:            testExits,
 		poolTxsToSend:    poolTxsToSend,
@@ -612,7 +621,6 @@ func TestTimeout(t *testing.T) {
 		true,
 		apiGinTO,
 		hdbTO,
-		nil,
 		l2DBTO,
 		&_config,
 	)

api/txspool.go

@@ -171,13 +171,13 @@ func (a *API) verifyPoolL2TxWrite(txw *l2db.PoolL2TxWrite) error {
 	if err != nil {
 		return tracerr.Wrap(err)
 	}
-	// Get public key
-	account, err := a.s.LastGetAccount(poolTx.FromIdx)
+	// Validate feeAmount
+	_, err = common.CalcFeeAmount(poolTx.Amount, poolTx.Fee)
 	if err != nil {
 		return tracerr.Wrap(err)
 	}
-	// Validate feeAmount
-	_, err = common.CalcFeeAmount(poolTx.Amount, poolTx.Fee)
+	// Get public key
+	account, err := a.h.GetCommonAccountAPI(poolTx.FromIdx)
 	if err != nil {
 		return tracerr.Wrap(err)
 	}

cli/node/cfg.buidler.toml

@@ -32,7 +32,6 @@ URL = "http://localhost:8545"
 [Synchronizer]
 SyncLoopInterval = "1s"
 StatsRefreshPeriod = "1s"
-StoreAccountUpdates = true
 
 [SmartContracts]
 Rollup   = "0x8EEaea23686c319133a7cC110b840d1591d9AeE0"

config/config.go

@@ -240,11 +240,6 @@ type Node struct {
 		// `Eth.LastBatch`).  This value only affects the reported % of
 		// synchronization of blocks and batches, nothing else.
 		StatsRefreshPeriod Duration `validate:"required"`
-		// StoreAccountUpdates when set to true makes the synchronizer
-		// store every account update in the account_update SQL table.
-		// This allows querying nonces and balances from the HistoryDB
-		// via SQL.
-		StoreAccountUpdates bool
 	} `validate:"required"`
 	SmartContracts struct {
 		// Rollup is the address of the Hermez.sol smart contract

db/historydb/apiqueries.go

@@ -833,10 +833,18 @@ func (hdb *HistoryDB) GetAccountAPI(idx common.Idx) (*AccountAPI, error) {
 	defer hdb.apiConnCon.Release()
 	account := &AccountAPI{}
 	err = meddler.QueryRow(hdb.db, account, `SELECT account.item_id, hez_idx(account.idx, 
-	token.symbol) as idx, account.batch_num, account.bjj, account.eth_addr,
-	token.token_id, token.item_id AS token_item_id, token.eth_block_num AS token_block,
-	token.eth_addr as token_eth_addr, token.name, token.symbol, token.decimals, token.usd, token.usd_update 
-	FROM account INNER JOIN token ON account.token_id = token.token_id WHERE idx = $1;`, idx)
+		token.symbol) as idx, account.batch_num, account.bjj, account.eth_addr,
+		token.token_id, token.item_id AS token_item_id, token.eth_block_num AS token_block,
+		token.eth_addr as token_eth_addr, token.name, token.symbol, token.decimals, token.usd,
+		token.usd_update, account_update.nonce, account_update.balance 
+		FROM account inner JOIN (
+			SELECT idx, nonce, balance 
+			FROM account_update
+			WHERE idx = $1
+			ORDER BY item_id DESC LIMIT 1
+		) AS account_update ON account_update.idx = account.idx 
+		INNER JOIN token ON account.token_id = token.token_id
+		WHERE account.idx = $1;`, idx)
 
 	if err != nil {
 		return nil, tracerr.Wrap(err)
@@ -864,8 +872,13 @@ func (hdb *HistoryDB) GetAccountsAPI(
 	queryStr := `SELECT account.item_id, hez_idx(account.idx, token.symbol) as idx, account.batch_num, 
 	account.bjj, account.eth_addr, token.token_id, token.item_id AS token_item_id, token.eth_block_num AS token_block,
 	token.eth_addr as token_eth_addr, token.name, token.symbol, token.decimals, token.usd, token.usd_update, 
-	COUNT(*) OVER() AS total_items
-	FROM account INNER JOIN token ON account.token_id = token.token_id `
+	account_update.nonce, account_update.balance, COUNT(*) OVER() AS total_items
+	FROM account inner JOIN (
+		SELECT DISTINCT idx,
+		first_value(nonce) over(partition by idx ORDER BY item_id DESC) as nonce,
+		first_value(balance) over(partition by idx ORDER BY item_id DESC) as balance
+		FROM account_update
+	) AS account_update ON account_update.idx = account.idx INNER JOIN token ON account.token_id = token.token_id `
 	// Apply filters
 	nextIsAnd := false
 	// ethAddr filter
@@ -1024,3 +1037,18 @@ func (hdb *HistoryDB) GetAvgTxFeeAPI() (float64, error) {
 
 	return avgTransactionFee, nil
 }
+
+// GetCommonAccountAPI returns the account associated to an account idx
+func (hdb *HistoryDB) GetCommonAccountAPI(idx common.Idx) (*common.Account, error) {
+	cancel, err := hdb.apiConnCon.Acquire()
+	defer cancel()
+	if err != nil {
+		return nil, tracerr.Wrap(err)
+	}
+	defer hdb.apiConnCon.Release()
+	account := &common.Account{}
+	err = meddler.QueryRow(
+		hdb.db, account, `SELECT * FROM account WHERE idx = $1;`, idx,
+	)
+	return account, tracerr.Wrap(err)
+}

db/historydb/views.go

@@ -239,8 +239,8 @@ type AccountAPI struct {
 	BatchNum         common.BatchNum     `meddler:"batch_num"`
 	PublicKey        apitypes.HezBJJ     `meddler:"bjj"`
 	EthAddr          apitypes.HezEthAddr `meddler:"eth_addr"`
-	Nonce            common.Nonce        `meddler:"-"` // max of 40 bits used
-	Balance          *apitypes.BigIntStr `meddler:"-"` // max of 192 bits used
+	Nonce            common.Nonce        `meddler:"nonce"`   // max of 40 bits used
+	Balance          *apitypes.BigIntStr `meddler:"balance"` // max of 192 bits used
 	TotalItems       uint64              `meddler:"total_items"`
 	FirstItem        uint64              `meddler:"first_item"`
 	LastItem         uint64              `meddler:"last_item"`

db/migrations/0001.sql

@@ -662,35 +662,35 @@ CREATE TABLE account_creation_auth (
 );
 
 -- +migrate Down
--- drop triggers
-DROP TRIGGER trigger_token_usd_update ON token;
-DROP TRIGGER trigger_set_tx ON tx;
-DROP TRIGGER trigger_forge_l1_txs ON batch;
-DROP TRIGGER trigger_set_pool_tx ON tx_pool;
--- drop functions
-DROP FUNCTION hez_idx;
-DROP FUNCTION set_token_usd_update;
-DROP FUNCTION fee_percentage;
-DROP FUNCTION set_tx;
-DROP FUNCTION forge_l1_user_txs;
-DROP FUNCTION set_pool_tx;
--- drop tables
-DROP TABLE account_creation_auth;
-DROP TABLE tx_pool;
-DROP TABLE auction_vars;
-DROP TABLE rollup_vars;
-DROP TABLE escape_hatch_withdrawal;
-DROP TABLE bucket_update;
-DROP TABLE token_exchange;
-DROP TABLE wdelayer_vars;
-DROP TABLE tx;
-DROP TABLE exit_tree;
-DROP TABLE account_update;
-DROP TABLE account;
-DROP TABLE token;
-DROP TABLE bid;
-DROP TABLE batch;
-DROP TABLE coordinator;
-DROP TABLE block;
--- drop sequences
-DROP SEQUENCE tx_item_id;
+-- triggers
+DROP TRIGGER IF EXISTS trigger_token_usd_update ON token;
+DROP TRIGGER IF EXISTS trigger_set_tx ON tx;
+DROP TRIGGER IF EXISTS trigger_forge_l1_txs ON batch;
+DROP TRIGGER IF EXISTS trigger_set_pool_tx ON tx_pool;
+-- functions
+DROP FUNCTION IF EXISTS hez_idx;
+DROP FUNCTION IF EXISTS set_token_usd_update;
+DROP FUNCTION IF EXISTS fee_percentage;
+DROP FUNCTION IF EXISTS set_tx;
+DROP FUNCTION IF EXISTS forge_l1_user_txs;
+DROP FUNCTION IF EXISTS set_pool_tx;
+-- drop tables IF EXISTS
+DROP TABLE IF EXISTS account_creation_auth;
+DROP TABLE IF EXISTS tx_pool;
+DROP TABLE IF EXISTS auction_vars;
+DROP TABLE IF EXISTS rollup_vars;
+DROP TABLE IF EXISTS escape_hatch_withdrawal;
+DROP TABLE IF EXISTS bucket_update;
+DROP TABLE IF EXISTS token_exchange;
+DROP TABLE IF EXISTS wdelayer_vars;
+DROP TABLE IF EXISTS tx;
+DROP TABLE IF EXISTS exit_tree;
+DROP TABLE IF EXISTS account_update;
+DROP TABLE IF EXISTS account;
+DROP TABLE IF EXISTS token;
+DROP TABLE IF EXISTS bid;
+DROP TABLE IF EXISTS batch;
+DROP TABLE IF EXISTS coordinator;
+DROP TABLE IF EXISTS block;
+-- sequences
+DROP SEQUENCE IF EXISTS tx_item_id;

node/node.go

@@ -183,9 +183,8 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) {
 	}
 
 	sync, err := synchronizer.NewSynchronizer(client, historyDB, stateDB, synchronizer.Config{
-		StatsRefreshPeriod:  cfg.Synchronizer.StatsRefreshPeriod.Duration,
-		StoreAccountUpdates: cfg.Synchronizer.StoreAccountUpdates,
-		ChainID:             chainIDU16,
+		StatsRefreshPeriod: cfg.Synchronizer.StatsRefreshPeriod.Duration,
+		ChainID:            chainIDU16,
 	})
 	if err != nil {
 		return nil, tracerr.Wrap(err)
@@ -431,7 +430,6 @@ func NewNodeAPI(
 		coordinatorEndpoints, explorerEndpoints,
 		engine,
 		hdb,
-		sdb,
 		l2db,
 		config,
 	)

synchronizer/synchronizer.go

@@ -206,9 +206,8 @@ type SCConsts struct {
 
 // Config is the Synchronizer configuration
 type Config struct {
-	StatsRefreshPeriod  time.Duration
-	StoreAccountUpdates bool
-	ChainID             uint16
+	StatsRefreshPeriod time.Duration
+	ChainID            uint16
 }
 
 // Synchronizer implements the Synchronizer type
@@ -994,19 +993,17 @@ func (s *Synchronizer) rollupSync(ethBlock *common.Block) (*common.RollupData, e
 		}
 		batchData.CreatedAccounts = processTxsOut.CreatedAccounts
 
-		if s.cfg.StoreAccountUpdates {
-			batchData.UpdatedAccounts = make([]common.AccountUpdate, 0,
-				len(processTxsOut.UpdatedAccounts))
-			for _, acc := range processTxsOut.UpdatedAccounts {
-				batchData.UpdatedAccounts = append(batchData.UpdatedAccounts,
-					common.AccountUpdate{
-						EthBlockNum: blockNum,
-						BatchNum:    batchNum,
-						Idx:         acc.Idx,
-						Nonce:       acc.Nonce,
-						Balance:     acc.Balance,
-					})
-			}
+		batchData.UpdatedAccounts = make([]common.AccountUpdate, 0,
+			len(processTxsOut.UpdatedAccounts))
+		for _, acc := range processTxsOut.UpdatedAccounts {
+			batchData.UpdatedAccounts = append(batchData.UpdatedAccounts,
+				common.AccountUpdate{
+					EthBlockNum: blockNum,
+					BatchNum:    batchNum,
+					Idx:         acc.Idx,
+					Nonce:       acc.Nonce,
+					Balance:     acc.Balance,
+				})
 		}
 
 		slotNum := int64(0)

synchronizer/synchronizer_test.go

@@ -346,8 +346,7 @@ func TestSyncGeneral(t *testing.T) {
 
 	// Create Synchronizer
 	s, err := NewSynchronizer(client, historyDB, stateDB, Config{
-		StatsRefreshPeriod:  0 * time.Second,
-		StoreAccountUpdates: true,
+		StatsRefreshPeriod: 0 * time.Second,
 	})
 	require.NoError(t, err)
 
@@ -738,8 +737,7 @@ func TestSyncForgerCommitment(t *testing.T) {
 
 	// Create Synchronizer
 	s, err := NewSynchronizer(client, historyDB, stateDB, Config{
-		StatsRefreshPeriod:  0 * time.Second,
-		StoreAccountUpdates: true,
+		StatsRefreshPeriod: 0 * time.Second,
 	})
 	require.NoError(t, err)
 
@@ -839,8 +837,7 @@ func TestSyncForgerCommitment(t *testing.T) {
 		syncCommitment[syncBlock.Block.Num] = stats.Sync.Auction.CurrentSlot.ForgerCommitment
 
 		s2, err := NewSynchronizer(client, historyDB, stateDB, Config{
-			StatsRefreshPeriod:  0 * time.Second,
-			StoreAccountUpdates: true,
+			StatsRefreshPeriod: 0 * time.Second,
 		})
 		require.NoError(t, err)
 		stats = s2.Stats()