From c83047f5270d83b31396e2791db149a7894657b7 Mon Sep 17 00:00:00 2001 From: arnaubennassar Date: Mon, 22 Feb 2021 14:28:34 +0100 Subject: [PATCH] Stop using stateDB in API --- api/account.go | 33 ---------------- api/api.go | 4 -- api/api_test.go | 50 ++++++++++++++---------- api/txspool.go | 8 ++-- cli/node/cfg.buidler.toml | 1 - config/config.go | 5 --- db/historydb/apiqueries.go | 40 ++++++++++++++++--- db/historydb/views.go | 4 +- db/migrations/0001.sql | 64 +++++++++++++++---------------- node/node.go | 6 +-- synchronizer/synchronizer.go | 29 +++++++------- synchronizer/synchronizer_test.go | 9 ++--- 12 files changed, 119 insertions(+), 134 deletions(-) diff --git a/api/account.go b/api/account.go index 54725c7..677af5a 100644 --- a/api/account.go +++ b/api/account.go @@ -4,10 +4,7 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/hermeznetwork/hermez-node/apitypes" "github.com/hermeznetwork/hermez-node/db/historydb" - "github.com/hermeznetwork/hermez-node/db/statedb" - "github.com/hermeznetwork/tracerr" ) func (a *API) getAccount(c *gin.Context) { @@ -23,16 +20,6 @@ func (a *API) getAccount(c *gin.Context) { return } - // Get balance from stateDB - account, err := a.s.LastGetAccount(*idx) - if err != nil { - retSQLErr(err, c) - return - } - - apiAccount.Balance = apitypes.NewBigIntStr(account.Balance) - apiAccount.Nonce = account.Nonce - c.JSON(http.StatusOK, apiAccount) } @@ -57,26 +44,6 @@ func (a *API) getAccounts(c *gin.Context) { return } - // Get balances from stateDB - if err := a.s.LastRead(func(sdb *statedb.Last) error { - for x, apiAccount := range apiAccounts { - idx, err := stringToIdx(string(apiAccount.Idx), "Account Idx") - if err != nil { - return tracerr.Wrap(err) - } - account, err := sdb.GetAccount(*idx) - if err != nil { - return tracerr.Wrap(err) - } - apiAccounts[x].Balance = apitypes.NewBigIntStr(account.Balance) - apiAccounts[x].Nonce = account.Nonce - } - return nil - }); err != nil { - retSQLErr(err, c) - return - } - // Build succesfull response type accountResponse struct { Accounts []historydb.AccountAPI `json:"accounts"` diff --git a/api/api.go b/api/api.go index c05fb35..b6a9764 100644 --- a/api/api.go +++ b/api/api.go @@ -9,7 +9,6 @@ import ( "github.com/hermeznetwork/hermez-node/common" "github.com/hermeznetwork/hermez-node/db/historydb" "github.com/hermeznetwork/hermez-node/db/l2db" - "github.com/hermeznetwork/hermez-node/db/statedb" "github.com/hermeznetwork/tracerr" ) @@ -34,7 +33,6 @@ type Status struct { type API struct { h *historydb.HistoryDB cg *configAPI - s *statedb.StateDB l2 *l2db.L2DB status Status chainID uint16 @@ -46,7 +44,6 @@ func NewAPI( coordinatorEndpoints, explorerEndpoints bool, server *gin.Engine, hdb *historydb.HistoryDB, - sdb *statedb.StateDB, l2db *l2db.L2DB, config *Config, ) (*API, error) { @@ -66,7 +63,6 @@ func NewAPI( AuctionConstants: config.AuctionConstants, WDelayerConstants: config.WDelayerConstants, }, - s: sdb, l2: l2db, status: Status{}, chainID: config.ChainID, diff --git a/api/api_test.go b/api/api_test.go index 9699a30..728a88f 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -22,7 +22,6 @@ import ( "github.com/hermeznetwork/hermez-node/db" "github.com/hermeznetwork/hermez-node/db/historydb" "github.com/hermeznetwork/hermez-node/db/l2db" - "github.com/hermeznetwork/hermez-node/db/statedb" "github.com/hermeznetwork/hermez-node/log" "github.com/hermeznetwork/hermez-node/test" "github.com/hermeznetwork/hermez-node/test/til" @@ -216,10 +215,6 @@ func TestMain(m *testing.M) { panic(err) } }() - sdb, err := statedb.NewStateDB(statedb.Config{Path: dir, Keep: 128, Type: statedb.TypeTxSelector, NLevels: 0}) - if err != nil { - panic(err) - } // L2DB l2DB := l2db.NewL2DB(database, 10, 1000, 24*time.Hour, apiConnCon) test.WipeDB(l2DB.DB()) // this will clean HistoryDB and L2DB @@ -239,7 +234,6 @@ func TestMain(m *testing.M) { true, apiGin, hdb, - sdb, l2DB, &_config, ) @@ -350,19 +344,6 @@ func TestMain(m *testing.M) { } } - // lastBlockNum2 := blocksData[len(blocksData)-1].Block.EthBlockNum - - // Add accounts to StateDB - for i := 0; i < len(commonAccounts); i++ { - if _, err := api.s.CreateAccount(commonAccounts[i].Idx, &commonAccounts[i]); err != nil { - panic(err) - } - } - // Make a checkpoint to make the accounts available in Last - if err := api.s.MakeCheckpoint(); err != nil { - panic(err) - } - // Generate Coordinators and add them to HistoryDB const nCoords = 10 commonCoords := test.GenCoordinators(nCoords, commonBlocks) @@ -529,13 +510,41 @@ func TestMain(m *testing.M) { testTxs := genTestTxs(commonL1Txs, commonL2Txs, commonAccounts, testTokens, commonBlocks) testBatches, testFullBatches := genTestBatches(commonBlocks, commonBatches, testTxs) poolTxsToSend, poolTxsToReceive := genTestPoolTxs(commonPoolTxs, testTokens, commonAccounts) + // Add balance and nonce to historyDB + accounts := genTestAccounts(commonAccounts, testTokens) + accUpdates := []common.AccountUpdate{} + for i := 0; i < len(accounts); i++ { + balance := new(big.Int) + balance.SetString(string(*accounts[i].Balance), 10) + idx, err := stringToIdx(string(accounts[i].Idx), "foo") + if err != nil { + panic(err) + } + accUpdates = append(accUpdates, common.AccountUpdate{ + EthBlockNum: 0, + BatchNum: 1, + Idx: *idx, + Nonce: 0, + Balance: balance, + }) + accUpdates = append(accUpdates, common.AccountUpdate{ + EthBlockNum: 0, + BatchNum: 1, + Idx: *idx, + Nonce: accounts[i].Nonce, + Balance: balance, + }) + } + if err := api.h.AddAccountUpdates(accUpdates); err != nil { + panic(err) + } tc = testCommon{ blocks: commonBlocks, tokens: testTokens, batches: testBatches, fullBatches: testFullBatches, coordinators: testCoords, - accounts: genTestAccounts(commonAccounts, testTokens), + accounts: accounts, txs: testTxs, exits: testExits, poolTxsToSend: poolTxsToSend, @@ -612,7 +621,6 @@ func TestTimeout(t *testing.T) { true, apiGinTO, hdbTO, - nil, l2DBTO, &_config, ) diff --git a/api/txspool.go b/api/txspool.go index b990818..782839a 100644 --- a/api/txspool.go +++ b/api/txspool.go @@ -170,13 +170,13 @@ func (a *API) verifyPoolL2TxWrite(txw *l2db.PoolL2TxWrite) error { if err != nil { return tracerr.Wrap(err) } - // Get public key - account, err := a.s.LastGetAccount(poolTx.FromIdx) + // Validate feeAmount + _, err = common.CalcFeeAmount(poolTx.Amount, poolTx.Fee) if err != nil { return tracerr.Wrap(err) } - // Validate feeAmount - _, err = common.CalcFeeAmount(poolTx.Amount, poolTx.Fee) + // Get public key + account, err := a.h.GetCommonAccountAPI(poolTx.FromIdx) if err != nil { return tracerr.Wrap(err) } diff --git a/cli/node/cfg.buidler.toml b/cli/node/cfg.buidler.toml index 941b6ed..82eb067 100644 --- a/cli/node/cfg.buidler.toml +++ b/cli/node/cfg.buidler.toml @@ -32,7 +32,6 @@ URL = "http://localhost:8545" [Synchronizer] SyncLoopInterval = "1s" StatsRefreshPeriod = "1s" -StoreAccountUpdates = true [SmartContracts] Rollup = "0x8EEaea23686c319133a7cC110b840d1591d9AeE0" diff --git a/config/config.go b/config/config.go index 07921f6..38a9d0e 100644 --- a/config/config.go +++ b/config/config.go @@ -236,11 +236,6 @@ type Node struct { // `Eth.LastBatch`). This value only affects the reported % of // synchronization of blocks and batches, nothing else. StatsRefreshPeriod Duration `validate:"required"` - // StoreAccountUpdates when set to true makes the synchronizer - // store every account update in the account_update SQL table. - // This allows querying nonces and balances from the HistoryDB - // via SQL. - StoreAccountUpdates bool } `validate:"required"` SmartContracts struct { // Rollup is the address of the Hermez.sol smart contract diff --git a/db/historydb/apiqueries.go b/db/historydb/apiqueries.go index d490ba9..7c30168 100644 --- a/db/historydb/apiqueries.go +++ b/db/historydb/apiqueries.go @@ -833,10 +833,18 @@ func (hdb *HistoryDB) GetAccountAPI(idx common.Idx) (*AccountAPI, error) { defer hdb.apiConnCon.Release() account := &AccountAPI{} err = meddler.QueryRow(hdb.db, account, `SELECT account.item_id, hez_idx(account.idx, - token.symbol) as idx, account.batch_num, account.bjj, account.eth_addr, - token.token_id, token.item_id AS token_item_id, token.eth_block_num AS token_block, - token.eth_addr as token_eth_addr, token.name, token.symbol, token.decimals, token.usd, token.usd_update - FROM account INNER JOIN token ON account.token_id = token.token_id WHERE idx = $1;`, idx) + token.symbol) as idx, account.batch_num, account.bjj, account.eth_addr, + token.token_id, token.item_id AS token_item_id, token.eth_block_num AS token_block, + token.eth_addr as token_eth_addr, token.name, token.symbol, token.decimals, token.usd, + token.usd_update, account_update.nonce, account_update.balance + FROM account inner JOIN ( + SELECT idx, nonce, balance + FROM account_update + WHERE idx = $1 + ORDER BY item_id DESC LIMIT 1 + ) AS account_update ON account_update.idx = account.idx + INNER JOIN token ON account.token_id = token.token_id + WHERE account.idx = $1;`, idx) if err != nil { return nil, tracerr.Wrap(err) @@ -864,8 +872,13 @@ func (hdb *HistoryDB) GetAccountsAPI( queryStr := `SELECT account.item_id, hez_idx(account.idx, token.symbol) as idx, account.batch_num, account.bjj, account.eth_addr, token.token_id, token.item_id AS token_item_id, token.eth_block_num AS token_block, token.eth_addr as token_eth_addr, token.name, token.symbol, token.decimals, token.usd, token.usd_update, - COUNT(*) OVER() AS total_items - FROM account INNER JOIN token ON account.token_id = token.token_id ` + account_update.nonce, account_update.balance, COUNT(*) OVER() AS total_items + FROM account inner JOIN ( + SELECT DISTINCT idx, + first_value(nonce) over(partition by idx ORDER BY item_id DESC) as nonce, + first_value(balance) over(partition by idx ORDER BY item_id DESC) as balance + FROM account_update + ) AS account_update ON account_update.idx = account.idx INNER JOIN token ON account.token_id = token.token_id ` // Apply filters nextIsAnd := false // ethAddr filter @@ -1024,3 +1037,18 @@ func (hdb *HistoryDB) GetAvgTxFeeAPI() (float64, error) { return avgTransactionFee, nil } + +// GetCommonAccountAPI returns the account associated to an account idx +func (hdb *HistoryDB) GetCommonAccountAPI(idx common.Idx) (*common.Account, error) { + cancel, err := hdb.apiConnCon.Acquire() + defer cancel() + if err != nil { + return nil, tracerr.Wrap(err) + } + defer hdb.apiConnCon.Release() + account := &common.Account{} + err = meddler.QueryRow( + hdb.db, account, `SELECT * FROM account WHERE idx = $1;`, idx, + ) + return account, tracerr.Wrap(err) +} diff --git a/db/historydb/views.go b/db/historydb/views.go index 8cfe182..1f8e018 100644 --- a/db/historydb/views.go +++ b/db/historydb/views.go @@ -239,8 +239,8 @@ type AccountAPI struct { BatchNum common.BatchNum `meddler:"batch_num"` PublicKey apitypes.HezBJJ `meddler:"bjj"` EthAddr apitypes.HezEthAddr `meddler:"eth_addr"` - Nonce common.Nonce `meddler:"-"` // max of 40 bits used - Balance *apitypes.BigIntStr `meddler:"-"` // max of 192 bits used + Nonce common.Nonce `meddler:"nonce"` // max of 40 bits used + Balance *apitypes.BigIntStr `meddler:"balance"` // max of 192 bits used TotalItems uint64 `meddler:"total_items"` FirstItem uint64 `meddler:"first_item"` LastItem uint64 `meddler:"last_item"` diff --git a/db/migrations/0001.sql b/db/migrations/0001.sql index 6e73402..c408ba8 100644 --- a/db/migrations/0001.sql +++ b/db/migrations/0001.sql @@ -662,35 +662,35 @@ CREATE TABLE account_creation_auth ( ); -- +migrate Down --- drop triggers -DROP TRIGGER trigger_token_usd_update ON token; -DROP TRIGGER trigger_set_tx ON tx; -DROP TRIGGER trigger_forge_l1_txs ON batch; -DROP TRIGGER trigger_set_pool_tx ON tx_pool; --- drop functions -DROP FUNCTION hez_idx; -DROP FUNCTION set_token_usd_update; -DROP FUNCTION fee_percentage; -DROP FUNCTION set_tx; -DROP FUNCTION forge_l1_user_txs; -DROP FUNCTION set_pool_tx; --- drop tables -DROP TABLE account_creation_auth; -DROP TABLE tx_pool; -DROP TABLE auction_vars; -DROP TABLE rollup_vars; -DROP TABLE escape_hatch_withdrawal; -DROP TABLE bucket_update; -DROP TABLE token_exchange; -DROP TABLE wdelayer_vars; -DROP TABLE tx; -DROP TABLE exit_tree; -DROP TABLE account_update; -DROP TABLE account; -DROP TABLE token; -DROP TABLE bid; -DROP TABLE batch; -DROP TABLE coordinator; -DROP TABLE block; --- drop sequences -DROP SEQUENCE tx_item_id; +-- triggers +DROP TRIGGER IF EXISTS trigger_token_usd_update ON token; +DROP TRIGGER IF EXISTS trigger_set_tx ON tx; +DROP TRIGGER IF EXISTS trigger_forge_l1_txs ON batch; +DROP TRIGGER IF EXISTS trigger_set_pool_tx ON tx_pool; +-- functions +DROP FUNCTION IF EXISTS hez_idx; +DROP FUNCTION IF EXISTS set_token_usd_update; +DROP FUNCTION IF EXISTS fee_percentage; +DROP FUNCTION IF EXISTS set_tx; +DROP FUNCTION IF EXISTS forge_l1_user_txs; +DROP FUNCTION IF EXISTS set_pool_tx; +-- drop tables IF EXISTS +DROP TABLE IF EXISTS account_creation_auth; +DROP TABLE IF EXISTS tx_pool; +DROP TABLE IF EXISTS auction_vars; +DROP TABLE IF EXISTS rollup_vars; +DROP TABLE IF EXISTS escape_hatch_withdrawal; +DROP TABLE IF EXISTS bucket_update; +DROP TABLE IF EXISTS token_exchange; +DROP TABLE IF EXISTS wdelayer_vars; +DROP TABLE IF EXISTS tx; +DROP TABLE IF EXISTS exit_tree; +DROP TABLE IF EXISTS account_update; +DROP TABLE IF EXISTS account; +DROP TABLE IF EXISTS token; +DROP TABLE IF EXISTS bid; +DROP TABLE IF EXISTS batch; +DROP TABLE IF EXISTS coordinator; +DROP TABLE IF EXISTS block; +-- sequences +DROP SEQUENCE IF EXISTS tx_item_id; diff --git a/node/node.go b/node/node.go index c5e0899..e133910 100644 --- a/node/node.go +++ b/node/node.go @@ -183,9 +183,8 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) { } sync, err := synchronizer.NewSynchronizer(client, historyDB, stateDB, synchronizer.Config{ - StatsRefreshPeriod: cfg.Synchronizer.StatsRefreshPeriod.Duration, - StoreAccountUpdates: cfg.Synchronizer.StoreAccountUpdates, - ChainID: chainIDU16, + StatsRefreshPeriod: cfg.Synchronizer.StatsRefreshPeriod.Duration, + ChainID: chainIDU16, }) if err != nil { return nil, tracerr.Wrap(err) @@ -430,7 +429,6 @@ func NewNodeAPI( coordinatorEndpoints, explorerEndpoints, engine, hdb, - sdb, l2db, config, ) diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index f2814c0..745cc4e 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -206,9 +206,8 @@ type SCConsts struct { // Config is the Synchronizer configuration type Config struct { - StatsRefreshPeriod time.Duration - StoreAccountUpdates bool - ChainID uint16 + StatsRefreshPeriod time.Duration + ChainID uint16 } // Synchronizer implements the Synchronizer type @@ -994,19 +993,17 @@ func (s *Synchronizer) rollupSync(ethBlock *common.Block) (*common.RollupData, e } batchData.CreatedAccounts = processTxsOut.CreatedAccounts - if s.cfg.StoreAccountUpdates { - batchData.UpdatedAccounts = make([]common.AccountUpdate, 0, - len(processTxsOut.UpdatedAccounts)) - for _, acc := range processTxsOut.UpdatedAccounts { - batchData.UpdatedAccounts = append(batchData.UpdatedAccounts, - common.AccountUpdate{ - EthBlockNum: blockNum, - BatchNum: batchNum, - Idx: acc.Idx, - Nonce: acc.Nonce, - Balance: acc.Balance, - }) - } + batchData.UpdatedAccounts = make([]common.AccountUpdate, 0, + len(processTxsOut.UpdatedAccounts)) + for _, acc := range processTxsOut.UpdatedAccounts { + batchData.UpdatedAccounts = append(batchData.UpdatedAccounts, + common.AccountUpdate{ + EthBlockNum: blockNum, + BatchNum: batchNum, + Idx: acc.Idx, + Nonce: acc.Nonce, + Balance: acc.Balance, + }) } slotNum := int64(0) diff --git a/synchronizer/synchronizer_test.go b/synchronizer/synchronizer_test.go index f22b3d1..ed50c81 100644 --- a/synchronizer/synchronizer_test.go +++ b/synchronizer/synchronizer_test.go @@ -346,8 +346,7 @@ func TestSyncGeneral(t *testing.T) { // Create Synchronizer s, err := NewSynchronizer(client, historyDB, stateDB, Config{ - StatsRefreshPeriod: 0 * time.Second, - StoreAccountUpdates: true, + StatsRefreshPeriod: 0 * time.Second, }) require.NoError(t, err) @@ -738,8 +737,7 @@ func TestSyncForgerCommitment(t *testing.T) { // Create Synchronizer s, err := NewSynchronizer(client, historyDB, stateDB, Config{ - StatsRefreshPeriod: 0 * time.Second, - StoreAccountUpdates: true, + StatsRefreshPeriod: 0 * time.Second, }) require.NoError(t, err) @@ -839,8 +837,7 @@ func TestSyncForgerCommitment(t *testing.T) { syncCommitment[syncBlock.Block.Num] = stats.Sync.Auction.CurrentSlot.ForgerCommitment s2, err := NewSynchronizer(client, historyDB, stateDB, Config{ - StatsRefreshPeriod: 0 * time.Second, - StoreAccountUpdates: true, + StatsRefreshPeriod: 0 * time.Second, }) require.NoError(t, err) stats = s2.Stats()