From 5ccea68905d179aeb199e8ba9d541cf6a6e74db3 Mon Sep 17 00:00:00 2001 From: Eduard S Date: Thu, 18 Feb 2021 10:24:54 +0100 Subject: [PATCH] WIP --- db/statedb/statedb.go | 3 +- node/node.go | 5 +- test/debugapi/debugapi.go | 107 +++++++++++++++++++++++-- test/debugapi/debugapi_test.go | 139 ++++++++++++++++++++++++++++++++- 4 files changed, 244 insertions(+), 10 deletions(-) diff --git a/db/statedb/statedb.go b/db/statedb/statedb.go index 12c8200..fc536b1 100644 --- a/db/statedb/statedb.go +++ b/db/statedb/statedb.go @@ -275,7 +275,8 @@ func (s *StateDB) GetAccount(idx common.Idx) (*common.Account, error) { return GetAccountInTreeDB(s.db.DB(), idx) } -func accountsIter(db db.Storage, fn func(a *common.Account) (bool, error)) error { +// AccountsIter iterates over all the accounts in db, calling fn for each one +func AccountsIter(db db.Storage, fn func(a *common.Account) (bool, error)) error { idxDB := db.WithPrefix(PrefixKeyIdx) if err := idxDB.Iterate(func(k []byte, v []byte) (bool, error) { idx, err := common.IdxFromBytes(k) diff --git a/node/node.go b/node/node.go index 4d28ec8..d2472b8 100644 --- a/node/node.go +++ b/node/node.go @@ -376,7 +376,7 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) { } var debugAPI *debugapi.DebugAPI if cfg.Debug.APIAddress != "" { - debugAPI = debugapi.NewDebugAPI(cfg.Debug.APIAddress, stateDB, sync) + debugAPI = debugapi.NewDebugAPI(cfg.Debug.APIAddress, historyDB, stateDB, sync) } priceUpdater, err := priceupdater.NewPriceUpdater(cfg.PriceUpdater.URL, priceupdater.APIType(cfg.PriceUpdater.Type), historyDB) @@ -547,6 +547,9 @@ func (n *Node) syncLoopFn(ctx context.Context, lastBlock *common.Block) (*common WDelayer: blockData.WDelayer.Vars, } n.handleNewBlock(ctx, stats, vars, blockData.Rollup.Batches) + if n.debugAPI != nil { + n.debugAPI.SyncBlockHook() + } return &blockData.Block, time.Duration(0), nil } else { // case: no block diff --git a/test/debugapi/debugapi.go b/test/debugapi/debugapi.go index 9ff170f..a115934 100644 --- a/test/debugapi/debugapi.go +++ b/test/debugapi/debugapi.go @@ -2,12 +2,16 @@ package debugapi import ( "context" + "fmt" + "math/big" "net/http" + "sync" "time" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" "github.com/hermeznetwork/hermez-node/common" + "github.com/hermeznetwork/hermez-node/db/historydb" "github.com/hermeznetwork/hermez-node/db/statedb" "github.com/hermeznetwork/hermez-node/log" "github.com/hermeznetwork/hermez-node/synchronizer" @@ -31,22 +35,109 @@ func badReq(err error, c *gin.Context) { }) } +const ( + statusUpdating = "updating" + statusOK = "ok" +) + +type tokenBalances struct { + sync.RWMutex + Value struct { + Status string + Block *common.Block + Batch *common.Batch + Balances map[common.TokenID]*big.Int + } +} + +func (t *tokenBalances) Update(historyDB *historydb.HistoryDB, sdb *statedb.StateDB) (err error) { + var block *common.Block + var batch *common.Batch + var balances map[common.TokenID]*big.Int + defer func() { + t.Lock() + if err == nil { + t.Value.Status = statusOK + t.Value.Block = block + t.Value.Batch = batch + t.Value.Balances = balances + } else { + t.Value.Status = fmt.Sprintf("tokenBalances.Update: %v", err) + t.Value.Block = nil + t.Value.Batch = nil + t.Value.Balances = nil + } + t.Unlock() + }() + + if block, err = historyDB.GetLastBlock(); err != nil { + return tracerr.Wrap(err) + } + if batch, err = historyDB.GetLastBatch(); err != nil { + return tracerr.Wrap(err) + } + balances = make(map[common.TokenID]*big.Int) + sdb.LastRead(func(sdbLast *statedb.Last) error { + return tracerr.Wrap( + statedb.AccountsIter(sdbLast.DB(), func(a *common.Account) (bool, error) { + if balance, ok := balances[a.TokenID]; !ok { + balances[a.TokenID] = a.Balance + } else { + balance.Add(balance, a.Balance) + } + return true, nil + }), + ) + }) + return nil +} + // DebugAPI is an http API with debugging endpoints type DebugAPI struct { - addr string - stateDB *statedb.StateDB // synchronizer statedb - sync *synchronizer.Synchronizer + addr string + historyDB *historydb.HistoryDB + stateDB *statedb.StateDB // synchronizer statedb + sync *synchronizer.Synchronizer + tokenBalances tokenBalances } // NewDebugAPI creates a new DebugAPI -func NewDebugAPI(addr string, stateDB *statedb.StateDB, sync *synchronizer.Synchronizer) *DebugAPI { +func NewDebugAPI(addr string, historyDB *historydb.HistoryDB, stateDB *statedb.StateDB, + sync *synchronizer.Synchronizer) *DebugAPI { return &DebugAPI{ - addr: addr, - stateDB: stateDB, - sync: sync, + addr: addr, + historyDB: historyDB, + stateDB: stateDB, + sync: sync, + } +} + +// SyncBlockHook is a hook function that the node will call after every new synchronized block +func (a *DebugAPI) SyncBlockHook() { + a.tokenBalances.RLock() + updateTokenBalances := a.tokenBalances.Value.Status == statusUpdating + a.tokenBalances.RUnlock() + if updateTokenBalances { + if err := a.tokenBalances.Update(a.historyDB, a.stateDB); err != nil { + log.Errorw("DebugAPI.tokenBalances.Upate", "err", err) + } } } +func (a *DebugAPI) handleTokenBalances(c *gin.Context) { + a.tokenBalances.RLock() + tokenBalances := a.tokenBalances.Value + a.tokenBalances.RUnlock() + c.JSON(http.StatusOK, tokenBalances) +} + +func (a *DebugAPI) handlePostTokenBalances(c *gin.Context) { + a.tokenBalances.Lock() + a.tokenBalances.Value.Status = statusUpdating + a.tokenBalances.Unlock() + c.JSON(http.StatusOK, nil) +} + func (a *DebugAPI) handleAccount(c *gin.Context) { uri := struct { Idx uint32 @@ -114,6 +205,8 @@ func (a *DebugAPI) Run(ctx context.Context) error { // is created. debugAPI.GET("sdb/accounts", a.handleAccounts) debugAPI.GET("sdb/accounts/:Idx", a.handleAccount) + debugAPI.POST("sdb/tokenbalances", a.handlePostTokenBalances) + debugAPI.GET("sdb/tokenbalances", a.handleTokenBalances) debugAPI.GET("sync/stats", a.handleSyncStats) diff --git a/test/debugapi/debugapi_test.go b/test/debugapi/debugapi_test.go index 8cd900a..8bed818 100644 --- a/test/debugapi/debugapi_test.go +++ b/test/debugapi/debugapi_test.go @@ -6,13 +6,21 @@ import ( "io/ioutil" "math/big" "net/http" + "os" "strconv" + "sync" "testing" "github.com/dghubble/sling" + ethCommon "github.com/ethereum/go-ethereum/common" ethCrypto "github.com/ethereum/go-ethereum/crypto" "github.com/hermeznetwork/hermez-node/common" + dbUtils "github.com/hermeznetwork/hermez-node/db" + "github.com/hermeznetwork/hermez-node/db/historydb" "github.com/hermeznetwork/hermez-node/db/statedb" + "github.com/hermeznetwork/hermez-node/test" + "github.com/hermeznetwork/hermez-node/test/til" + "github.com/hermeznetwork/hermez-node/txprocessor" "github.com/iden3/go-iden3-crypto/babyjub" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -51,12 +59,15 @@ func TestDebugAPI(t *testing.T) { addr := "localhost:12345" // We won't test the sync/stats endpoint, so we can se the Syncrhonizer to nil - debugAPI := NewDebugAPI(addr, sdb, nil) + debugAPI := NewDebugAPI(addr, nil, sdb, nil) ctx, cancel := context.WithCancel(context.Background()) + wg := sync.WaitGroup{} + wg.Add(1) go func() { err := debugAPI.Run(ctx) require.Nil(t, err) + wg.Done() }() var accounts []common.Account @@ -102,4 +113,130 @@ func TestDebugAPI(t *testing.T) { assert.Equal(t, accounts, accountsAPI) cancel() + wg.Wait() +} + +func TestDebugAPITokenBalances(t *testing.T) { + dir, err := ioutil.TempDir("", "tmpdb") + require.Nil(t, err) + + sdb, err := statedb.NewStateDB(statedb.Config{Path: dir, Keep: 128, Type: statedb.TypeSynchronizer, NLevels: 32}) + require.Nil(t, err) + + // Init History DB + pass := os.Getenv("POSTGRES_PASS") + db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez") + require.NoError(t, err) + historyDB := historydb.NewHistoryDB(db, nil) + // Clear DB + test.WipeDB(historyDB.DB()) + + set := ` + Type: Blockchain + + AddToken(1) + AddToken(2) + AddToken(3) + + CreateAccountDeposit(1) A: 1000 + CreateAccountDeposit(2) A: 2000 + CreateAccountDeposit(1) B: 100 + CreateAccountDeposit(2) B: 200 + CreateAccountDeposit(2) C: 400 + + > batchL1 // forge L1UserTxs{nil}, freeze defined L1UserTxs{5} + > batchL1 // forge defined L1UserTxs{5}, freeze L1UserTxs{nil} + > block // blockNum=2 + ` + + chainID := uint16(0) + tc := til.NewContext(chainID, common.RollupConstMaxL1UserTx) + tilCfgExtra := til.ConfigExtra{ + BootCoordAddr: ethCommon.HexToAddress("0xE39fEc6224708f0772D2A74fd3f9055A90E0A9f2"), + CoordUser: "A", + } + blocks, err := tc.GenerateBlocks(set) + require.NoError(t, err) + + err = tc.FillBlocksExtra(blocks, &tilCfgExtra) + require.NoError(t, err) + tc.FillBlocksL1UserTxsBatchNum(blocks) + err = tc.FillBlocksForgedL1UserTxs(blocks) + require.NoError(t, err) + + tpc := txprocessor.Config{ + NLevels: 32, + MaxTx: 100, + ChainID: chainID, + MaxFeeTx: common.RollupConstMaxFeeIdxCoordinator, + MaxL1Tx: common.RollupConstMaxL1Tx, + } + tp := txprocessor.NewTxProcessor(sdb, tpc) + + for _, block := range blocks { + require.NoError(t, historyDB.AddBlockSCData(&block)) + for _, batch := range block.Rollup.Batches { + _, err := tp.ProcessTxs(batch.Batch.FeeIdxsCoordinator, + batch.L1UserTxs, batch.L1CoordinatorTxs, []common.PoolL2Tx{}) + require.NoError(t, err) + } + } + + addr := "localhost:12345" + // We won't test the sync/stats endpoint, so we can se the Syncrhonizer to nil + debugAPI := NewDebugAPI(addr, historyDB, sdb, nil) + + ctx, cancel := context.WithCancel(context.Background()) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + err := debugAPI.Run(ctx) + require.Nil(t, err) + wg.Done() + }() + + var accounts []common.Account + for i := 0; i < 16; i++ { + account := newAccount(t, i) + accounts = append(accounts, *account) + _, err = sdb.CreateAccount(account.Idx, account) + require.Nil(t, err) + } + // Make a checkpoint (batchNum 2) to make the accounts available in Last + err = sdb.MakeCheckpoint() + require.Nil(t, err) + + url := fmt.Sprintf("http://%v/debug/", addr) + + var batchNum common.BatchNum + req, err := sling.New().Get(url).Path("sdb/batchnum").ReceiveSuccess(&batchNum) + require.Equal(t, http.StatusOK, req.StatusCode) + require.Nil(t, err) + assert.Equal(t, common.BatchNum(2), batchNum) + + var mtroot *big.Int + req, err = sling.New().Get(url).Path("sdb/mtroot").ReceiveSuccess(&mtroot) + require.Equal(t, http.StatusOK, req.StatusCode) + require.Nil(t, err) + // Testing against a hardcoded value obtained by running the test and + // printing the value previously. + assert.Equal(t, "21765339739823365993496282904432398015268846626944509989242908567129545640185", + mtroot.String()) + + var accountAPI common.Account + req, err = sling.New().Get(url). + Path(fmt.Sprintf("sdb/accounts/%v", accounts[0].Idx)). + ReceiveSuccess(&accountAPI) + require.Equal(t, http.StatusOK, req.StatusCode) + require.Nil(t, err) + assert.Equal(t, accounts[0], accountAPI) + + var accountsAPI []common.Account + req, err = sling.New().Get(url).Path("sdb/accounts").ReceiveSuccess(&accountsAPI) + require.Equal(t, http.StatusOK, req.StatusCode) + require.Nil(t, err) + assert.Equal(t, accounts, accountsAPI) + + cancel() + wg.Wait() }