From 3e7173b1da10a64a16e02933f8cd8e6fbe39bd94 Mon Sep 17 00:00:00 2001 From: Arnau B Date: Tue, 25 Aug 2020 17:47:23 +0200 Subject: [PATCH] Impl historyDB methods for sync main loop --- common/batch.go | 23 +++--- common/scvars.go | 2 +- db/historydb/historydb.go | 90 +++++++++++++++++------ db/historydb/historydb_test.go | 106 +++++++++++++++++++++++---- db/historydb/migrations/001_init.sql | 1 - go.sum | 1 + 6 files changed, 169 insertions(+), 54 deletions(-) diff --git a/common/batch.go b/common/batch.go index daa3357..1eadb33 100644 --- a/common/batch.go +++ b/common/batch.go @@ -10,20 +10,15 @@ import ( // Batch is a struct that represents Hermez network batch type Batch struct { - BatchNum BatchNum - SlotNum SlotNum // Slot in which the batch is forged - EthTxHash ethCommon.Hash - EthBlockNum uint64 // Ethereum block in which the batch is forged - ExitRoot Hash - OldStateRoot Hash - NewStateRoot Hash - OldNumAccounts int - NewNumAccounts int - ToForgeL1TxsNum uint32 // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch. - ToForgeL1TxsHash ethCommon.Hash // optional, Only when the batch forges L1 txs. Frozen from pendingL1TxsHash (which are the group of L1UserTxs), to be forged in ToForgeL1TxsNum + 1. - ForgedL1TxsHash ethCommon.Hash // optional, Only when the batch forges L1 txs. This will be the Hash of the group of L1 txs (L1UserTxs + L1CoordinatorTx) forged in the current batch. - CollectedFees map[TokenID]*big.Int - ForgerAddr ethCommon.Address // TODO: Should this be retrieved via slot reference? + BatchNum BatchNum `meddler:"batch_num"` + EthBlockNum uint64 `meddler:"eth_block_num"` // Ethereum block in which the batch is forged + ForgerAddr ethCommon.Address `meddler:"forger_addr"` // TODO: Should this be retrieved via slot reference? + CollectedFees map[TokenID]*big.Int `meddler:"fees_collected,json"` + StateRoot Hash `meddler:"state_root"` + NumAccounts int `meddler:"num_accounts"` + ExitRoot Hash `meddler:"exit_root"` + ForgeL1TxsNum uint32 `meddler:"forge_l1_txs_num"` // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch. + SlotNum SlotNum `meddler:"slot_num"` // Slot in which the batch is forged } // BatchNum identifies a batch diff --git a/common/scvars.go b/common/scvars.go index 50da3d3..415de48 100644 --- a/common/scvars.go +++ b/common/scvars.go @@ -15,7 +15,7 @@ type RollupVars struct { Governance eth.Address } -type PoDVars struct { +type AuctionVars struct { EthBlockNum uint64 SlotDeadline uint CloseAuctionSlots uint diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index 7deae53..33f23cf 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -41,22 +41,18 @@ func NewHistoryDB(port int, host, user, password, dbname string) (*HistoryDB, er } // AddBlock insert a block into the DB -func (hdb *HistoryDB) AddBlock(blocks *common.Block) error { - return nil -} - -// addBlocks insert blocks into the DB. TODO: move method to test -func (hdb *HistoryDB) addBlocks(blocks []common.Block) error { - return db.BulkInsert( - hdb.db, - "INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s", - blocks[:], - ) +func (hdb *HistoryDB) AddBlock(block *common.Block) error { + return meddler.Insert(hdb.db, "block", block) } // GetBlock retrieve a block from the DB, given a block number func (hdb *HistoryDB) GetBlock(blockNum uint64) (*common.Block, error) { - return nil, nil + block := &common.Block{} + err := meddler.QueryRow( + hdb.db, block, + "SELECT * FROM block WHERE eth_block_num = $1;", blockNum, + ) + return block, err } // GetBlocks retrieve blocks from the DB, given a range of block numbers defined by from and to @@ -72,16 +68,58 @@ func (hdb *HistoryDB) GetBlocks(from, to uint64) ([]*common.Block, error) { // GetLastBlock retrieve the block with the highest block number from the DB func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) { - return nil, nil + block := &common.Block{} + err := meddler.QueryRow( + hdb.db, block, "SELECT * FROM block ORDER BY eth_block_num DESC LIMIT 1;", + ) + return block, err +} + +// addBatches insert Bids into the DB +func (hdb *HistoryDB) addBatches(batches []common.Batch) error { + return db.BulkInsert( + hdb.db, + `INSERT INTO batch ( + batch_num, + eth_block_num, + forger_addr, + fees_collected, + state_root, + num_accounts, + exit_root, + forge_l1_txs_num, + slot_num + ) VALUES %s;`, + batches[:], + ) +} + +// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to +func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]*common.Batch, error) { + var batches []*common.Batch + err := meddler.QueryAll( + hdb.db, &batches, + "SELECT * FROM batch WHERE $1 <= batch_num AND batch_num < $2", + from, to, + ) + return batches, err } // GetLastBatchNum returns the BatchNum of the latest forged batch -func (hdb *HistoryDB) GetLastBatchNum() (*common.BatchNum, error) { - return nil, nil +func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) { + row := hdb.db.QueryRow("SELECT batch_num FROM batch ORDER BY batch_num DESC LIMIT 1;") + var batchNum common.BatchNum + return batchNum, row.Scan(&batchNum) +} + +// GetLastL1TxsNum returns the greatest ForgeL1TxsNum in the DB +func (hdb *HistoryDB) GetLastL1TxsNum() (uint32, error) { + row := hdb.db.QueryRow("SELECT MAX(forge_l1_txs_num) FROM batch;") + var lastL1TxsNum uint32 + return lastL1TxsNum, row.Scan(&lastL1TxsNum) } // Reorg deletes all the information that was added into the DB after the lastValidBlock -// WARNING: this is a draaft of the function, useful at the moment for tests func (hdb *HistoryDB) Reorg(lastValidBlock uint64) error { _, err := hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock) return err @@ -93,12 +131,16 @@ func (hdb *HistoryDB) SyncRollup( l1txs []common.L1Tx, l2txs []common.L2Tx, registeredAccounts []common.Account, - exitTree common.ExitTreeLeaf, - withdrawals common.ExitTreeLeaf, + exitTree common.ExitInfo, + withdrawals common.ExitInfo, registeredTokens []common.Token, - batch *common.Batch, + batches []common.Batch, vars *common.RollupVars, ) error { + // TODO: make all in a single DB commit + if err := hdb.addBatches(batches); err != nil { + return err + } return nil } @@ -107,7 +149,7 @@ func (hdb *HistoryDB) SyncPoD( blockNum uint64, bids []common.Bid, coordinators []common.Coordinator, - vars *common.PoDVars, + vars *common.AuctionVars, ) error { return nil } @@ -122,13 +164,13 @@ func (hdb *HistoryDB) addBids(bids []common.Bid) error { ) } -// GetBidsByBlock return the bids done between the block from and to -func (hdb *HistoryDB) GetBidsByBlock(from, to uint64) ([]*common.Bid, error) { +// GetBidsBySlot return the bids for a specific slot +func (hdb *HistoryDB) GetBidsBySlot(slotNum common.SlotNum) ([]*common.Bid, error) { var bids []*common.Bid err := meddler.QueryAll( hdb.db, &bids, - "SELECT * FROM bid WHERE $1 <= eth_block_num AND eth_block_num < $2", - from, to, + "SELECT * FROM bid WHERE $1 = slot_num;", + slotNum, ) return bids, err } diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index 9a48a25..c662c52 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -9,6 +9,7 @@ import ( eth "github.com/ethereum/go-ethereum/common" "github.com/hermeznetwork/hermez-node/common" + "github.com/hermeznetwork/hermez-node/db" "github.com/stretchr/testify/assert" ) @@ -39,7 +40,7 @@ func TestMain(m *testing.M) { os.Exit(result) } -func TestAddBlock(t *testing.T) { +func TestBlocks(t *testing.T) { var fromBlock, toBlock uint64 fromBlock = 1 toBlock = 5 @@ -48,17 +49,83 @@ func TestAddBlock(t *testing.T) { // Generate fake blocks blocks := genBlocks(fromBlock, toBlock) // Insert blocks into DB - err := historyDB.addBlocks(blocks) - assert.NoError(t, err) - // Get blocks from DB + for _, block := range blocks { + err := historyDB.AddBlock(&block) + assert.NoError(t, err) + } + // Get all blocks from DB fetchedBlocks, err := historyDB.GetBlocks(fromBlock, toBlock) + assert.Equal(t, len(blocks), len(fetchedBlocks)) // Compare generated vs getted blocks assert.NoError(t, err) for i, fetchedBlock := range fetchedBlocks { - assert.Equal(t, blocks[i].EthBlockNum, fetchedBlock.EthBlockNum) - assert.Equal(t, blocks[i].Hash, fetchedBlock.Hash) - assert.Equal(t, blocks[i].Timestamp.Unix(), fetchedBlock.Timestamp.Unix()) + assertEqualBlock(t, &blocks[i], fetchedBlock) + } + // Get blocks from the DB one by one + for i := fromBlock; i < toBlock; i++ { + fetchedBlock, err := historyDB.GetBlock(i) + assert.NoError(t, err) + assertEqualBlock(t, &blocks[i-1], fetchedBlock) + } + // Get last block + lastBlock, err := historyDB.GetLastBlock() + assert.NoError(t, err) + assertEqualBlock(t, &blocks[len(blocks)-1], lastBlock) +} + +func assertEqualBlock(t *testing.T, expected *common.Block, actual *common.Block) { + assert.Equal(t, expected.EthBlockNum, actual.EthBlockNum) + assert.Equal(t, expected.Hash, actual.Hash) + assert.Equal(t, expected.Timestamp.Unix(), actual.Timestamp.Unix()) +} + +func TestBatches(t *testing.T) { + const fromBlock uint64 = 1 + const toBlock uint64 = 3 + const nBatchesPerBlock = 3 + // Prepare blocks in the DB + setTestBlocks(fromBlock, toBlock) + // Generate fake batches + var batches []common.Batch + collectedFees := make(map[common.TokenID]*big.Int) + for i := 0; i < 64; i++ { + collectedFees[common.TokenID(i)] = big.NewInt(int64(i)) } + for i := fromBlock; i < toBlock; i++ { + for j := 0; j < nBatchesPerBlock; j++ { + batch := common.Batch{ + BatchNum: common.BatchNum(int(i-1)*nBatchesPerBlock + j), + EthBlockNum: uint64(i), + ForgerAddr: eth.BigToAddress(big.NewInt(239457111187)), + CollectedFees: collectedFees, + StateRoot: common.Hash([]byte("duhdqlwiucgwqeiu")), + NumAccounts: j, + ExitRoot: common.Hash([]byte("tykertheuhtgenuer3iuw3b")), + SlotNum: common.SlotNum(j), + } + if j%2 == 0 { + batch.ForgeL1TxsNum = uint32(i) + } + batches = append(batches, batch) + } + } + // Add batches to the DB + err := historyDB.addBatches(batches) + assert.NoError(t, err) + // Get batches from the DB + fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(int(toBlock-fromBlock)*nBatchesPerBlock)) + assert.NoError(t, err) + for i, fetchedBatch := range fetchedBatches { + assert.Equal(t, batches[i], *fetchedBatch) + } + // Test GetLastBatchNum + fetchedLastBatchNum, err := historyDB.GetLastBatchNum() + assert.NoError(t, err) + assert.Equal(t, batches[len(batches)-1].BatchNum, fetchedLastBatchNum) + // Test GetLastL1TxsNum + fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum() + assert.NoError(t, err) + assert.Equal(t, batches[len(batches)-1-(int(toBlock-fromBlock+1)%nBatchesPerBlock)].ForgeL1TxsNum, fetchedLastL1TxsNum) } func TestBids(t *testing.T) { @@ -82,14 +149,16 @@ func TestBids(t *testing.T) { err := historyDB.addBids(bids) assert.NoError(t, err) // Fetch bids - fetchedBidsPtr, err := historyDB.GetBidsByBlock(fromBlock, toBlock) - assert.NoError(t, err) + var fetchedBids []*common.Bid + for i := fromBlock; i < toBlock; i++ { + fetchedBidsSlot, err := historyDB.GetBidsBySlot(common.SlotNum(i)) + assert.NoError(t, err) + fetchedBids = append(fetchedBids, fetchedBidsSlot...) + } // Compare fetched bids vs generated bids - fetchedBids := make([]common.Bid, 0, (toBlock-fromBlock)*bidsPerSlot) - for _, bid := range fetchedBidsPtr { - fetchedBids = append(fetchedBids, *bid) + for i, bid := range fetchedBids { + assert.Equal(t, bids[i], *bid) } - assert.Equal(t, bids, fetchedBids) } // setTestBlocks WARNING: this will delete the blocks and recreate them @@ -104,7 +173,7 @@ func setTestBlocks(from, to uint64) { } } blocks := genBlocks(from, to) - if err := historyDB.addBlocks(blocks); err != nil { + if err := addBlocks(blocks); err != nil { panic(err) } } @@ -120,3 +189,12 @@ func genBlocks(from, to uint64) []common.Block { } return blocks } + +// addBlocks insert blocks into the DB. TODO: move method to test +func addBlocks(blocks []common.Block) error { + return db.BulkInsert( + historyDB.db, + "INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s", + blocks[:], + ) +} diff --git a/db/historydb/migrations/001_init.sql b/db/historydb/migrations/001_init.sql index 9a33d8d..c5d255f 100644 --- a/db/historydb/migrations/001_init.sql +++ b/db/historydb/migrations/001_init.sql @@ -8,7 +8,6 @@ CREATE TABLE block ( CREATE TABLE coordianator ( forger_addr BYTEA NOT NULL, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, - beneficiary_addr BYTEA NOT NULL, withdraw_addr BYTEA NOT NULL, url VARCHAR(200) NOT NULL, PRIMARY KEY (forger_addr, eth_block_num) diff --git a/go.sum b/go.sum index 6230cf6..d9fa081 100644 --- a/go.sum +++ b/go.sum @@ -544,6 +544,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=