Impl historyDB methods for sync main loop

common/batch.go

@@ -10,20 +10,15 @@ import (
 
 // Batch is a struct that represents Hermez network batch
 type Batch struct {
-	BatchNum         BatchNum
-	SlotNum          SlotNum // Slot in which the batch is forged
-	EthTxHash        ethCommon.Hash
-	EthBlockNum      uint64 // Ethereum block in which the batch is forged
-	ExitRoot         Hash
-	OldStateRoot     Hash
-	NewStateRoot     Hash
-	OldNumAccounts   int
-	NewNumAccounts   int
-	ToForgeL1TxsNum  uint32         // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch.
-	ToForgeL1TxsHash ethCommon.Hash // optional, Only when the batch forges L1 txs. Frozen from pendingL1TxsHash (which are the group of L1UserTxs), to be forged in ToForgeL1TxsNum + 1.
-	ForgedL1TxsHash  ethCommon.Hash // optional, Only when the batch forges L1 txs. This will be the Hash of the group of L1 txs (L1UserTxs + L1CoordinatorTx) forged in the current batch.
-	CollectedFees    map[TokenID]*big.Int
-	ForgerAddr       ethCommon.Address // TODO: Should this be retrieved via slot reference?
+	BatchNum      BatchNum             `meddler:"batch_num"`
+	EthBlockNum   uint64               `meddler:"eth_block_num"` // Ethereum block in which the batch is forged
+	ForgerAddr    ethCommon.Address    `meddler:"forger_addr"`   // TODO: Should this be retrieved via slot reference?
+	CollectedFees map[TokenID]*big.Int `meddler:"fees_collected,json"`
+	StateRoot     Hash                 `meddler:"state_root"`
+	NumAccounts   int                  `meddler:"num_accounts"`
+	ExitRoot      Hash                 `meddler:"exit_root"`
+	ForgeL1TxsNum uint32               `meddler:"forge_l1_txs_num"` // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch.
+	SlotNum       SlotNum              `meddler:"slot_num"`         // Slot in which the batch is forged
 }
 
 // BatchNum identifies a batch

common/scvars.go

@@ -15,7 +15,7 @@ type RollupVars struct {
 	Governance     eth.Address
 }
 
-type PoDVars struct {
+type AuctionVars struct {
 	EthBlockNum       uint64
 	SlotDeadline      uint
 	CloseAuctionSlots uint

db/historydb/historydb.go

@@ -41,22 +41,18 @@ func NewHistoryDB(port int, host, user, password, dbname string) (*HistoryDB, er
 }
 
 // AddBlock insert a block into the DB
-func (hdb *HistoryDB) AddBlock(blocks *common.Block) error {
-	return nil
-}
-
-// addBlocks insert blocks into the DB. TODO: move method to test
-func (hdb *HistoryDB) addBlocks(blocks []common.Block) error {
-	return db.BulkInsert(
-		hdb.db,
-		"INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s",
-		blocks[:],
-	)
+func (hdb *HistoryDB) AddBlock(block *common.Block) error {
+	return meddler.Insert(hdb.db, "block", block)
 }
 
 // GetBlock retrieve a block from the DB, given a block number
 func (hdb *HistoryDB) GetBlock(blockNum uint64) (*common.Block, error) {
-	return nil, nil
+	block := &common.Block{}
+	err := meddler.QueryRow(
+		hdb.db, block,
+		"SELECT * FROM block WHERE eth_block_num = $1;", blockNum,
+	)
+	return block, err
 }
 
 // GetBlocks retrieve blocks from the DB, given a range of block numbers defined by from and to
@@ -72,16 +68,58 @@ func (hdb *HistoryDB) GetBlocks(from, to uint64) ([]*common.Block, error) {
 
 // GetLastBlock retrieve the block with the highest block number from the DB
 func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) {
-	return nil, nil
+	block := &common.Block{}
+	err := meddler.QueryRow(
+		hdb.db, block, "SELECT * FROM block ORDER BY eth_block_num DESC LIMIT 1;",
+	)
+	return block, err
+}
+
+// addBatches insert Bids into the DB
+func (hdb *HistoryDB) addBatches(batches []common.Batch) error {
+	return db.BulkInsert(
+		hdb.db,
+		`INSERT INTO batch (
+			batch_num,
+			eth_block_num,
+			forger_addr,
+			fees_collected,
+			state_root,
+			num_accounts,
+			exit_root,
+			forge_l1_txs_num,
+			slot_num
+		) VALUES %s;`,
+		batches[:],
+	)
+}
+
+// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to
+func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]*common.Batch, error) {
+	var batches []*common.Batch
+	err := meddler.QueryAll(
+		hdb.db, &batches,
+		"SELECT * FROM batch WHERE $1 <= batch_num AND batch_num < $2",
+		from, to,
+	)
+	return batches, err
 }
 
 // GetLastBatchNum returns the BatchNum of the latest forged batch
-func (hdb *HistoryDB) GetLastBatchNum() (*common.BatchNum, error) {
-	return nil, nil
+func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) {
+	row := hdb.db.QueryRow("SELECT batch_num FROM batch ORDER BY batch_num DESC LIMIT 1;")
+	var batchNum common.BatchNum
+	return batchNum, row.Scan(&batchNum)
+}
+
+// GetLastL1TxsNum returns the greatest ForgeL1TxsNum in the DB
+func (hdb *HistoryDB) GetLastL1TxsNum() (uint32, error) {
+	row := hdb.db.QueryRow("SELECT MAX(forge_l1_txs_num) FROM batch;")
+	var lastL1TxsNum uint32
+	return lastL1TxsNum, row.Scan(&lastL1TxsNum)
 }
 
 // Reorg deletes all the information that was added into the DB after the lastValidBlock
-// WARNING: this is a draaft of the function, useful at the moment for tests
 func (hdb *HistoryDB) Reorg(lastValidBlock uint64) error {
 	_, err := hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock)
 	return err
@@ -93,12 +131,16 @@ func (hdb *HistoryDB) SyncRollup(
 	l1txs []common.L1Tx,
 	l2txs []common.L2Tx,
 	registeredAccounts []common.Account,
-	exitTree common.ExitTreeLeaf,
-	withdrawals common.ExitTreeLeaf,
+	exitTree common.ExitInfo,
+	withdrawals common.ExitInfo,
 	registeredTokens []common.Token,
-	batch *common.Batch,
+	batches []common.Batch,
 	vars *common.RollupVars,
 ) error {
+	// TODO: make all in a single DB commit
+	if err := hdb.addBatches(batches); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -107,7 +149,7 @@ func (hdb *HistoryDB) SyncPoD(
 	blockNum uint64,
 	bids []common.Bid,
 	coordinators []common.Coordinator,
-	vars *common.PoDVars,
+	vars *common.AuctionVars,
 ) error {
 	return nil
 }
@@ -122,13 +164,13 @@ func (hdb *HistoryDB) addBids(bids []common.Bid) error {
 	)
 }
 
-// GetBidsByBlock return the bids done between the block from and to
-func (hdb *HistoryDB) GetBidsByBlock(from, to uint64) ([]*common.Bid, error) {
+// GetBidsBySlot return the bids for a specific slot
+func (hdb *HistoryDB) GetBidsBySlot(slotNum common.SlotNum) ([]*common.Bid, error) {
 	var bids []*common.Bid
 	err := meddler.QueryAll(
 		hdb.db, &bids,
-		"SELECT * FROM bid WHERE $1 <= eth_block_num AND eth_block_num < $2",
-		from, to,
+		"SELECT * FROM bid WHERE $1 = slot_num;",
+		slotNum,
 	)
 	return bids, err
 }

db/historydb/historydb_test.go

@@ -9,6 +9,7 @@ import (
 
 	eth "github.com/ethereum/go-ethereum/common"
 	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/hermeznetwork/hermez-node/db"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -39,7 +40,7 @@ func TestMain(m *testing.M) {
 	os.Exit(result)
 }
 
-func TestAddBlock(t *testing.T) {
+func TestBlocks(t *testing.T) {
 	var fromBlock, toBlock uint64
 	fromBlock = 1
 	toBlock = 5
@@ -48,17 +49,83 @@ func TestAddBlock(t *testing.T) {
 	// Generate fake blocks
 	blocks := genBlocks(fromBlock, toBlock)
 	// Insert blocks into DB
-	err := historyDB.addBlocks(blocks)
-	assert.NoError(t, err)
-	// Get blocks from DB
+	for _, block := range blocks {
+		err := historyDB.AddBlock(&block)
+		assert.NoError(t, err)
+	}
+	// Get all blocks from DB
 	fetchedBlocks, err := historyDB.GetBlocks(fromBlock, toBlock)
+	assert.Equal(t, len(blocks), len(fetchedBlocks))
 	// Compare generated vs getted blocks
 	assert.NoError(t, err)
 	for i, fetchedBlock := range fetchedBlocks {
-		assert.Equal(t, blocks[i].EthBlockNum, fetchedBlock.EthBlockNum)
-		assert.Equal(t, blocks[i].Hash, fetchedBlock.Hash)
-		assert.Equal(t, blocks[i].Timestamp.Unix(), fetchedBlock.Timestamp.Unix())
+		assertEqualBlock(t, &blocks[i], fetchedBlock)
+	}
+	// Get blocks from the DB one by one
+	for i := fromBlock; i < toBlock; i++ {
+		fetchedBlock, err := historyDB.GetBlock(i)
+		assert.NoError(t, err)
+		assertEqualBlock(t, &blocks[i-1], fetchedBlock)
+	}
+	// Get last block
+	lastBlock, err := historyDB.GetLastBlock()
+	assert.NoError(t, err)
+	assertEqualBlock(t, &blocks[len(blocks)-1], lastBlock)
+}
+
+func assertEqualBlock(t *testing.T, expected *common.Block, actual *common.Block) {
+	assert.Equal(t, expected.EthBlockNum, actual.EthBlockNum)
+	assert.Equal(t, expected.Hash, actual.Hash)
+	assert.Equal(t, expected.Timestamp.Unix(), actual.Timestamp.Unix())
+}
+
+func TestBatches(t *testing.T) {
+	const fromBlock uint64 = 1
+	const toBlock uint64 = 3
+	const nBatchesPerBlock = 3
+	// Prepare blocks in the DB
+	setTestBlocks(fromBlock, toBlock)
+	// Generate fake batches
+	var batches []common.Batch
+	collectedFees := make(map[common.TokenID]*big.Int)
+	for i := 0; i < 64; i++ {
+		collectedFees[common.TokenID(i)] = big.NewInt(int64(i))
 	}
+	for i := fromBlock; i < toBlock; i++ {
+		for j := 0; j < nBatchesPerBlock; j++ {
+			batch := common.Batch{
+				BatchNum:      common.BatchNum(int(i-1)*nBatchesPerBlock + j),
+				EthBlockNum:   uint64(i),
+				ForgerAddr:    eth.BigToAddress(big.NewInt(239457111187)),
+				CollectedFees: collectedFees,
+				StateRoot:     common.Hash([]byte("duhdqlwiucgwqeiu")),
+				NumAccounts:   j,
+				ExitRoot:      common.Hash([]byte("tykertheuhtgenuer3iuw3b")),
+				SlotNum:       common.SlotNum(j),
+			}
+			if j%2 == 0 {
+				batch.ForgeL1TxsNum = uint32(i)
+			}
+			batches = append(batches, batch)
+		}
+	}
+	// Add batches to the DB
+	err := historyDB.addBatches(batches)
+	assert.NoError(t, err)
+	// Get batches from the DB
+	fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(int(toBlock-fromBlock)*nBatchesPerBlock))
+	assert.NoError(t, err)
+	for i, fetchedBatch := range fetchedBatches {
+		assert.Equal(t, batches[i], *fetchedBatch)
+	}
+	// Test GetLastBatchNum
+	fetchedLastBatchNum, err := historyDB.GetLastBatchNum()
+	assert.NoError(t, err)
+	assert.Equal(t, batches[len(batches)-1].BatchNum, fetchedLastBatchNum)
+	// Test GetLastL1TxsNum
+	fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum()
+	assert.NoError(t, err)
+	assert.Equal(t, batches[len(batches)-1-(int(toBlock-fromBlock+1)%nBatchesPerBlock)].ForgeL1TxsNum, fetchedLastL1TxsNum)
 }
 
 func TestBids(t *testing.T) {
@@ -82,14 +149,16 @@ func TestBids(t *testing.T) {
 	err := historyDB.addBids(bids)
 	assert.NoError(t, err)
 	// Fetch bids
-	fetchedBidsPtr, err := historyDB.GetBidsByBlock(fromBlock, toBlock)
-	assert.NoError(t, err)
+	var fetchedBids []*common.Bid
+	for i := fromBlock; i < toBlock; i++ {
+		fetchedBidsSlot, err := historyDB.GetBidsBySlot(common.SlotNum(i))
+		assert.NoError(t, err)
+		fetchedBids = append(fetchedBids, fetchedBidsSlot...)
+	}
 	// Compare fetched bids vs generated bids
-	fetchedBids := make([]common.Bid, 0, (toBlock-fromBlock)*bidsPerSlot)
-	for _, bid := range fetchedBidsPtr {
-		fetchedBids = append(fetchedBids, *bid)
+	for i, bid := range fetchedBids {
+		assert.Equal(t, bids[i], *bid)
 	}
-	assert.Equal(t, bids, fetchedBids)
 }
 
 // setTestBlocks WARNING: this will delete the blocks and recreate them
@@ -104,7 +173,7 @@ func setTestBlocks(from, to uint64) {
 		}
 	}
 	blocks := genBlocks(from, to)
-	if err := historyDB.addBlocks(blocks); err != nil {
+	if err := addBlocks(blocks); err != nil {
 		panic(err)
 	}
 }
@@ -120,3 +189,12 @@ func genBlocks(from, to uint64) []common.Block {
 	}
 	return blocks
 }
+
+// addBlocks insert blocks into the DB. TODO: move method to test
+func addBlocks(blocks []common.Block) error {
+	return db.BulkInsert(
+		historyDB.db,
+		"INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s",
+		blocks[:],
+	)
+}

db/historydb/migrations/001_init.sql

@@ -8,7 +8,6 @@ CREATE TABLE block (
 CREATE TABLE coordianator (
     forger_addr BYTEA NOT NULL,
     eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
-    beneficiary_addr BYTEA NOT NULL,
     withdraw_addr BYTEA NOT NULL,
     url VARCHAR(200) NOT NULL,
     PRIMARY KEY (forger_addr, eth_block_num)

go.sum

@@ -544,6 +544,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=