diff --git a/common/batch.go b/common/batch.go index daa3357..1eadb33 100644 --- a/common/batch.go +++ b/common/batch.go @@ -10,20 +10,15 @@ import ( // Batch is a struct that represents Hermez network batch type Batch struct { - BatchNum BatchNum - SlotNum SlotNum // Slot in which the batch is forged - EthTxHash ethCommon.Hash - EthBlockNum uint64 // Ethereum block in which the batch is forged - ExitRoot Hash - OldStateRoot Hash - NewStateRoot Hash - OldNumAccounts int - NewNumAccounts int - ToForgeL1TxsNum uint32 // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch. - ToForgeL1TxsHash ethCommon.Hash // optional, Only when the batch forges L1 txs. Frozen from pendingL1TxsHash (which are the group of L1UserTxs), to be forged in ToForgeL1TxsNum + 1. - ForgedL1TxsHash ethCommon.Hash // optional, Only when the batch forges L1 txs. This will be the Hash of the group of L1 txs (L1UserTxs + L1CoordinatorTx) forged in the current batch. - CollectedFees map[TokenID]*big.Int - ForgerAddr ethCommon.Address // TODO: Should this be retrieved via slot reference? + BatchNum BatchNum `meddler:"batch_num"` + EthBlockNum uint64 `meddler:"eth_block_num"` // Ethereum block in which the batch is forged + ForgerAddr ethCommon.Address `meddler:"forger_addr"` // TODO: Should this be retrieved via slot reference? + CollectedFees map[TokenID]*big.Int `meddler:"fees_collected,json"` + StateRoot Hash `meddler:"state_root"` + NumAccounts int `meddler:"num_accounts"` + ExitRoot Hash `meddler:"exit_root"` + ForgeL1TxsNum uint32 `meddler:"forge_l1_txs_num"` // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch. + SlotNum SlotNum `meddler:"slot_num"` // Slot in which the batch is forged } // BatchNum identifies a batch diff --git a/common/exittree.go b/common/exittree.go new file mode 100644 index 0000000..201d3da --- /dev/null +++ b/common/exittree.go @@ -0,0 +1,12 @@ +package common + +import ( + "math/big" +) + +type ExitInfo struct { + AccountIdx Idx + MerkleProof []byte + Balance *big.Int + Nullifier *big.Int +} diff --git a/common/l1tx.go b/common/l1tx.go index f6cdd47..49ee142 100644 --- a/common/l1tx.go +++ b/common/l1tx.go @@ -22,8 +22,7 @@ type L1Tx struct { Amount *big.Int `meddler:"amount,bigint"` LoadAmount *big.Int `meddler:"load_amount,bigint"` EthBlockNum uint64 `meddler:"eth_block_num"` // Ethereum Block Number in which this L1Tx was added to the queue - // Extra metadata, may be uninitialized - Type TxType `meddler:"-"` // optional, descrives which kind of tx it's + Type TxType `meddler:"tx_type"` } func (tx *L1Tx) Tx() *Tx { @@ -32,8 +31,6 @@ func (tx *L1Tx) Tx() *Tx { FromIdx: tx.FromIdx, ToIdx: tx.ToIdx, Amount: tx.Amount, - Nonce: 0, - Fee: 0, Type: tx.Type, } } diff --git a/common/l2tx.go b/common/l2tx.go index c61040d..5690a2c 100644 --- a/common/l2tx.go +++ b/common/l2tx.go @@ -15,8 +15,7 @@ type L2Tx struct { Amount *big.Int `meddler:"amount,bigint"` Fee FeeSelector `meddler:"fee"` Nonce Nonce `meddler:"nonce"` - // Extra metadata, may be uninitialized - Type TxType `meddler:"-"` // optional, descrives which kind of tx it's + Type TxType `meddler:"tx_type"` } func (tx *L2Tx) Tx() *Tx { diff --git a/common/pooll2tx.go b/common/pooll2tx.go index 1180f05..355ad6d 100644 --- a/common/pooll2tx.go +++ b/common/pooll2tx.go @@ -62,8 +62,8 @@ type PoolL2Tx struct { RqNonce uint64 `meddler:"rq_nonce,zeroisnull"` // effective 48 bits used AbsoluteFee float64 `meddler:"absolute_fee,zeroisnull"` AbsoluteFeeUpdate time.Time `meddler:"absolute_fee_update,utctimez"` + Type TxType `meddler:"tx_type"` // Extra metadata, may be uninitialized - Type TxType `meddler:"-"` // optional, descrives which kind of tx it's RqTxCompressedData []byte `meddler:"-"` // 253 bits, optional for atomic txs } diff --git a/common/scvars.go b/common/scvars.go new file mode 100644 index 0000000..415de48 --- /dev/null +++ b/common/scvars.go @@ -0,0 +1,37 @@ +package common + +import ( + "math/big" + + eth "github.com/ethereum/go-ethereum/common" +) + +type RollupVars struct { + EthBlockNum uint64 + ForgeL1Timeout *big.Int + FeeL1UserTx *big.Int + FeeAddToken *big.Int + TokensHEZ eth.Address + Governance eth.Address +} + +type AuctionVars struct { + EthBlockNum uint64 + SlotDeadline uint + CloseAuctionSlots uint + OpenAuctionSlots uint + Governance eth.Address + MinBidSlots MinBidSlots + Outbidding int + DonationAddress eth.Address + GovernanceAddress eth.Address + AllocationRatio AllocationRatio +} + +type MinBidSlots [6]uint + +type AllocationRatio struct { + Donation uint + Burn uint + Forger uint +} diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index 1ad962e..33f23cf 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -40,16 +40,22 @@ func NewHistoryDB(port int, host, user, password, dbname string) (*HistoryDB, er return &HistoryDB{hdb}, nil } -// addBlocks insert blocks into the DB -func (hdb *HistoryDB) addBlocks(blocks []common.Block) error { - return db.BulkInsert( - hdb.db, - "INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s", - blocks[:], +// AddBlock insert a block into the DB +func (hdb *HistoryDB) AddBlock(block *common.Block) error { + return meddler.Insert(hdb.db, "block", block) +} + +// GetBlock retrieve a block from the DB, given a block number +func (hdb *HistoryDB) GetBlock(blockNum uint64) (*common.Block, error) { + block := &common.Block{} + err := meddler.QueryRow( + hdb.db, block, + "SELECT * FROM block WHERE eth_block_num = $1;", blockNum, ) + return block, err } -// GetBlocks retrrieve blocks from the DB +// GetBlocks retrieve blocks from the DB, given a range of block numbers defined by from and to func (hdb *HistoryDB) GetBlocks(from, to uint64) ([]*common.Block, error) { var blocks []*common.Block err := meddler.QueryAll( @@ -60,13 +66,94 @@ func (hdb *HistoryDB) GetBlocks(from, to uint64) ([]*common.Block, error) { return blocks, err } -// reorg deletes all the information that was added into the DB after the lastValidBlock -// WARNING: this is a draaft of the function, useful at the moment for tests -func (hdb *HistoryDB) reorg(lastValidBlock uint64) error { +// GetLastBlock retrieve the block with the highest block number from the DB +func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) { + block := &common.Block{} + err := meddler.QueryRow( + hdb.db, block, "SELECT * FROM block ORDER BY eth_block_num DESC LIMIT 1;", + ) + return block, err +} + +// addBatches insert Bids into the DB +func (hdb *HistoryDB) addBatches(batches []common.Batch) error { + return db.BulkInsert( + hdb.db, + `INSERT INTO batch ( + batch_num, + eth_block_num, + forger_addr, + fees_collected, + state_root, + num_accounts, + exit_root, + forge_l1_txs_num, + slot_num + ) VALUES %s;`, + batches[:], + ) +} + +// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to +func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]*common.Batch, error) { + var batches []*common.Batch + err := meddler.QueryAll( + hdb.db, &batches, + "SELECT * FROM batch WHERE $1 <= batch_num AND batch_num < $2", + from, to, + ) + return batches, err +} + +// GetLastBatchNum returns the BatchNum of the latest forged batch +func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) { + row := hdb.db.QueryRow("SELECT batch_num FROM batch ORDER BY batch_num DESC LIMIT 1;") + var batchNum common.BatchNum + return batchNum, row.Scan(&batchNum) +} + +// GetLastL1TxsNum returns the greatest ForgeL1TxsNum in the DB +func (hdb *HistoryDB) GetLastL1TxsNum() (uint32, error) { + row := hdb.db.QueryRow("SELECT MAX(forge_l1_txs_num) FROM batch;") + var lastL1TxsNum uint32 + return lastL1TxsNum, row.Scan(&lastL1TxsNum) +} + +// Reorg deletes all the information that was added into the DB after the lastValidBlock +func (hdb *HistoryDB) Reorg(lastValidBlock uint64) error { _, err := hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock) return err } +// SyncRollup stores all the data that can be changed / added on a block in the Rollup SC +func (hdb *HistoryDB) SyncRollup( + blockNum uint64, + l1txs []common.L1Tx, + l2txs []common.L2Tx, + registeredAccounts []common.Account, + exitTree common.ExitInfo, + withdrawals common.ExitInfo, + registeredTokens []common.Token, + batches []common.Batch, + vars *common.RollupVars, +) error { + // TODO: make all in a single DB commit + if err := hdb.addBatches(batches); err != nil { + return err + } + return nil +} + +// SyncPoD stores all the data that can be changed / added on a block in the PoD SC +func (hdb *HistoryDB) SyncPoD( + blockNum uint64, + bids []common.Bid, + coordinators []common.Coordinator, + vars *common.AuctionVars, +) error { + return nil +} + // addBids insert Bids into the DB func (hdb *HistoryDB) addBids(bids []common.Bid) error { // TODO: check the coordinator info @@ -77,13 +164,13 @@ func (hdb *HistoryDB) addBids(bids []common.Bid) error { ) } -// GetBidsByBlock return the bids done between the block from and to -func (hdb *HistoryDB) GetBidsByBlock(from, to uint64) ([]*common.Bid, error) { +// GetBidsBySlot return the bids for a specific slot +func (hdb *HistoryDB) GetBidsBySlot(slotNum common.SlotNum) ([]*common.Bid, error) { var bids []*common.Bid err := meddler.QueryAll( hdb.db, &bids, - "SELECT * FROM bid WHERE $1 <= eth_block_num AND eth_block_num < $2", - from, to, + "SELECT * FROM bid WHERE $1 = slot_num;", + slotNum, ) return bids, err } diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index 1e166e8..c662c52 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -9,6 +9,7 @@ import ( eth "github.com/ethereum/go-ethereum/common" "github.com/hermeznetwork/hermez-node/common" + "github.com/hermeznetwork/hermez-node/db" "github.com/stretchr/testify/assert" ) @@ -39,26 +40,92 @@ func TestMain(m *testing.M) { os.Exit(result) } -func TestAddBlock(t *testing.T) { +func TestBlocks(t *testing.T) { var fromBlock, toBlock uint64 fromBlock = 1 toBlock = 5 // Delete peviously created rows (clean previous test execs) - assert.NoError(t, historyDB.reorg(fromBlock-1)) + assert.NoError(t, historyDB.Reorg(fromBlock-1)) // Generate fake blocks blocks := genBlocks(fromBlock, toBlock) // Insert blocks into DB - err := historyDB.addBlocks(blocks) - assert.NoError(t, err) - // Get blocks from DB + for _, block := range blocks { + err := historyDB.AddBlock(&block) + assert.NoError(t, err) + } + // Get all blocks from DB fetchedBlocks, err := historyDB.GetBlocks(fromBlock, toBlock) + assert.Equal(t, len(blocks), len(fetchedBlocks)) // Compare generated vs getted blocks assert.NoError(t, err) for i, fetchedBlock := range fetchedBlocks { - assert.Equal(t, blocks[i].EthBlockNum, fetchedBlock.EthBlockNum) - assert.Equal(t, blocks[i].Hash, fetchedBlock.Hash) - assert.Equal(t, blocks[i].Timestamp.Unix(), fetchedBlock.Timestamp.Unix()) + assertEqualBlock(t, &blocks[i], fetchedBlock) + } + // Get blocks from the DB one by one + for i := fromBlock; i < toBlock; i++ { + fetchedBlock, err := historyDB.GetBlock(i) + assert.NoError(t, err) + assertEqualBlock(t, &blocks[i-1], fetchedBlock) + } + // Get last block + lastBlock, err := historyDB.GetLastBlock() + assert.NoError(t, err) + assertEqualBlock(t, &blocks[len(blocks)-1], lastBlock) +} + +func assertEqualBlock(t *testing.T, expected *common.Block, actual *common.Block) { + assert.Equal(t, expected.EthBlockNum, actual.EthBlockNum) + assert.Equal(t, expected.Hash, actual.Hash) + assert.Equal(t, expected.Timestamp.Unix(), actual.Timestamp.Unix()) +} + +func TestBatches(t *testing.T) { + const fromBlock uint64 = 1 + const toBlock uint64 = 3 + const nBatchesPerBlock = 3 + // Prepare blocks in the DB + setTestBlocks(fromBlock, toBlock) + // Generate fake batches + var batches []common.Batch + collectedFees := make(map[common.TokenID]*big.Int) + for i := 0; i < 64; i++ { + collectedFees[common.TokenID(i)] = big.NewInt(int64(i)) } + for i := fromBlock; i < toBlock; i++ { + for j := 0; j < nBatchesPerBlock; j++ { + batch := common.Batch{ + BatchNum: common.BatchNum(int(i-1)*nBatchesPerBlock + j), + EthBlockNum: uint64(i), + ForgerAddr: eth.BigToAddress(big.NewInt(239457111187)), + CollectedFees: collectedFees, + StateRoot: common.Hash([]byte("duhdqlwiucgwqeiu")), + NumAccounts: j, + ExitRoot: common.Hash([]byte("tykertheuhtgenuer3iuw3b")), + SlotNum: common.SlotNum(j), + } + if j%2 == 0 { + batch.ForgeL1TxsNum = uint32(i) + } + batches = append(batches, batch) + } + } + // Add batches to the DB + err := historyDB.addBatches(batches) + assert.NoError(t, err) + // Get batches from the DB + fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(int(toBlock-fromBlock)*nBatchesPerBlock)) + assert.NoError(t, err) + for i, fetchedBatch := range fetchedBatches { + assert.Equal(t, batches[i], *fetchedBatch) + } + // Test GetLastBatchNum + fetchedLastBatchNum, err := historyDB.GetLastBatchNum() + assert.NoError(t, err) + assert.Equal(t, batches[len(batches)-1].BatchNum, fetchedLastBatchNum) + // Test GetLastL1TxsNum + fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum() + assert.NoError(t, err) + assert.Equal(t, batches[len(batches)-1-(int(toBlock-fromBlock+1)%nBatchesPerBlock)].ForgeL1TxsNum, fetchedLastL1TxsNum) } func TestBids(t *testing.T) { @@ -82,29 +149,31 @@ func TestBids(t *testing.T) { err := historyDB.addBids(bids) assert.NoError(t, err) // Fetch bids - fetchedBidsPtr, err := historyDB.GetBidsByBlock(fromBlock, toBlock) - assert.NoError(t, err) + var fetchedBids []*common.Bid + for i := fromBlock; i < toBlock; i++ { + fetchedBidsSlot, err := historyDB.GetBidsBySlot(common.SlotNum(i)) + assert.NoError(t, err) + fetchedBids = append(fetchedBids, fetchedBidsSlot...) + } // Compare fetched bids vs generated bids - fetchedBids := make([]common.Bid, 0, (toBlock-fromBlock)*bidsPerSlot) - for _, bid := range fetchedBidsPtr { - fetchedBids = append(fetchedBids, *bid) + for i, bid := range fetchedBids { + assert.Equal(t, bids[i], *bid) } - assert.Equal(t, bids, fetchedBids) } // setTestBlocks WARNING: this will delete the blocks and recreate them func setTestBlocks(from, to uint64) { if from == 0 { - if err := historyDB.reorg(from); err != nil { + if err := historyDB.Reorg(from); err != nil { panic(err) } } else { - if err := historyDB.reorg(from - 1); err != nil { + if err := historyDB.Reorg(from - 1); err != nil { panic(err) } } blocks := genBlocks(from, to) - if err := historyDB.addBlocks(blocks); err != nil { + if err := addBlocks(blocks); err != nil { panic(err) } } @@ -120,3 +189,12 @@ func genBlocks(from, to uint64) []common.Block { } return blocks } + +// addBlocks insert blocks into the DB. TODO: move method to test +func addBlocks(blocks []common.Block) error { + return db.BulkInsert( + historyDB.db, + "INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s", + blocks[:], + ) +} diff --git a/db/historydb/migrations/001_init.sql b/db/historydb/migrations/001_init.sql index ee1fcba..c5d255f 100644 --- a/db/historydb/migrations/001_init.sql +++ b/db/historydb/migrations/001_init.sql @@ -5,15 +5,9 @@ CREATE TABLE block ( hash BYTEA NOT NULL ); -CREATE TABLE slot_min_prices ( - eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE, - min_prices VARCHAR(200) NOT NULL -); - CREATE TABLE coordianator ( forger_addr BYTEA NOT NULL, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, - beneficiary_addr BYTEA NOT NULL, withdraw_addr BYTEA NOT NULL, url VARCHAR(200) NOT NULL, PRIMARY KEY (forger_addr, eth_block_num) @@ -32,14 +26,22 @@ CREATE TABLE batch ( ); CREATE TABLE exit_tree ( - batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE, - account_idx BIGINT NOT NULL, + batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE, + account_idx BIGINT, merkle_proof BYTEA NOT NULL, - amount NUMERIC NOT NULL, + balance NUMERIC NOT NULL, nullifier BYTEA NOT NULL, PRIMARY KEY (batch_num, account_idx) ); +CREATE TABLE withdrawal ( + batch_num BIGINT, + account_idx BIGINT, + eth_block_num BIGINT REFERENCES block (eth_block_num) ON DELETE CASCADE, + FOREIGN KEY (batch_num, account_idx) REFERENCES exit_tree (batch_num, account_idx) ON DELETE CASCADE, + PRIMARY KEY (batch_num, account_idx) +); + CREATE TABLE bid ( slot_num BIGINT NOT NULL, bid_value BYTEA NOT NULL, -- (check if we can do a max(), if not add float for order purposes) @@ -69,7 +71,8 @@ CREATE TABLE l1tx ( token_id INT NOT NULL REFERENCES token (token_id), amount NUMERIC NOT NULL, load_amount BYTEA NOT NULL, - eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE + eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, + tx_type VARCHAR(40) NOT NULL ); CREATE TABLE l2tx ( @@ -80,7 +83,8 @@ CREATE TABLE l2tx ( to_idx BIGINT NOT NULL, amount NUMERIC NOT NULL, fee INT NOT NULL, - nonce BIGINT NOT NULL + nonce BIGINT NOT NULL, + tx_type VARCHAR(40) NOT NULL ); CREATE TABLE account ( @@ -91,7 +95,30 @@ CREATE TABLE account ( eth_addr BYTEA NOT NULL ); +CREATE TABLE rollup_vars ( + eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE, + forge_l1_timeout BYTEA NOT NULL, + fee_l1_user_tx BYTEA NOT NULL, + fee_add_token BYTEA NOT NULL, + tokens_hez BYTEA NOT NULL, + governance BYTEA NOT NULL +); + +CREATE TABLE consensus_vars ( + eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE, + slot_deadline INT NOT NULL, + close_auction_slots INT NOT NULL, + open_auction_slots INT NOT NULL, + min_bid_slots VARCHAR(200) NOT NULL, + outbidding INT NOT NULL, + donation_address BYTEA NOT NULL, + governance_address BYTEA NOT NULL, + allocation_ratio vARCHAR(200) +); + -- +migrate Down +DROP TABLE consensus_vars; +DROP TABLE rollup_vars; DROP TABLE account; DROP TABLE l2tx; DROP TABLE l1tx; @@ -100,5 +127,4 @@ DROP TABLE bid; DROP TABLE exit_tree; DROP TABLE batch; DROP TABLE coordianator; -DROP TABLE slot_min_prices; DROP TABLE block; \ No newline at end of file diff --git a/db/l2db/migrations/001_init.sql b/db/l2db/migrations/001_init.sql index 2e6dab1..80ac665 100644 --- a/db/l2db/migrations/001_init.sql +++ b/db/l2db/migrations/001_init.sql @@ -22,7 +22,8 @@ CREATE TABLE tx_pool ( signature BYTEA NOT NULL, timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, absolute_fee NUMERIC, - absolute_fee_update TIMESTAMP WITHOUT TIME ZONE + absolute_fee_update TIMESTAMP WITHOUT TIME ZONE, + tx_type VARCHAR(40) NOT NULL ); CREATE TABLE account_creation_auth ( diff --git a/go.sum b/go.sum index 6230cf6..d9fa081 100644 --- a/go.sum +++ b/go.sum @@ -544,6 +544,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=