Browse Source

Merge pull request #67 from hermeznetwork/feature/sql-sc-vars

Add sync interface
feature/sql-semaphore1
Eduardo Antuña Díez 4 years ago
committed by GitHub
parent
commit
361af765ab
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 298 additions and 65 deletions
  1. +9
    -14
      common/batch.go
  2. +12
    -0
      common/exittree.go
  3. +1
    -4
      common/l1tx.go
  4. +1
    -2
      common/l2tx.go
  5. +1
    -1
      common/pooll2tx.go
  6. +37
    -0
      common/scvars.go
  7. +101
    -14
      db/historydb/historydb.go
  8. +95
    -17
      db/historydb/historydb_test.go
  9. +38
    -12
      db/historydb/migrations/001_init.sql
  10. +2
    -1
      db/l2db/migrations/001_init.sql
  11. +1
    -0
      go.sum

+ 9
- 14
common/batch.go

@ -10,20 +10,15 @@ import (
// Batch is a struct that represents Hermez network batch
type Batch struct {
BatchNum BatchNum
SlotNum SlotNum // Slot in which the batch is forged
EthTxHash ethCommon.Hash
EthBlockNum uint64 // Ethereum block in which the batch is forged
ExitRoot Hash
OldStateRoot Hash
NewStateRoot Hash
OldNumAccounts int
NewNumAccounts int
ToForgeL1TxsNum uint32 // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch.
ToForgeL1TxsHash ethCommon.Hash // optional, Only when the batch forges L1 txs. Frozen from pendingL1TxsHash (which are the group of L1UserTxs), to be forged in ToForgeL1TxsNum + 1.
ForgedL1TxsHash ethCommon.Hash // optional, Only when the batch forges L1 txs. This will be the Hash of the group of L1 txs (L1UserTxs + L1CoordinatorTx) forged in the current batch.
CollectedFees map[TokenID]*big.Int
ForgerAddr ethCommon.Address // TODO: Should this be retrieved via slot reference?
BatchNum BatchNum `meddler:"batch_num"`
EthBlockNum uint64 `meddler:"eth_block_num"` // Ethereum block in which the batch is forged
ForgerAddr ethCommon.Address `meddler:"forger_addr"` // TODO: Should this be retrieved via slot reference?
CollectedFees map[TokenID]*big.Int `meddler:"fees_collected,json"`
StateRoot Hash `meddler:"state_root"`
NumAccounts int `meddler:"num_accounts"`
ExitRoot Hash `meddler:"exit_root"`
ForgeL1TxsNum uint32 `meddler:"forge_l1_txs_num"` // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch.
SlotNum SlotNum `meddler:"slot_num"` // Slot in which the batch is forged
}
// BatchNum identifies a batch

+ 12
- 0
common/exittree.go

@ -0,0 +1,12 @@
package common
import (
"math/big"
)
type ExitInfo struct {
AccountIdx Idx
MerkleProof []byte
Balance *big.Int
Nullifier *big.Int
}

+ 1
- 4
common/l1tx.go

@ -22,8 +22,7 @@ type L1Tx struct {
Amount *big.Int `meddler:"amount,bigint"`
LoadAmount *big.Int `meddler:"load_amount,bigint"`
EthBlockNum uint64 `meddler:"eth_block_num"` // Ethereum Block Number in which this L1Tx was added to the queue
// Extra metadata, may be uninitialized
Type TxType `meddler:"-"` // optional, descrives which kind of tx it's
Type TxType `meddler:"tx_type"`
}
func (tx *L1Tx) Tx() *Tx {
@ -32,8 +31,6 @@ func (tx *L1Tx) Tx() *Tx {
FromIdx: tx.FromIdx,
ToIdx: tx.ToIdx,
Amount: tx.Amount,
Nonce: 0,
Fee: 0,
Type: tx.Type,
}
}

+ 1
- 2
common/l2tx.go

@ -15,8 +15,7 @@ type L2Tx struct {
Amount *big.Int `meddler:"amount,bigint"`
Fee FeeSelector `meddler:"fee"`
Nonce Nonce `meddler:"nonce"`
// Extra metadata, may be uninitialized
Type TxType `meddler:"-"` // optional, descrives which kind of tx it's
Type TxType `meddler:"tx_type"`
}
func (tx *L2Tx) Tx() *Tx {

+ 1
- 1
common/pooll2tx.go

@ -62,8 +62,8 @@ type PoolL2Tx struct {
RqNonce uint64 `meddler:"rq_nonce,zeroisnull"` // effective 48 bits used
AbsoluteFee float64 `meddler:"absolute_fee,zeroisnull"`
AbsoluteFeeUpdate time.Time `meddler:"absolute_fee_update,utctimez"`
Type TxType `meddler:"tx_type"`
// Extra metadata, may be uninitialized
Type TxType `meddler:"-"` // optional, descrives which kind of tx it's
RqTxCompressedData []byte `meddler:"-"` // 253 bits, optional for atomic txs
}

+ 37
- 0
common/scvars.go

@ -0,0 +1,37 @@
package common
import (
"math/big"
eth "github.com/ethereum/go-ethereum/common"
)
type RollupVars struct {
EthBlockNum uint64
ForgeL1Timeout *big.Int
FeeL1UserTx *big.Int
FeeAddToken *big.Int
TokensHEZ eth.Address
Governance eth.Address
}
type AuctionVars struct {
EthBlockNum uint64
SlotDeadline uint
CloseAuctionSlots uint
OpenAuctionSlots uint
Governance eth.Address
MinBidSlots MinBidSlots
Outbidding int
DonationAddress eth.Address
GovernanceAddress eth.Address
AllocationRatio AllocationRatio
}
type MinBidSlots [6]uint
type AllocationRatio struct {
Donation uint
Burn uint
Forger uint
}

+ 101
- 14
db/historydb/historydb.go

@ -40,16 +40,22 @@ func NewHistoryDB(port int, host, user, password, dbname string) (*HistoryDB, er
return &HistoryDB{hdb}, nil
}
// addBlocks insert blocks into the DB
func (hdb *HistoryDB) addBlocks(blocks []common.Block) error {
return db.BulkInsert(
hdb.db,
"INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s",
blocks[:],
// AddBlock insert a block into the DB
func (hdb *HistoryDB) AddBlock(block *common.Block) error {
return meddler.Insert(hdb.db, "block", block)
}
// GetBlock retrieve a block from the DB, given a block number
func (hdb *HistoryDB) GetBlock(blockNum uint64) (*common.Block, error) {
block := &common.Block{}
err := meddler.QueryRow(
hdb.db, block,
"SELECT * FROM block WHERE eth_block_num = $1;", blockNum,
)
return block, err
}
// GetBlocks retrrieve blocks from the DB
// GetBlocks retrieve blocks from the DB, given a range of block numbers defined by from and to
func (hdb *HistoryDB) GetBlocks(from, to uint64) ([]*common.Block, error) {
var blocks []*common.Block
err := meddler.QueryAll(
@ -60,13 +66,94 @@ func (hdb *HistoryDB) GetBlocks(from, to uint64) ([]*common.Block, error) {
return blocks, err
}
// reorg deletes all the information that was added into the DB after the lastValidBlock
// WARNING: this is a draaft of the function, useful at the moment for tests
func (hdb *HistoryDB) reorg(lastValidBlock uint64) error {
// GetLastBlock retrieve the block with the highest block number from the DB
func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) {
block := &common.Block{}
err := meddler.QueryRow(
hdb.db, block, "SELECT * FROM block ORDER BY eth_block_num DESC LIMIT 1;",
)
return block, err
}
// addBatches insert Bids into the DB
func (hdb *HistoryDB) addBatches(batches []common.Batch) error {
return db.BulkInsert(
hdb.db,
`INSERT INTO batch (
batch_num,
eth_block_num,
forger_addr,
fees_collected,
state_root,
num_accounts,
exit_root,
forge_l1_txs_num,
slot_num
) VALUES %s;`,
batches[:],
)
}
// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to
func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]*common.Batch, error) {
var batches []*common.Batch
err := meddler.QueryAll(
hdb.db, &batches,
"SELECT * FROM batch WHERE $1 <= batch_num AND batch_num < $2",
from, to,
)
return batches, err
}
// GetLastBatchNum returns the BatchNum of the latest forged batch
func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) {
row := hdb.db.QueryRow("SELECT batch_num FROM batch ORDER BY batch_num DESC LIMIT 1;")
var batchNum common.BatchNum
return batchNum, row.Scan(&batchNum)
}
// GetLastL1TxsNum returns the greatest ForgeL1TxsNum in the DB
func (hdb *HistoryDB) GetLastL1TxsNum() (uint32, error) {
row := hdb.db.QueryRow("SELECT MAX(forge_l1_txs_num) FROM batch;")
var lastL1TxsNum uint32
return lastL1TxsNum, row.Scan(&lastL1TxsNum)
}
// Reorg deletes all the information that was added into the DB after the lastValidBlock
func (hdb *HistoryDB) Reorg(lastValidBlock uint64) error {
_, err := hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock)
return err
}
// SyncRollup stores all the data that can be changed / added on a block in the Rollup SC
func (hdb *HistoryDB) SyncRollup(
blockNum uint64,
l1txs []common.L1Tx,
l2txs []common.L2Tx,
registeredAccounts []common.Account,
exitTree common.ExitInfo,
withdrawals common.ExitInfo,
registeredTokens []common.Token,
batches []common.Batch,
vars *common.RollupVars,
) error {
// TODO: make all in a single DB commit
if err := hdb.addBatches(batches); err != nil {
return err
}
return nil
}
// SyncPoD stores all the data that can be changed / added on a block in the PoD SC
func (hdb *HistoryDB) SyncPoD(
blockNum uint64,
bids []common.Bid,
coordinators []common.Coordinator,
vars *common.AuctionVars,
) error {
return nil
}
// addBids insert Bids into the DB
func (hdb *HistoryDB) addBids(bids []common.Bid) error {
// TODO: check the coordinator info
@ -77,13 +164,13 @@ func (hdb *HistoryDB) addBids(bids []common.Bid) error {
)
}
// GetBidsByBlock return the bids done between the block from and to
func (hdb *HistoryDB) GetBidsByBlock(from, to uint64) ([]*common.Bid, error) {
// GetBidsBySlot return the bids for a specific slot
func (hdb *HistoryDB) GetBidsBySlot(slotNum common.SlotNum) ([]*common.Bid, error) {
var bids []*common.Bid
err := meddler.QueryAll(
hdb.db, &bids,
"SELECT * FROM bid WHERE $1 <= eth_block_num AND eth_block_num < $2",
from, to,
"SELECT * FROM bid WHERE $1 = slot_num;",
slotNum,
)
return bids, err
}

+ 95
- 17
db/historydb/historydb_test.go

@ -9,6 +9,7 @@ import (
eth "github.com/ethereum/go-ethereum/common"
"github.com/hermeznetwork/hermez-node/common"
"github.com/hermeznetwork/hermez-node/db"
"github.com/stretchr/testify/assert"
)
@ -39,26 +40,92 @@ func TestMain(m *testing.M) {
os.Exit(result)
}
func TestAddBlock(t *testing.T) {
func TestBlocks(t *testing.T) {
var fromBlock, toBlock uint64
fromBlock = 1
toBlock = 5
// Delete peviously created rows (clean previous test execs)
assert.NoError(t, historyDB.reorg(fromBlock-1))
assert.NoError(t, historyDB.Reorg(fromBlock-1))
// Generate fake blocks
blocks := genBlocks(fromBlock, toBlock)
// Insert blocks into DB
err := historyDB.addBlocks(blocks)
assert.NoError(t, err)
// Get blocks from DB
for _, block := range blocks {
err := historyDB.AddBlock(&block)
assert.NoError(t, err)
}
// Get all blocks from DB
fetchedBlocks, err := historyDB.GetBlocks(fromBlock, toBlock)
assert.Equal(t, len(blocks), len(fetchedBlocks))
// Compare generated vs getted blocks
assert.NoError(t, err)
for i, fetchedBlock := range fetchedBlocks {
assert.Equal(t, blocks[i].EthBlockNum, fetchedBlock.EthBlockNum)
assert.Equal(t, blocks[i].Hash, fetchedBlock.Hash)
assert.Equal(t, blocks[i].Timestamp.Unix(), fetchedBlock.Timestamp.Unix())
assertEqualBlock(t, &blocks[i], fetchedBlock)
}
// Get blocks from the DB one by one
for i := fromBlock; i < toBlock; i++ {
fetchedBlock, err := historyDB.GetBlock(i)
assert.NoError(t, err)
assertEqualBlock(t, &blocks[i-1], fetchedBlock)
}
// Get last block
lastBlock, err := historyDB.GetLastBlock()
assert.NoError(t, err)
assertEqualBlock(t, &blocks[len(blocks)-1], lastBlock)
}
func assertEqualBlock(t *testing.T, expected *common.Block, actual *common.Block) {
assert.Equal(t, expected.EthBlockNum, actual.EthBlockNum)
assert.Equal(t, expected.Hash, actual.Hash)
assert.Equal(t, expected.Timestamp.Unix(), actual.Timestamp.Unix())
}
func TestBatches(t *testing.T) {
const fromBlock uint64 = 1
const toBlock uint64 = 3
const nBatchesPerBlock = 3
// Prepare blocks in the DB
setTestBlocks(fromBlock, toBlock)
// Generate fake batches
var batches []common.Batch
collectedFees := make(map[common.TokenID]*big.Int)
for i := 0; i < 64; i++ {
collectedFees[common.TokenID(i)] = big.NewInt(int64(i))
}
for i := fromBlock; i < toBlock; i++ {
for j := 0; j < nBatchesPerBlock; j++ {
batch := common.Batch{
BatchNum: common.BatchNum(int(i-1)*nBatchesPerBlock + j),
EthBlockNum: uint64(i),
ForgerAddr: eth.BigToAddress(big.NewInt(239457111187)),
CollectedFees: collectedFees,
StateRoot: common.Hash([]byte("duhdqlwiucgwqeiu")),
NumAccounts: j,
ExitRoot: common.Hash([]byte("tykertheuhtgenuer3iuw3b")),
SlotNum: common.SlotNum(j),
}
if j%2 == 0 {
batch.ForgeL1TxsNum = uint32(i)
}
batches = append(batches, batch)
}
}
// Add batches to the DB
err := historyDB.addBatches(batches)
assert.NoError(t, err)
// Get batches from the DB
fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(int(toBlock-fromBlock)*nBatchesPerBlock))
assert.NoError(t, err)
for i, fetchedBatch := range fetchedBatches {
assert.Equal(t, batches[i], *fetchedBatch)
}
// Test GetLastBatchNum
fetchedLastBatchNum, err := historyDB.GetLastBatchNum()
assert.NoError(t, err)
assert.Equal(t, batches[len(batches)-1].BatchNum, fetchedLastBatchNum)
// Test GetLastL1TxsNum
fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum()
assert.NoError(t, err)
assert.Equal(t, batches[len(batches)-1-(int(toBlock-fromBlock+1)%nBatchesPerBlock)].ForgeL1TxsNum, fetchedLastL1TxsNum)
}
func TestBids(t *testing.T) {
@ -82,29 +149,31 @@ func TestBids(t *testing.T) {
err := historyDB.addBids(bids)
assert.NoError(t, err)
// Fetch bids
fetchedBidsPtr, err := historyDB.GetBidsByBlock(fromBlock, toBlock)
assert.NoError(t, err)
var fetchedBids []*common.Bid
for i := fromBlock; i < toBlock; i++ {
fetchedBidsSlot, err := historyDB.GetBidsBySlot(common.SlotNum(i))
assert.NoError(t, err)
fetchedBids = append(fetchedBids, fetchedBidsSlot...)
}
// Compare fetched bids vs generated bids
fetchedBids := make([]common.Bid, 0, (toBlock-fromBlock)*bidsPerSlot)
for _, bid := range fetchedBidsPtr {
fetchedBids = append(fetchedBids, *bid)
for i, bid := range fetchedBids {
assert.Equal(t, bids[i], *bid)
}
assert.Equal(t, bids, fetchedBids)
}
// setTestBlocks WARNING: this will delete the blocks and recreate them
func setTestBlocks(from, to uint64) {
if from == 0 {
if err := historyDB.reorg(from); err != nil {
if err := historyDB.Reorg(from); err != nil {
panic(err)
}
} else {
if err := historyDB.reorg(from - 1); err != nil {
if err := historyDB.Reorg(from - 1); err != nil {
panic(err)
}
}
blocks := genBlocks(from, to)
if err := historyDB.addBlocks(blocks); err != nil {
if err := addBlocks(blocks); err != nil {
panic(err)
}
}
@ -120,3 +189,12 @@ func genBlocks(from, to uint64) []common.Block {
}
return blocks
}
// addBlocks insert blocks into the DB. TODO: move method to test
func addBlocks(blocks []common.Block) error {
return db.BulkInsert(
historyDB.db,
"INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s",
blocks[:],
)
}

+ 38
- 12
db/historydb/migrations/001_init.sql

@ -5,15 +5,9 @@ CREATE TABLE block (
hash BYTEA NOT NULL
);
CREATE TABLE slot_min_prices (
eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE,
min_prices VARCHAR(200) NOT NULL
);
CREATE TABLE coordianator (
forger_addr BYTEA NOT NULL,
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
beneficiary_addr BYTEA NOT NULL,
withdraw_addr BYTEA NOT NULL,
url VARCHAR(200) NOT NULL,
PRIMARY KEY (forger_addr, eth_block_num)
@ -32,14 +26,22 @@ CREATE TABLE batch (
);
CREATE TABLE exit_tree (
batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
account_idx BIGINT NOT NULL,
batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE,
account_idx BIGINT,
merkle_proof BYTEA NOT NULL,
amount NUMERIC NOT NULL,
balance NUMERIC NOT NULL,
nullifier BYTEA NOT NULL,
PRIMARY KEY (batch_num, account_idx)
);
CREATE TABLE withdrawal (
batch_num BIGINT,
account_idx BIGINT,
eth_block_num BIGINT REFERENCES block (eth_block_num) ON DELETE CASCADE,
FOREIGN KEY (batch_num, account_idx) REFERENCES exit_tree (batch_num, account_idx) ON DELETE CASCADE,
PRIMARY KEY (batch_num, account_idx)
);
CREATE TABLE bid (
slot_num BIGINT NOT NULL,
bid_value BYTEA NOT NULL, -- (check if we can do a max(), if not add float for order purposes)
@ -69,7 +71,8 @@ CREATE TABLE l1tx (
token_id INT NOT NULL REFERENCES token (token_id),
amount NUMERIC NOT NULL,
load_amount BYTEA NOT NULL,
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
tx_type VARCHAR(40) NOT NULL
);
CREATE TABLE l2tx (
@ -80,7 +83,8 @@ CREATE TABLE l2tx (
to_idx BIGINT NOT NULL,
amount NUMERIC NOT NULL,
fee INT NOT NULL,
nonce BIGINT NOT NULL
nonce BIGINT NOT NULL,
tx_type VARCHAR(40) NOT NULL
);
CREATE TABLE account (
@ -91,7 +95,30 @@ CREATE TABLE account (
eth_addr BYTEA NOT NULL
);
CREATE TABLE rollup_vars (
eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE,
forge_l1_timeout BYTEA NOT NULL,
fee_l1_user_tx BYTEA NOT NULL,
fee_add_token BYTEA NOT NULL,
tokens_hez BYTEA NOT NULL,
governance BYTEA NOT NULL
);
CREATE TABLE consensus_vars (
eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE,
slot_deadline INT NOT NULL,
close_auction_slots INT NOT NULL,
open_auction_slots INT NOT NULL,
min_bid_slots VARCHAR(200) NOT NULL,
outbidding INT NOT NULL,
donation_address BYTEA NOT NULL,
governance_address BYTEA NOT NULL,
allocation_ratio vARCHAR(200)
);
-- +migrate Down
DROP TABLE consensus_vars;
DROP TABLE rollup_vars;
DROP TABLE account;
DROP TABLE l2tx;
DROP TABLE l1tx;
@ -100,5 +127,4 @@ DROP TABLE bid;
DROP TABLE exit_tree;
DROP TABLE batch;
DROP TABLE coordianator;
DROP TABLE slot_min_prices;
DROP TABLE block;

+ 2
- 1
db/l2db/migrations/001_init.sql

@ -22,7 +22,8 @@ CREATE TABLE tx_pool (
signature BYTEA NOT NULL,
timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL,
absolute_fee NUMERIC,
absolute_fee_update TIMESTAMP WITHOUT TIME ZONE
absolute_fee_update TIMESTAMP WITHOUT TIME ZONE,
tx_type VARCHAR(40) NOT NULL
);
CREATE TABLE account_creation_auth (

+ 1
- 0
go.sum

@ -544,6 +544,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

Loading…
Cancel
Save