create sql tables (#40)

Co-authored-by: Eduard S <eduard@iden3.io>
2026-02-07 03:16:45 +01:00 · 2020-08-11 16:25:55 +02:00
parent 1313a5aabd
commit 428dcd7590
10 changed files with 780 additions and 14 deletions
--- a/db/historydb/historydb.go
+++ b/db/historydb/historydb.go
@@ -0,0 +1,94 @@
+package historydb
+
+import (
+	"fmt"
+
+	"github.com/gobuffalo/packr/v2"
+	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/hermeznetwork/hermez-node/db"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq" // driver for postgres DB
+	migrate "github.com/rubenv/sql-migrate"
+	"github.com/russross/meddler"
+)
+
+// HistoryDB persist the historic of the rollup
+type HistoryDB struct {
+	db *sqlx.DB
+}
+
+// NewHistoryDB initialize the DB
+func NewHistoryDB(port int, host, user, password, dbname string) (*HistoryDB, error) {
+	// Connect to DB
+	psqlconn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", host, port, user, password, dbname)
+	hdb, err := sqlx.Connect("postgres", psqlconn)
+	if err != nil {
+		return nil, err
+	}
+	// Init meddler
+	db.InitMeddler()
+	meddler.Default = meddler.PostgreSQL
+
+	// Run DB migrations
+	migrations := &migrate.PackrMigrationSource{
+		Box: packr.New("history-migrations", "./migrations"),
+	}
+	if _, err := migrate.Exec(hdb.DB, "postgres", migrations, migrate.Up); err != nil {
+		return nil, err
+	}
+
+	return &HistoryDB{hdb}, nil
+}
+
+// addBlocks insert blocks into the DB
+func (hdb *HistoryDB) addBlocks(blocks []common.Block) error {
+	return db.BulkInsert(
+		hdb.db,
+		"INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s",
+		blocks[:],
+	)
+}
+
+// GetBlocks retrrieve blocks from the DB
+func (hdb *HistoryDB) GetBlocks(from, to uint64) ([]*common.Block, error) {
+	var blocks []*common.Block
+	err := meddler.QueryAll(
+		hdb.db, &blocks,
+		"SELECT * FROM block WHERE $1 <= eth_block_num AND eth_block_num < $2",
+		from, to,
+	)
+	return blocks, err
+}
+
+// reorg deletes all the information that was added into the DB after the lastValidBlock
+// WARNING: this is a draaft of the function, useful at the moment for tests
+func (hdb *HistoryDB) reorg(lastValidBlock uint64) error {
+	_, err := hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock)
+	return err
+}
+
+// addBids insert Bids into the DB
+func (hdb *HistoryDB) addBids(bids []common.Bid) error {
+	// TODO: check the coordinator info
+	return db.BulkInsert(
+		hdb.db,
+		"INSERT INTO bid (slot_num, forger_addr, bid_value, eth_block_num) VALUES %s",
+		bids[:],
+	)
+}
+
+// GetBidsByBlock return the bids done between the block from and to
+func (hdb *HistoryDB) GetBidsByBlock(from, to uint64) ([]*common.Bid, error) {
+	var bids []*common.Bid
+	err := meddler.QueryAll(
+		hdb.db, &bids,
+		"SELECT * FROM bid WHERE $1 <= eth_block_num AND eth_block_num < $2",
+		from, to,
+	)
+	return bids, err
+}
+
+// Close frees the resources used by HistoryDB
+func (hdb *HistoryDB) Close() error {
+	return hdb.db.Close()
+}
--- a/db/historydb/historydb_test.go
+++ b/db/historydb/historydb_test.go
@@ -0,0 +1,122 @@
+package historydb
+
+import (
+	"fmt"
+	"math/big"
+	"os"
+	"testing"
+	"time"
+
+	eth "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/stretchr/testify/assert"
+)
+
+var historyDB *HistoryDB
+
+// In order to run the test you need to run a Posgres DB with
+// a database named "history" that is accessible by
+// user: "hermez"
+// pass: set it using the env var POSTGRES_PASS
+// This can be achieved by running: POSTGRES_PASS=your_strong_pass && sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=history -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD=$POSTGRES_PASS -d postgres && sleep 2s && sudo docker exec -it hermez-db-test psql -a history -U hermez -c "CREATE DATABASE l2;"
+// After running the test you can stop the container by running: sudo docker kill hermez-db-test
+// If you already did that for the L2DB you don't have to do it again
+
+func TestMain(m *testing.M) {
+	// init DB
+	var err error
+	pass := os.Getenv("POSTGRES_PASS")
+	historyDB, err = NewHistoryDB(5432, "localhost", "hermez", pass, "history")
+	if err != nil {
+		panic(err)
+	}
+	// Run tests
+	result := m.Run()
+	// Close DB
+	if err := historyDB.Close(); err != nil {
+		fmt.Println("Error closing the history DB:", err)
+	}
+	os.Exit(result)
+}
+
+func TestAddBlock(t *testing.T) {
+	var fromBlock, toBlock uint64
+	fromBlock = 1
+	toBlock = 5
+	// Delete peviously created rows (clean previous test execs)
+	assert.NoError(t, historyDB.reorg(fromBlock-1))
+	// Generate fake blocks
+	blocks := genBlocks(fromBlock, toBlock)
+	// Insert blocks into DB
+	err := historyDB.addBlocks(blocks)
+	assert.NoError(t, err)
+	// Get blocks from DB
+	fetchedBlocks, err := historyDB.GetBlocks(fromBlock, toBlock)
+	// Compare generated vs getted blocks
+	assert.NoError(t, err)
+	for i, fetchedBlock := range fetchedBlocks {
+		assert.Equal(t, blocks[i].EthBlockNum, fetchedBlock.EthBlockNum)
+		assert.Equal(t, blocks[i].Hash, fetchedBlock.Hash)
+		assert.Equal(t, blocks[i].Timestamp.Unix(), fetchedBlock.Timestamp.Unix())
+	}
+}
+
+func TestBids(t *testing.T) {
+	const fromBlock uint64 = 1
+	const toBlock uint64 = 5
+	const bidsPerSlot = 5
+	// Prepare blocks in the DB
+	setTestBlocks(fromBlock, toBlock)
+	// Generate fake bids
+	bids := make([]common.Bid, 0, (toBlock-fromBlock)*bidsPerSlot)
+	for i := fromBlock; i < toBlock; i++ {
+		for j := 0; j < bidsPerSlot; j++ {
+			bids = append(bids, common.Bid{
+				SlotNum:     common.SlotNum(i),
+				BidValue:    big.NewInt(int64(j)),
+				EthBlockNum: i,
+				ForgerAddr:  eth.BigToAddress(big.NewInt(int64(j))),
+			})
+		}
+	}
+	err := historyDB.addBids(bids)
+	assert.NoError(t, err)
+	// Fetch bids
+	fetchedBidsPtr, err := historyDB.GetBidsByBlock(fromBlock, toBlock)
+	assert.NoError(t, err)
+	// Compare fetched bids vs generated bids
+	fetchedBids := make([]common.Bid, 0, (toBlock-fromBlock)*bidsPerSlot)
+	for _, bid := range fetchedBidsPtr {
+		fetchedBids = append(fetchedBids, *bid)
+	}
+	assert.Equal(t, bids, fetchedBids)
+}
+
+// setTestBlocks WARNING: this will delete the blocks and recreate them
+func setTestBlocks(from, to uint64) {
+	if from == 0 {
+		if err := historyDB.reorg(from); err != nil {
+			panic(err)
+		}
+	} else {
+		if err := historyDB.reorg(from - 1); err != nil {
+			panic(err)
+		}
+	}
+	blocks := genBlocks(from, to)
+	if err := historyDB.addBlocks(blocks); err != nil {
+		panic(err)
+	}
+}
+
+func genBlocks(from, to uint64) []common.Block {
+	var blocks []common.Block
+	for i := from; i < to; i++ {
+		blocks = append(blocks, common.Block{
+			EthBlockNum: i,
+			Timestamp:   time.Now().Add(time.Second * 13).UTC(),
+			Hash:        eth.BigToHash(big.NewInt(int64(i))),
+		})
+	}
+	return blocks
+}
--- a/db/historydb/migrations/001_init.sql
+++ b/db/historydb/migrations/001_init.sql
@@ -0,0 +1,106 @@
+-- +migrate Up
+CREATE TABLE block (
+    eth_block_num BIGINT PRIMARY KEY,
+    timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+    hash BYTEA NOT NULL
+);
+
+CREATE TABLE slot_min_prices (
+    eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE,
+    min_prices VARCHAR(200) NOT NULL
+);
+
+CREATE TABLE coordiantor (
+    forger_addr BYTEA NOT NULL,
+    eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
+    beneficiary_addr BYTEA NOT NULL,
+    withdraw_addr BYTEA NOT NULL,
+    url VARCHAR(200) NOT NULL,
+    PRIMARY KEY (forger_addr, eth_block_num)
+);
+
+CREATE TABLE batch (
+    batch_num BIGINT PRIMARY KEY,
+    eth_block_num BIGINT REFERENCES block (eth_block_num) ON DELETE CASCADE,
+    forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator
+    fees_collected BYTEA NOT NULL,
+    state_root BYTEA NOT NULL,
+    num_accounts BIGINT NOT NULL,
+    exit_root BYTEA NOT NULL,
+    forge_l1_txs_num BIGINT,
+    slot_num BIGINT NOT NULL
+);
+
+CREATE TABLE exit_tree (
+    batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
+    account_idx BIGINT NOT NULL,
+    merkle_proof BYTEA NOT NULL,
+    amount NUMERIC NOT NULL,
+    nullifier BYTEA NOT NULL,
+    PRIMARY KEY (batch_num, account_idx)
+);
+
+CREATE TABLE bid (
+    slot_num BIGINT NOT NULL,
+    bid_value BYTEA NOT NULL, -- (check if we can do a max(), if not add float for order purposes)
+    eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
+    forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator
+    PRIMARY KEY (slot_num, bid_value)
+);
+
+CREATE TABLE token (
+    token_id INT PRIMARY KEY,
+    eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
+    eth_addr BYTEA UNIQUE NOT NULL,
+    name VARCHAR(20) NOT NULL,
+    symbol VARCHAR(10) NOT NULL,
+    decimals INT NOT NULL
+);
+
+CREATE TABLE l1tx (
+    tx_id BYTEA PRIMARY KEY,
+    from_idx BIGINT NOT NULL,
+    to_idx BIGINT NOT NULL,
+    token_id INT NOT NULL REFERENCES token (token_id),
+    amount NUMERIC NOT NULL,
+    nonce BIGINT NOT NULL,
+    fee INT NOT NULL,
+    eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
+    to_forge_l1_txs_num BIGINT NOT NULL,
+    position INT NOT NULL,
+    origin_user BOOLEAN NOT NULL,
+    from_eth_addr BYTEA NOT NULL,
+    from_bjj BYTEA NOT NULL,
+    load_amount BYTEA NOT NULL
+);
+
+CREATE TABLE l2tx (
+    tx_id BYTEA PRIMARY KEY,
+    batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
+    position INT NOT NULL,
+    from_idx BIGINT NOT NULL,
+    to_idx BIGINT NOT NULL,
+    amount NUMERIC NOT NULL,
+    fee INT NOT NULL,
+    nonce BIGINT NOT NULL
+);
+
+CREATE TABLE account (
+    idx BIGINT PRIMARY KEY,
+    token_id INT NOT NULL REFERENCES token (token_id),
+    batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
+    bjj BYTEA NOT NULL,
+    eth_addr BYTEA NOT NULL
+);
+
+-- +migrate Down
+DROP TABLE account;
+DROP TABLE l2tx;
+DROP TABLE l1tx;
+DROP TABLE token;
+DROP TABLE bid;
+DROP TABLE exit_tree;
+DROP TABLE batch;
+DROP TABLE coordiantor;
+DROP TABLE slot_min_prices;
+DROP TABLE block;