Add HistoryDB SQL triggers (#125)

README.md

@@ -2,20 +2,25 @@
 
 Go implementation of the Hermez node.
 
-
 ## Test
+
 - First run a docker instance of the PostgresSQL (where `yourpasswordhere` should be your password)
+
 ```
 POSTGRES_PASS=yourpasswordhere; sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=history -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$POSTGRES_PASS" -d postgres && sleep 2s && sudo docker exec hermez-db-test psql -a history -U hermez -c "CREATE DATABASE l2;"
 ```
+
 - Then, run the tests with the password as env var
+
 ```
 POSTGRES_PASS=yourpasswordhere ETHCLIENT_DIAL_URL=yourethereumurlhere go test ./...
 ```
 
 ## Lint
+
 - Install [golangci-lint](https://golangci-lint.run)
 - Once installed, to check the lints
+
 ```
 golangci-lint run --timeout=5m -E whitespace -E gosec -E gci -E misspell -E gomnd -E gofmt -E goimports -E golint --exclude-use-default=false --max-same-issues 0
 ```

common/account.go

@@ -65,11 +65,13 @@ func IdxFromBigInt(b *big.Int) (Idx, error) {
 
 // Account is a struct that gives information of the holdings of an address and a specific token. Is the data structure that generates the Value stored in the leaf of the MerkleTree
 type Account struct {
-	TokenID   TokenID
+	Idx       Idx                `meddler:"idx"`
-	Nonce     Nonce    // max of 40 bits used
+	TokenID   TokenID            `meddler:"token_id"`
-	Balance   *big.Int // max of 192 bits used
+	BatchNum  BatchNum           `meddler:"batch_num"`
-	PublicKey *babyjub.PublicKey
+	PublicKey *babyjub.PublicKey `meddler:"bjj"`
-	EthAddr   ethCommon.Address
+	EthAddr   ethCommon.Address  `meddler:"eth_addr"`
+	Nonce     Nonce              `meddler:"-"` // max of 40 bits used
+	Balance   *big.Int           `meddler:"-"` // max of 192 bits used
 }
 
 func (a *Account) String() string {

common/block.go

@@ -9,7 +9,7 @@ import (
 // Block represents of an Ethereum block
 type Block struct {
 	EthBlockNum int64          `meddler:"eth_block_num"`
-	Timestamp   time.Time      `meddler:"timestamp"`
+	Timestamp   time.Time      `meddler:"timestamp,utctime"`
 	Hash        ethCommon.Hash `meddler:"hash"`
 	ParentHash  ethCommon.Hash `meddler:"-"`
 }

common/coordinator.go

@@ -7,8 +7,8 @@ import (
 // Coordinator represents a Hermez network coordinator who wins an auction for an specific slot
 // WARNING: this is strongly based on the previous implementation, once the new spec is done, this may change a lot.
 type Coordinator struct {
+	EthBlockNum int64             // block in which the coordinator was registered
 	Forger      ethCommon.Address // address of the forger
-	Beneficiary ethCommon.Address // address of the beneficiary
 	Withdraw    ethCommon.Address // address of the withdraw
 	URL         string            // URL of the coordinators API
 }

common/l1tx.go

@@ -18,31 +18,49 @@ const (
 // L1Tx is a struct that represents a L1 tx
 type L1Tx struct {
 	// Stored in DB: mandatory fileds
-	TxID            TxID               `meddler:"tx_id"`
+	TxID            TxID
-	ToForgeL1TxsNum uint32             `meddler:"to_forge_l1_txs_num"` // toForgeL1TxsNum in which the tx was forged / will be forged
+	ToForgeL1TxsNum uint32 // toForgeL1TxsNum in which the tx was forged / will be forged
-	Position        int                `meddler:"position"`
+	Position        int
-	UserOrigin      bool               `meddler:"user_origin"` // true if the tx was originated by a user, false if it was aoriginated by a coordinator. Note that this differ from the spec for implementation simplification purpposes
+	UserOrigin      bool // true if the tx was originated by a user, false if it was aoriginated by a coordinator. Note that this differ from the spec for implementation simplification purpposes
-	FromIdx         Idx                `meddler:"from_idx"`    // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit)
+	FromIdx         Idx  // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit)
-	FromEthAddr     ethCommon.Address  `meddler:"from_eth_addr"`
+	FromEthAddr     ethCommon.Address
-	FromBJJ         *babyjub.PublicKey `meddler:"from_bjj"`
+	FromBJJ         *babyjub.PublicKey
-	ToIdx           Idx                `meddler:"to_idx"` // ToIdx is ignored in L1Tx/Deposit, but used in the L1Tx/DepositAndTransfer
+	ToIdx           Idx // ToIdx is ignored in L1Tx/Deposit, but used in the L1Tx/DepositAndTransfer
-	TokenID         TokenID            `meddler:"token_id"`
+	TokenID         TokenID
-	Amount          *big.Int           `meddler:"amount,bigint"`
+	Amount          *big.Int
-	LoadAmount      *big.Int           `meddler:"load_amount,bigint"`
+	LoadAmount      *big.Int
-	EthBlockNum     uint64             `meddler:"eth_block_num"` // Ethereum Block Number in which this L1Tx was added to the queue
+	EthBlockNum     int64 // Ethereum Block Number in which this L1Tx was added to the queue
-	Type            TxType             `meddler:"tx_type"`
+	Type            TxType
-	BatchNum        BatchNum           `meddler:"-"`
+	BatchNum        BatchNum
 }
 
 // Tx returns a *Tx from the L1Tx
 func (tx *L1Tx) Tx() *Tx {
-	return &Tx{
+	f := new(big.Float).SetInt(tx.Amount)
+	amountFloat, _ := f.Float64()
+	genericTx := &Tx{
+		IsL1:            true,
 		TxID:            tx.TxID,
+		Type:            tx.Type,
+		Position:        tx.Position,
 		FromIdx:         tx.FromIdx,
 		ToIdx:           tx.ToIdx,
 		Amount:          tx.Amount,
-		Type:    tx.Type,
+		AmountFloat:     amountFloat,
+		TokenID:         tx.TokenID,
+		ToForgeL1TxsNum: tx.ToForgeL1TxsNum,
+		UserOrigin:      tx.UserOrigin,
+		FromEthAddr:     tx.FromEthAddr,
+		FromBJJ:         tx.FromBJJ,
+		LoadAmount:      tx.LoadAmount,
+		EthBlockNum:     tx.EthBlockNum,
 	}
+	if tx.LoadAmount != nil {
+		lf := new(big.Float).SetInt(tx.LoadAmount)
+		loadAmountFloat, _ := lf.Float64()
+		genericTx.LoadAmountFloat = loadAmountFloat
+	}
+	return genericTx
 }
 
 // Bytes encodes a L1Tx into []byte

common/l2tx.go

@@ -7,27 +7,35 @@ import (
 // L2Tx is a struct that represents an already forged L2 tx
 type L2Tx struct {
 	// Stored in DB: mandatory fileds
-	TxID     TxID        `meddler:"tx_id"`
+	TxID        TxID
-	BatchNum BatchNum    `meddler:"batch_num"` // batchNum in which this tx was forged.
+	BatchNum    BatchNum // batchNum in which this tx was forged.
-	Position int         `meddler:"position"`
+	Position    int
-	FromIdx  Idx         `meddler:"from_idx"`
+	FromIdx     Idx
-	ToIdx    Idx         `meddler:"to_idx"`
+	ToIdx       Idx
-	Amount   *big.Int    `meddler:"amount,bigint"`
+	Amount      *big.Int
-	Fee      FeeSelector `meddler:"fee"`
+	Fee         FeeSelector
-	Nonce    Nonce       `meddler:"nonce"`
+	Nonce       Nonce
-	Type     TxType      `meddler:"tx_type"`
+	Type        TxType
+	EthBlockNum int64 // Ethereum Block Number in which this L2Tx was added to the queue
 }
 
 // Tx returns a *Tx from the L2Tx
 func (tx *L2Tx) Tx() *Tx {
+	f := new(big.Float).SetInt(tx.Amount)
+	amountFloat, _ := f.Float64()
 	return &Tx{
+		IsL1:        false,
 		TxID:        tx.TxID,
+		Type:        tx.Type,
+		Position:    tx.Position,
 		FromIdx:     tx.FromIdx,
 		ToIdx:       tx.ToIdx,
 		Amount:      tx.Amount,
-		Nonce:   tx.Nonce,
+		AmountFloat: amountFloat,
+		BatchNum:    tx.BatchNum,
+		EthBlockNum: tx.EthBlockNum,
 		Fee:         tx.Fee,
-		Type:    tx.Type,
+		Nonce:       tx.Nonce,
 	}
 }
 

common/token.go

@@ -10,13 +10,14 @@ import (
 
 // Token is a struct that represents an Ethereum token that is supported in Hermez network
 type Token struct {
-	TokenID     TokenID
+	TokenID     TokenID           `meddler:"token_id"`
-	EthAddr     ethCommon.Address
+	EthBlockNum int64             `meddler:"eth_block_num"` // Ethereum block number in which this token was registered
-	Name        string
+	EthAddr     ethCommon.Address `meddler:"eth_addr"`
-	Symbol      string
+	Name        string            `meddler:"name"`
-	Decimals    uint64
+	Symbol      string            `meddler:"symbol"`
-	EthTxHash   ethCommon.Hash // Ethereum TxHash in which this token was registered
+	Decimals    uint64            `meddler:"decimals"`
-	EthBlockNum uint64         // Ethereum block number in which this token was registered
+	USD         float32           `meddler:"usd,zeroisnull"`
+	USDUpdate   time.Time         `meddler:"usd_update,utctimez"`
 }
 
 // TokenInfo provides the price of the token in USD

common/tx.go

@@ -2,6 +2,9 @@ package common
 
 import (
 	"math/big"
+
+	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/iden3/go-iden3-crypto/babyjub"
 )
 
 // TxID is the identifier of a Hermez network transaction
@@ -37,12 +40,29 @@ const (
 
 // Tx is a struct used by the TxSelector & BatchBuilder as a generic type generated from L1Tx & PoolL2Tx
 type Tx struct {
-	TxID     TxID
+	// Generic
-	FromIdx  Idx // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit)
+	IsL1        bool     `meddler:"is_l1"`
-	ToIdx    Idx // ToIdx is ignored in L1Tx/Deposit, but used in the L1Tx/DepositTransfer
+	TxID        TxID     `meddler:"id"`
-	Amount   *big.Int
+	Type        TxType   `meddler:"type"`
-	Nonce    Nonce // effective 40 bits used
+	Position    int      `meddler:"position"`
-	Fee      FeeSelector
+	FromIdx     Idx      `meddler:"from_idx"`
-	Type     TxType   // optional, descrives which kind of tx it's
+	ToIdx       Idx      `meddler:"to_idx"`
-	BatchNum BatchNum // batchNum in which this tx was forged. Presence indicates "forged" state.
+	Amount      *big.Int `meddler:"amount,bigint"`
+	AmountFloat float64  `meddler:"amount_f"`
+	TokenID     TokenID  `meddler:"token_id"`
+	USD         float64  `meddler:"amount_usd,zeroisnull"`
+	BatchNum    BatchNum `meddler:"batch_num,zeroisnull"` // batchNum in which this tx was forged. If the tx is L2, this must be != 0
+	EthBlockNum int64    `meddler:"eth_block_num"`        // Ethereum Block Number in which this L1Tx was added to the queue
+	// L1
+	ToForgeL1TxsNum uint32             `meddler:"to_forge_l1_txs_num"` // toForgeL1TxsNum in which the tx was forged / will be forged
+	UserOrigin      bool               `meddler:"user_origin"`         // true if the tx was originated by a user, false if it was aoriginated by a coordinator. Note that this differ from the spec for implementation simplification purpposes
+	FromEthAddr     ethCommon.Address  `meddler:"from_eth_addr"`
+	FromBJJ         *babyjub.PublicKey `meddler:"from_bjj"`
+	LoadAmount      *big.Int           `meddler:"load_amount,bigintnull"`
+	LoadAmountFloat float64            `meddler:"load_amount_f"`
+	LoadAmountUSD   float64            `meddler:"load_amount_usd,zeroisnull"`
+	// L2
+	Fee    FeeSelector `meddler:"fee,zeroisnull"`
+	FeeUSD float64     `meddler:"fee_usd,zeroisnull"`
+	Nonce  Nonce       `meddler:"nonce,zeroisnull"`
 }

db/historydb/historydb.go

@@ -47,6 +47,19 @@ func (hdb *HistoryDB) AddBlock(block *common.Block) error {
 	return meddler.Insert(hdb.db, "block", block)
 }
 
+// AddBlocks inserts blocks into the DB
+func (hdb *HistoryDB) AddBlocks(blocks []common.Block) error {
+	return db.BulkInsert(
+		hdb.db,
+		`INSERT INTO block (
+			eth_block_num,
+			timestamp,
+			hash
+		) VALUES %s;`,
+		blocks[:],
+	)
+}
+
 // GetBlock retrieve a block from the DB, given a block number
 func (hdb *HistoryDB) GetBlock(blockNum int64) (*common.Block, error) {
 	block := &common.Block{}
@@ -77,8 +90,8 @@ func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) {
 	return block, err
 }
 
-// addBatches insert Bids into the DB
+// AddBatches insert Bids into the DB
-func (hdb *HistoryDB) addBatches(batches []common.Batch) error {
+func (hdb *HistoryDB) AddBatches(batches []common.Batch) error {
 	return db.BulkInsert(
 		hdb.db,
 		`INSERT INTO batch (
@@ -129,7 +142,7 @@ func (hdb *HistoryDB) Reorg(lastValidBlock int64) error {
 
 // SyncRollup stores all the data that can be changed / added on a block in the Rollup SC
 func (hdb *HistoryDB) SyncRollup(
-	blockNum int64,
+	blockNum uint64,
 	l1txs []common.L1Tx,
 	l2txs []common.L2Tx,
 	registeredAccounts []common.Account,
@@ -140,7 +153,7 @@ func (hdb *HistoryDB) SyncRollup(
 	vars *common.RollupVars,
 ) error {
 	// TODO: make all in a single DB commit
-	if err := hdb.addBatches(batches); err != nil {
+	if err := hdb.AddBatches(batches); err != nil {
 		return err
 	}
 	return nil
@@ -148,7 +161,7 @@ func (hdb *HistoryDB) SyncRollup(
 
 // SyncPoD stores all the data that can be changed / added on a block in the PoD SC
 func (hdb *HistoryDB) SyncPoD(
-	blockNum int64,
+	blockNum uint64,
 	bids []common.Bid,
 	coordinators []common.Coordinator,
 	vars *common.AuctionVars,
@@ -166,17 +179,154 @@ func (hdb *HistoryDB) addBids(bids []common.Bid) error {
 	)
 }
 
-// GetBidsBySlot return the bids for a specific slot
+// GetBids return the bids
-func (hdb *HistoryDB) GetBidsBySlot(slotNum common.SlotNum) ([]*common.Bid, error) {
+func (hdb *HistoryDB) GetBids() ([]*common.Bid, error) {
 	var bids []*common.Bid
 	err := meddler.QueryAll(
 		hdb.db, &bids,
-		"SELECT * FROM bid WHERE $1 = slot_num;",
+		"SELECT * FROM bid;",
-		slotNum,
 	)
 	return bids, err
 }
 
+// AddToken insert a token into the DB
+func (hdb *HistoryDB) AddToken(token *common.Token) error {
+	return meddler.Insert(hdb.db, "token", token)
+}
+
+// AddTokens insert tokens into the DB
+func (hdb *HistoryDB) AddTokens(tokens []common.Token) error {
+	return db.BulkInsert(
+		hdb.db,
+		`INSERT INTO token (
+			token_id,
+			eth_block_num,
+			eth_addr,
+			name,
+			symbol,
+			decimals,
+			usd,
+			usd_update
+		) VALUES %s;`,
+		tokens[:],
+	)
+}
+
+// UpdateTokenValue updates the USD value of a token
+func (hdb *HistoryDB) UpdateTokenValue(tokenID common.TokenID, value float32) error {
+	_, err := hdb.db.Exec(
+		"UPDATE token SET usd = $1 WHERE token_id = $2;",
+		value, tokenID,
+	)
+	return err
+}
+
+// GetTokens returns a list of tokens from the DB
+func (hdb *HistoryDB) GetTokens() ([]*common.Token, error) {
+	var tokens []*common.Token
+	err := meddler.QueryAll(
+		hdb.db, &tokens,
+		"SELECT * FROM token ORDER BY token_id;",
+	)
+	return tokens, err
+}
+
+// AddAccounts insert accounts into the DB
+func (hdb *HistoryDB) AddAccounts(accounts []common.Account) error {
+	return db.BulkInsert(
+		hdb.db,
+		`INSERT INTO account (
+			idx,
+			token_id,
+			batch_num,
+			bjj,
+			eth_addr
+		) VALUES %s;`,
+		accounts[:],
+	)
+}
+
+// GetAccounts returns a list of accounts from the DB
+func (hdb *HistoryDB) GetAccounts() ([]*common.Account, error) {
+	var accs []*common.Account
+	err := meddler.QueryAll(
+		hdb.db, &accs,
+		"SELECT * FROM account ORDER BY idx;",
+	)
+	return accs, err
+}
+
+// AddL1Txs inserts L1 txs to the DB
+func (hdb *HistoryDB) AddL1Txs(l1txs []common.L1Tx) error {
+	txs := []common.Tx{}
+	for _, tx := range l1txs {
+		txs = append(txs, *tx.Tx())
+	}
+	return hdb.AddTxs(txs)
+}
+
+// AddL2Txs inserts L2 txs to the DB
+func (hdb *HistoryDB) AddL2Txs(l2txs []common.L2Tx) error {
+	txs := []common.Tx{}
+	for _, tx := range l2txs {
+		txs = append(txs, *tx.Tx())
+	}
+	return hdb.AddTxs(txs)
+}
+
+// AddTxs insert L1 txs into the DB
+func (hdb *HistoryDB) AddTxs(txs []common.Tx) error {
+	return db.BulkInsert(
+		hdb.db,
+		`INSERT INTO tx (
+			is_l1,
+			id,
+			type,
+			position,
+			from_idx,
+			to_idx,
+			amount,
+			amount_f,
+			token_id,
+			amount_usd,
+			batch_num,
+			eth_block_num,
+			to_forge_l1_txs_num,
+			user_origin,
+			from_eth_addr,
+			from_bjj,
+			load_amount,
+			load_amount_f,
+			load_amount_usd,
+			fee,
+			fee_usd,
+			nonce
+		) VALUES %s;`,
+		txs[:],
+	)
+}
+
+// GetTxs returns a list of txs from the DB
+func (hdb *HistoryDB) GetTxs() ([]*common.Tx, error) {
+	var txs []*common.Tx
+	err := meddler.QueryAll(
+		hdb.db, &txs,
+		`SELECT * FROM tx 
+		ORDER BY (batch_num, position) ASC`,
+	)
+	return txs, err
+}
+
+// GetTx returns a tx from the DB
+func (hdb *HistoryDB) GetTx(txID common.TxID) (*common.Tx, error) {
+	tx := new(common.Tx)
+	return tx, meddler.QueryRow(
+		hdb.db, tx,
+		"SELECT * FROM tx WHERE id = $1;",
+		txID,
+	)
+}
+
 // Close frees the resources used by HistoryDB
 func (hdb *HistoryDB) Close() error {
 	return hdb.db.Close()

db/historydb/historydb_test.go

@@ -2,14 +2,16 @@ package historydb
 
 import (
 	"fmt"
+	"math"
 	"math/big"
 	"os"
 	"testing"
 	"time"
 
-	eth "github.com/ethereum/go-ethereum/common"
+	ethCommon "github.com/ethereum/go-ethereum/common"
 	"github.com/hermeznetwork/hermez-node/common"
-	"github.com/hermeznetwork/hermez-node/db"
+	"github.com/hermeznetwork/hermez-node/test"
+	"github.com/iden3/go-iden3-crypto/babyjub"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -47,7 +49,7 @@ func TestBlocks(t *testing.T) {
 	// Delete peviously created rows (clean previous test execs)
 	assert.NoError(t, historyDB.Reorg(fromBlock-1))
 	// Generate fake blocks
-	blocks := genBlocks(fromBlock, toBlock)
+	blocks := test.GenBlocks(fromBlock, toBlock)
 	// Insert blocks into DB
 	for i := 0; i < len(blocks); i++ {
 		err := historyDB.AddBlock(&blocks[i])
@@ -82,38 +84,16 @@ func assertEqualBlock(t *testing.T, expected *common.Block, actual *common.Block
 func TestBatches(t *testing.T) {
 	const fromBlock int64 = 1
 	const toBlock int64 = 3
-	const nBatchesPerBlock = 3
 	// Prepare blocks in the DB
-	setTestBlocks(fromBlock, toBlock)
+	blocks := setTestBlocks(fromBlock, toBlock)
 	// Generate fake batches
-	var batches []common.Batch
+	const nBatches = 9
-	collectedFees := make(map[common.TokenID]*big.Int)
+	batches := test.GenBatches(nBatches, blocks)
-	for i := 0; i < 64; i++ {
-		collectedFees[common.TokenID(i)] = big.NewInt(int64(i))
-	}
-	for i := fromBlock; i < toBlock; i++ {
-		for j := 0; j < nBatchesPerBlock; j++ {
-			batch := common.Batch{
-				BatchNum:      common.BatchNum(int(i-1)*nBatchesPerBlock + j),
-				EthBlockNum:   int64(i),
-				ForgerAddr:    eth.BigToAddress(big.NewInt(239457111187)),
-				CollectedFees: collectedFees,
-				StateRoot:     common.Hash([]byte("duhdqlwiucgwqeiu")),
-				NumAccounts:   j,
-				ExitRoot:      common.Hash([]byte("tykertheuhtgenuer3iuw3b")),
-				SlotNum:       common.SlotNum(j),
-			}
-			if j%2 == 0 {
-				batch.ForgeL1TxsNum = uint32(i)
-			}
-			batches = append(batches, batch)
-		}
-	}
 	// Add batches to the DB
-	err := historyDB.addBatches(batches)
+	err := historyDB.AddBatches(batches)
 	assert.NoError(t, err)
 	// Get batches from the DB
-	fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(int(toBlock-fromBlock)*nBatchesPerBlock))
+	fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(nBatches))
 	assert.NoError(t, err)
 	for i, fetchedBatch := range fetchedBatches {
 		assert.Equal(t, batches[i], *fetchedBatch)
@@ -125,76 +105,222 @@ func TestBatches(t *testing.T) {
 	// Test GetLastL1TxsNum
 	fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum()
 	assert.NoError(t, err)
-	assert.Equal(t, batches[len(batches)-1-(int(toBlock-fromBlock+1)%nBatchesPerBlock)].ForgeL1TxsNum, fetchedLastL1TxsNum)
+	assert.Equal(t, batches[nBatches-1].ForgeL1TxsNum, fetchedLastL1TxsNum)
 }
 
 func TestBids(t *testing.T) {
 	const fromBlock int64 = 1
 	const toBlock int64 = 5
-	const bidsPerSlot = 5
 	// Prepare blocks in the DB
-	setTestBlocks(fromBlock, toBlock)
+	blocks := setTestBlocks(fromBlock, toBlock)
+	// Generate fake coordinators
+	const nCoords = 5
+	coords := test.GenCoordinators(nCoords, blocks)
 	// Generate fake bids
-	bids := make([]common.Bid, 0, (toBlock-fromBlock)*bidsPerSlot)
+	const nBids = 20
-	for i := fromBlock; i < toBlock; i++ {
+	bids := test.GenBids(nBids, blocks, coords)
-		for j := 0; j < bidsPerSlot; j++ {
-			bids = append(bids, common.Bid{
-				SlotNum:     common.SlotNum(i),
-				BidValue:    big.NewInt(int64(j)),
-				EthBlockNum: i,
-				ForgerAddr:  eth.BigToAddress(big.NewInt(int64(j))),
-			})
-		}
-	}
 	err := historyDB.addBids(bids)
 	assert.NoError(t, err)
 	// Fetch bids
-	var fetchedBids []*common.Bid
+	fetchedBids, err := historyDB.GetBids()
-	for i := fromBlock; i < toBlock; i++ {
-		fetchedBidsSlot, err := historyDB.GetBidsBySlot(common.SlotNum(i))
 	assert.NoError(t, err)
-		fetchedBids = append(fetchedBids, fetchedBidsSlot...)
-	}
 	// Compare fetched bids vs generated bids
 	for i, bid := range fetchedBids {
 		assert.Equal(t, bids[i], *bid)
 	}
 }
 
-// setTestBlocks WARNING: this will delete the blocks and recreate them
+func TestTokens(t *testing.T) {
-func setTestBlocks(from, to int64) {
+	const fromBlock int64 = 1
-	if from == 0 {
+	const toBlock int64 = 5
-		if err := historyDB.Reorg(from); err != nil {
+	// Prepare blocks in the DB
-			panic(err)
+	blocks := setTestBlocks(fromBlock, toBlock)
-		}
+	// Generate fake tokens
-	} else {
+	const nTokens = 5
-		if err := historyDB.Reorg(from - 1); err != nil {
+	tokens := test.GenTokens(nTokens, blocks)
-			panic(err)
+	err := historyDB.AddTokens(tokens)
+	assert.NoError(t, err)
+	// Update price of generated tokens without price
+	for i := 0; i < len(tokens); i++ {
+		if tokens[i].USD == 0 {
+			value := 3.33 + float32(i)
+			tokens[i].USD = value
+			err := historyDB.UpdateTokenValue(tokens[i].TokenID, value)
+			assert.NoError(t, err)
 		}
 	}
-	blocks := genBlocks(from, to)
+	// Fetch tokens
-	if err := addBlocks(blocks); err != nil {
+	fetchedTokens, err := historyDB.GetTokens()
-		panic(err)
+	assert.NoError(t, err)
+	// Compare fetched tokens vs generated tokens
+	// All the tokens should have USDUpdate setted by the DB trigger
+	for i, token := range fetchedTokens {
+		assert.Equal(t, tokens[i].TokenID, token.TokenID)
+		assert.Equal(t, tokens[i].EthBlockNum, token.EthBlockNum)
+		assert.Equal(t, tokens[i].EthAddr, token.EthAddr)
+		assert.Equal(t, tokens[i].Name, token.Name)
+		assert.Equal(t, tokens[i].Symbol, token.Symbol)
+		assert.Equal(t, tokens[i].USD, token.USD)
+		assert.Greater(t, int64(1*time.Second), int64(time.Since(token.USDUpdate)))
 	}
 }
 
-func genBlocks(from, to int64) []common.Block {
+func TestAccounts(t *testing.T) {
-	var blocks []common.Block
+	const fromBlock int64 = 1
-	for i := from; i < to; i++ {
+	const toBlock int64 = 5
-		blocks = append(blocks, common.Block{
+	// Prepare blocks in the DB
-			EthBlockNum: i,
+	blocks := setTestBlocks(fromBlock, toBlock)
-			Timestamp:   time.Now().Add(time.Second * 13).UTC(),
+	// Generate fake tokens
-			Hash:        eth.BigToHash(big.NewInt(int64(i))),
+	const nTokens = 5
-		})
+	tokens := test.GenTokens(nTokens, blocks)
+	err := historyDB.AddTokens(tokens)
+	assert.NoError(t, err)
+	// Generate fake batches
+	const nBatches = 10
+	batches := test.GenBatches(nBatches, blocks)
+	err = historyDB.AddBatches(batches)
+	assert.NoError(t, err)
+	// Generate fake accounts
+	const nAccounts = 3
+	accs := test.GenAccounts(nAccounts, 0, tokens, nil, batches)
+	err = historyDB.AddAccounts(accs)
+	assert.NoError(t, err)
+	// Fetch accounts
+	fetchedAccs, err := historyDB.GetAccounts()
+	assert.NoError(t, err)
+	// Compare fetched accounts vs generated accounts
+	for i, acc := range fetchedAccs {
+		assert.Equal(t, accs[i], *acc)
+	}
+}
+
+func TestTxs(t *testing.T) {
+	const fromBlock int64 = 1
+	const toBlock int64 = 5
+	// Prepare blocks in the DB
+	blocks := setTestBlocks(fromBlock, toBlock)
+	// Generate fake tokens
+	const nTokens = 5
+	const tokenValue = 1.23456
+	tokens := test.GenTokens(nTokens, blocks)
+	for i := 0; i < len(tokens); i++ {
+		tokens[i].USD = tokenValue
+	}
+	err := historyDB.AddTokens(tokens)
+	assert.NoError(t, err)
+	// Generate fake batches
+	const nBatches = 10
+	batches := test.GenBatches(nBatches, blocks)
+	err = historyDB.AddBatches(batches)
+	assert.NoError(t, err)
+	// Generate fake accounts
+	const nAccounts = 3
+	accs := test.GenAccounts(nAccounts, 0, tokens, nil, batches)
+	err = historyDB.AddAccounts(accs)
+	assert.NoError(t, err)
+	// Generate fake L1 txs
+	const nL1s = 30
+	_, l1txs := test.GenL1Txs(0, nL1s, 0, nil, accs, tokens, blocks, batches)
+	err = historyDB.AddL1Txs(l1txs)
+	assert.NoError(t, err)
+	// Generate fake L2 txs
+	const nL2s = 20
+	_, l2txs := test.GenL2Txs(0, nL2s, 0, nil, accs, tokens, blocks, batches)
+	err = historyDB.AddL2Txs(l2txs)
+	assert.NoError(t, err)
+	// Compare fetched txs vs generated txs.
+	for i := 0; i < len(l1txs); i++ {
+		tx := l1txs[i].Tx()
+		fetchedTx, err := historyDB.GetTx(tx.TxID)
+		assert.NoError(t, err)
+		tx.USD = tokenValue * tx.AmountFloat
+		if fetchedTx.USD > tx.USD {
+			assert.Less(t, 0.999, tx.USD/fetchedTx.USD)
+		} else {
+			assert.Less(t, 0.999, fetchedTx.USD/tx.USD)
+		}
+		tx.LoadAmountUSD = tokenValue * tx.LoadAmountFloat
+		if fetchedTx.LoadAmountUSD > tx.LoadAmountUSD {
+			assert.Less(t, 0.999, tx.LoadAmountUSD/fetchedTx.LoadAmountUSD)
+		} else {
+			assert.Less(t, 0.999, fetchedTx.LoadAmountUSD/tx.LoadAmountUSD)
+		}
+		tx.LoadAmountUSD = 0
+		tx.USD = 0
+		fetchedTx.LoadAmountUSD = 0
+		fetchedTx.USD = 0
+		assert.Equal(t, tx, fetchedTx)
+	}
+	for i := 0; i < len(l2txs); i++ {
+		tx := l2txs[i].Tx()
+		fetchedTx, err := historyDB.GetTx(tx.TxID)
+		assert.NoError(t, err)
+		tx.USD = tokenValue * tx.AmountFloat
+		if fetchedTx.USD > tx.USD {
+			assert.Less(t, 0.999, tx.USD/fetchedTx.USD)
+		} else {
+			assert.Less(t, 0.999, fetchedTx.USD/tx.USD)
+		}
+		if tx.Fee == 0 {
+			tx.FeeUSD = 0
+		} else if tx.Fee <= 32 {
+			tx.FeeUSD = tx.USD * math.Pow(10, -24+(float64(tx.Fee)/2))
+		} else if tx.Fee <= 223 {
+			tx.FeeUSD = tx.USD * math.Pow(10, -8+(0.041666666666667*(float64(tx.Fee)-32)))
+		} else {
+			tx.FeeUSD = tx.USD * math.Pow(10, float64(tx.Fee)-224)
+		}
+		if fetchedTx.FeeUSD > tx.FeeUSD {
+			assert.Less(t, 0.999, tx.FeeUSD/fetchedTx.FeeUSD)
+		} else if fetchedTx.FeeUSD < tx.FeeUSD {
+			assert.Less(t, 0.999, fetchedTx.FeeUSD/tx.FeeUSD)
+		}
+		tx.FeeUSD = 0
+		tx.USD = 0
+		fetchedTx.FeeUSD = 0
+		fetchedTx.USD = 0
+		assert.Equal(t, tx, fetchedTx)
+	}
+	// Test trigger: L1 integrity
+	// from_eth_addr can't be null
+	l1txs[0].FromEthAddr = ethCommon.Address{}
+	err = historyDB.AddL1Txs(l1txs)
+	assert.Error(t, err)
+	l1txs[0].FromEthAddr = ethCommon.BigToAddress(big.NewInt(int64(5)))
+	// from_bjj can't be null
+	l1txs[0].FromBJJ = nil
+	err = historyDB.AddL1Txs(l1txs)
+	assert.Error(t, err)
+	privK := babyjub.NewRandPrivKey()
+	l1txs[0].FromBJJ = privK.Public()
+	// load_amount can't be null
+	l1txs[0].LoadAmount = nil
+	err = historyDB.AddL1Txs(l1txs)
+	assert.Error(t, err)
+	// Test trigger: L2 integrity
+	// batch_num can't be null
+	l2txs[0].BatchNum = 0
+	err = historyDB.AddL2Txs(l2txs)
+	assert.Error(t, err)
+	l2txs[0].BatchNum = 1
+	// nonce can't be null
+	l2txs[0].Nonce = 0
+	err = historyDB.AddL2Txs(l2txs)
+	assert.Error(t, err)
+}
+
+// setTestBlocks WARNING: this will delete the blocks and recreate them
+func setTestBlocks(from, to int64) []common.Block {
+	if err := cleanHistoryDB(); err != nil {
+		panic(err)
+	}
+	blocks := test.GenBlocks(from, to)
+	if err := historyDB.AddBlocks(blocks); err != nil {
+		panic(err)
 	}
 	return blocks
 }
 
-// addBlocks insert blocks into the DB. TODO: move method to test
+func cleanHistoryDB() error {
-func addBlocks(blocks []common.Block) error {
+	return historyDB.Reorg(0)
-	return db.BulkInsert(
-		historyDB.db,
-		"INSERT INTO block (eth_block_num, timestamp, hash) VALUES %s",
-		blocks[:],
-	)
 }

db/historydb/migrations/001_init.sql

@@ -27,6 +27,7 @@ CREATE TABLE batch (
 
 CREATE TABLE exit_tree (
     batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE,
+    withdrawn BIGINT REFERENCES batch (batch_num) ON DELETE SET NULL,
     account_idx BIGINT,
     merkle_proof BYTEA NOT NULL,
     balance NUMERIC NOT NULL,
@@ -34,17 +35,9 @@ CREATE TABLE exit_tree (
     PRIMARY KEY (batch_num, account_idx)
 );
 
-CREATE TABLE withdrawal (
-    batch_num BIGINT,
-    account_idx BIGINT,
-    eth_block_num BIGINT REFERENCES block (eth_block_num) ON DELETE CASCADE,
-    FOREIGN KEY (batch_num, account_idx) REFERENCES exit_tree (batch_num, account_idx) ON DELETE CASCADE,
-    PRIMARY KEY (batch_num, account_idx)
-);
-
 CREATE TABLE bid (
     slot_num BIGINT NOT NULL,
-    bid_value BYTEA NOT NULL, -- (check if we can do a max(), if not add float for order purposes)
+    bid_value BYTEA NOT NULL,
     eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
     forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator
     PRIMARY KEY (slot_num, bid_value)
@@ -56,40 +49,124 @@ CREATE TABLE token (
     eth_addr BYTEA UNIQUE NOT NULL,
     name VARCHAR(20) NOT NULL,
     symbol VARCHAR(10) NOT NULL,
-    decimals INT NOT NULL
+    decimals INT NOT NULL,
+    usd NUMERIC,
+    usd_update TIMESTAMP
 );
 
-CREATE TABLE l1tx (
+-- +migrate StatementBegin
-    tx_id BYTEA PRIMARY KEY,
+CREATE FUNCTION set_token_usd_update() 
-    to_forge_l1_txs_num BIGINT NOT NULL,
+    RETURNS TRIGGER 
+AS 
+$BODY$
+BEGIN
+    IF NEW."usd" IS NOT NULL AND NEW."usd_update" IS NULL THEN
+        NEW."usd_update" = timezone('utc', now());
+    END IF;
+    RETURN NEW;
+END;
+$BODY$
+LANGUAGE plpgsql;
+-- +migrate StatementEnd
+CREATE TRIGGER trigger_token_usd_update BEFORE UPDATE OR INSERT ON token
+FOR EACH ROW EXECUTE PROCEDURE set_token_usd_update();
+
+CREATE TABLE tx (
+    -- Generic TX
+    is_l1 BOOLEAN NOT NULL,
+    id BYTEA PRIMARY KEY,
+    type VARCHAR(40) NOT NULL,
     position INT NOT NULL,
-    user_origin BOOLEAN NOT NULL,
     from_idx BIGINT NOT NULL,
-    from_eth_addr BYTEA NOT NULL,
-    from_bjj BYTEA NOT NULL,
     to_idx BIGINT NOT NULL,
+    amount BYTEA NOT NULL,
+    amount_f NUMERIC NOT NULL,
     token_id INT NOT NULL REFERENCES token (token_id),
-    amount NUMERIC NOT NULL,
+    amount_usd NUMERIC, -- Value of the amount in USD at the moment the tx was inserted in the DB
-    load_amount BYTEA NOT NULL,
+    batch_num BIGINT REFERENCES batch (batch_num) ON DELETE SET NULL, -- Can be NULL in the case of L1 txs that are on the queue but not forged yet.
     eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
-    tx_type VARCHAR(40) NOT NULL
+    -- L1
+    to_forge_l1_txs_num BIGINT,
+    user_origin BOOLEAN,
+    from_eth_addr BYTEA,
+    from_bjj BYTEA,
+    load_amount BYTEA,
+    load_amount_f NUMERIC,
+    load_amount_usd NUMERIC,
+    -- L2
+    fee INT,
+    fee_usd NUMERIC,
+    nonce BIGINT
 );
 
-CREATE TABLE l2tx (
+-- +migrate StatementBegin
-    tx_id BYTEA PRIMARY KEY,
+CREATE FUNCTION set_tx()
-    batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
+    RETURNS TRIGGER 
-    position INT NOT NULL,
+AS 
-    from_idx BIGINT NOT NULL,
+$BODY$
-    to_idx BIGINT NOT NULL,
+DECLARE token_value NUMERIC := (SELECT usd FROM token WHERE token_id = NEW.token_id);
-    amount NUMERIC NOT NULL,
+BEGIN
-    fee INT NOT NULL,
+    -- Validate L1/L2 constrains
-    nonce BIGINT NOT NULL,
+    IF NEW.is_l1 AND (( -- L1 mandatory fields
-    tx_type VARCHAR(40) NOT NULL
+        NEW.user_origin IS NULL OR
-);
+        NEW.from_eth_addr IS NULL OR
+        NEW.from_bjj IS NULL OR
+        NEW.load_amount IS NULL OR
+        NEW.load_amount_f IS NULL
+    ) OR (NOT NEW.user_origin AND NEW.batch_num IS NULL)) THEN -- If is Coordinator L1, must include batch_num
+        RAISE EXCEPTION 'Invalid L1 tx.';
+    ELSIF NOT NEW.is_l1 THEN
+        IF NEW.fee IS NULL THEN
+            NEW.fee = (SELECT 0);
+        END IF;
+        IF NEW.batch_num IS NULL OR NEW.nonce IS NULL THEN
+            RAISE EXCEPTION 'Invalid L2 tx.';
+        END IF;
+    END IF;
+    -- If is L2, add token_id
+    IF NEW.token_id IS NULL THEN
+        NEW."token_id" = (SELECT token_id FROM account WHERE idx = NEW."from_idx");
+    END IF;
+    -- Set value_usd
+    NEW."amount_usd" = (SELECT token_value * NEW.amount_f);
+    NEW."load_amount_usd" = (SELECT token_value * NEW.load_amount_f);
+    IF NOT NEW.is_l1 THEN
+        NEW."fee_usd" = (SELECT token_value * NEW.amount_f * CASE
+            WHEN NEW.fee = 0 THEN 0	
+            WHEN NEW.fee >= 1 AND NEW.fee <= 32 THEN POWER(10,-24+(NEW.fee::float/2))	
+            WHEN NEW.fee >= 33 AND NEW.fee <= 223 THEN POWER(10,-8+(0.041666666666667*(NEW.fee::float-32)))	
+            WHEN NEW.fee >= 224 AND NEW.fee <= 255 THEN POWER(10,NEW.fee-224) END);
+    END IF;
+    RETURN NEW;
+END;
+$BODY$
+LANGUAGE plpgsql;
+-- +migrate StatementEnd
+CREATE TRIGGER trigger_set_tx BEFORE INSERT ON tx
+FOR EACH ROW EXECUTE PROCEDURE set_tx();
+
+-- +migrate StatementBegin
+CREATE FUNCTION forge_l1_user_txs() 
+    RETURNS TRIGGER 
+AS 
+$BODY$
+BEGIN
+    IF NEW.forge_l1_txs_num IS NOT NULL THEN
+        UPDATE tx 
+        SET batch_num = NEW.batch_num
+        WHERE user_origin AND NEW.forge_l1_txs_num = to_forge_l1_txs_num;
+    END IF;
+    RETURN NEW;
+END;
+$BODY$
+LANGUAGE plpgsql;
+-- +migrate StatementEnd
+CREATE TRIGGER trigger_forge_l1_txs AFTER INSERT ON batch
+FOR EACH ROW EXECUTE PROCEDURE forge_l1_user_txs();
 
 CREATE TABLE account (
     idx BIGINT PRIMARY KEY,
-    token_id INT NOT NULL REFERENCES token (token_id),
+    token_id INT NOT NULL REFERENCES token (token_id) ON DELETE CASCADE,
     batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
     bjj BYTEA NOT NULL,
     eth_addr BYTEA NOT NULL
@@ -113,7 +190,7 @@ CREATE TABLE consensus_vars (
     outbidding INT NOT NULL,
     donation_address BYTEA NOT NULL,
     governance_address BYTEA NOT NULL,
-    allocation_ratio vARCHAR(200)
+    allocation_ratio VARCHAR(200)
 );
 
 -- +migrate Down

test/historydb.go

@@ -0,0 +1,343 @@
+package test
+
+import (
+	"errors"
+	"fmt"
+	"math/big"
+	"strconv"
+	"time"
+
+	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/iden3/go-iden3-crypto/babyjub"
+)
+
+// WARNING: the generators in this file doesn't necessary follow the protocol
+// they are intended to check that the parsers between struct <==> DB are correct
+
+// GenBlocks generates block from, to block numbers. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenBlocks(from, to int64) []common.Block {
+	var blocks []common.Block
+	for i := from; i < to; i++ {
+		blocks = append(blocks, common.Block{
+			EthBlockNum: i,
+			//nolint:gomnd
+			Timestamp: time.Now().Add(time.Second * 13).UTC(),
+			Hash:      ethCommon.BigToHash(big.NewInt(int64(i))),
+		})
+	}
+	return blocks
+}
+
+// GenTokens generates tokens. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenTokens(nTokens int, blocks []common.Block) []common.Token {
+	tokens := []common.Token{}
+	for i := 0; i < nTokens; i++ {
+		token := common.Token{
+			TokenID:     common.TokenID(i),
+			Name:        fmt.Sprint(i),
+			Symbol:      fmt.Sprint(i),
+			Decimals:    uint64(i),
+			EthBlockNum: blocks[i%len(blocks)].EthBlockNum,
+			EthAddr:     ethCommon.BigToAddress(big.NewInt(int64(i))),
+		}
+		if i%2 == 0 {
+			token.USD = 3
+			token.USDUpdate = time.Now()
+		}
+		tokens = append(tokens, token)
+	}
+	return tokens
+}
+
+// GenBatches generates batches. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenBatches(nBatches int, blocks []common.Block) []common.Batch {
+	batches := []common.Batch{}
+	collectedFees := make(map[common.TokenID]*big.Int)
+	for i := 0; i < 64; i++ {
+		collectedFees[common.TokenID(i)] = big.NewInt(int64(i))
+	}
+	for i := 0; i < nBatches; i++ {
+		batch := common.Batch{
+			BatchNum:    common.BatchNum(i + 1),
+			EthBlockNum: blocks[i%len(blocks)].EthBlockNum,
+			//nolint:gomnd
+			ForgerAddr:    ethCommon.BigToAddress(big.NewInt(6886723)),
+			CollectedFees: collectedFees,
+			StateRoot:     common.Hash([]byte("duhdqlwiucgwqeiu")),
+			//nolint:gomnd
+			NumAccounts: 30,
+			ExitRoot:    common.Hash([]byte("tykertheuhtgenuer3iuw3b")),
+			SlotNum:     common.SlotNum(i),
+		}
+		if i%2 == 0 {
+			batch.ForgeL1TxsNum = uint32(i)
+		}
+		batches = append(batches, batch)
+	}
+	return batches
+}
+
+// GenAccounts generates accounts. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenAccounts(totalAccounts, userAccounts int, tokens []common.Token, userAddr *ethCommon.Address, batches []common.Batch) []common.Account {
+	if totalAccounts < userAccounts {
+		panic("totalAccounts must be greater than userAccounts")
+	}
+	privK := babyjub.NewRandPrivKey()
+	pubK := privK.Public()
+	accs := []common.Account{}
+	for i := 0; i < totalAccounts; i++ {
+		var addr ethCommon.Address
+		if i < userAccounts {
+			addr = *userAddr
+		} else {
+			addr = ethCommon.BigToAddress(big.NewInt(int64(i)))
+		}
+		accs = append(accs, common.Account{
+			Idx:       common.Idx(i),
+			TokenID:   tokens[i%len(tokens)].TokenID,
+			EthAddr:   addr,
+			BatchNum:  batches[i%len(batches)].BatchNum,
+			PublicKey: pubK,
+		})
+	}
+	return accs
+}
+
+// GenL1Txs generates L1 txs. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenL1Txs(
+	fromIdx int,
+	totalTxs, nUserTxs int,
+	userAddr *ethCommon.Address,
+	accounts []common.Account,
+	tokens []common.Token,
+	blocks []common.Block,
+	batches []common.Batch,
+) ([]common.L1Tx, []common.L1Tx) {
+	if totalTxs < nUserTxs {
+		panic("totalTxs must be greater than userTxs")
+	}
+	userTxs := []common.L1Tx{}
+	othersTxs := []common.L1Tx{}
+	for i := 0; i < totalTxs; i++ {
+		var tx common.L1Tx
+		if batches[i%len(batches)].ForgeL1TxsNum != 0 {
+			tx = common.L1Tx{
+				TxID:            common.TxID(common.Hash([]byte("L1_" + strconv.Itoa(fromIdx+i)))),
+				ToForgeL1TxsNum: batches[i%len(batches)].ForgeL1TxsNum,
+				Position:        i,
+				UserOrigin:      i%2 == 0,
+				TokenID:         tokens[i%len(tokens)].TokenID,
+				Amount:          big.NewInt(int64(i + 1)),
+				LoadAmount:      big.NewInt(int64(i + 1)),
+				EthBlockNum:     blocks[i%len(blocks)].EthBlockNum,
+				Type:            randomTxType(i),
+			}
+			if i%4 == 0 {
+				tx.BatchNum = batches[i%len(batches)].BatchNum
+			}
+		} else {
+			continue
+		}
+		if i < nUserTxs {
+			var from, to common.Account
+			var err error
+			if i%2 == 0 {
+				from, err = randomAccount(i, true, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+				to, err = randomAccount(i, false, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+			} else {
+				from, err = randomAccount(i, false, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+				to, err = randomAccount(i, true, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+			}
+			tx.FromIdx = from.Idx
+			tx.FromEthAddr = from.EthAddr
+			tx.FromBJJ = from.PublicKey
+			tx.ToIdx = to.Idx
+			userTxs = append(userTxs, tx)
+		} else {
+			from, err := randomAccount(i, false, userAddr, accounts)
+			if err != nil {
+				panic(err)
+			}
+			to, err := randomAccount(i, false, userAddr, accounts)
+			if err != nil {
+				panic(err)
+			}
+			tx.FromIdx = from.Idx
+			tx.FromEthAddr = from.EthAddr
+			tx.FromBJJ = from.PublicKey
+			tx.ToIdx = to.Idx
+			othersTxs = append(othersTxs, tx)
+		}
+	}
+	return userTxs, othersTxs
+}
+
+// GenL2Txs generates L2 txs. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenL2Txs(
+	fromIdx int,
+	totalTxs, nUserTxs int,
+	userAddr *ethCommon.Address,
+	accounts []common.Account,
+	tokens []common.Token,
+	blocks []common.Block,
+	batches []common.Batch,
+) ([]common.L2Tx, []common.L2Tx) {
+	if totalTxs < nUserTxs {
+		panic("totalTxs must be greater than userTxs")
+	}
+	userTxs := []common.L2Tx{}
+	othersTxs := []common.L2Tx{}
+	for i := 0; i < totalTxs; i++ {
+		tx := common.L2Tx{
+			TxID:     common.TxID(common.Hash([]byte("L2_" + strconv.Itoa(fromIdx+i)))),
+			BatchNum: batches[i%len(batches)].BatchNum,
+			Position: i,
+			//nolint:gomnd
+			Amount: big.NewInt(int64(i + 1)),
+			//nolint:gomnd
+			Fee:         common.FeeSelector(i % 256),
+			Nonce:       common.Nonce(i + 1),
+			EthBlockNum: blocks[i%len(blocks)].EthBlockNum,
+			Type:        randomTxType(i),
+		}
+		if i < nUserTxs {
+			var from, to common.Account
+			var err error
+			if i%2 == 0 {
+				from, err = randomAccount(i, true, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+				to, err = randomAccount(i, false, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+			} else {
+				from, err = randomAccount(i, false, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+				to, err = randomAccount(i, true, userAddr, accounts)
+				if err != nil {
+					panic(err)
+				}
+			}
+			tx.FromIdx = from.Idx
+			tx.ToIdx = to.Idx
+			userTxs = append(userTxs, tx)
+		} else {
+			from, err := randomAccount(i, false, userAddr, accounts)
+			if err != nil {
+				panic(err)
+			}
+			to, err := randomAccount(i, false, userAddr, accounts)
+			if err != nil {
+				panic(err)
+			}
+			tx.FromIdx = from.Idx
+			tx.ToIdx = to.Idx
+			othersTxs = append(othersTxs, tx)
+		}
+	}
+	return userTxs, othersTxs
+}
+
+// GenCoordinators generates coordinators. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenCoordinators(nCoords int, blocks []common.Block) []common.Coordinator {
+	coords := []common.Coordinator{}
+	for i := 0; i < nCoords; i++ {
+		coords = append(coords, common.Coordinator{
+			EthBlockNum: blocks[i%len(blocks)].EthBlockNum,
+			Forger:      ethCommon.BigToAddress(big.NewInt(int64(i))),
+			Withdraw:    ethCommon.BigToAddress(big.NewInt(int64(i))),
+			URL:         "https://foo.bar",
+		})
+	}
+	return coords
+}
+
+// GenBids generates bids. WARNING: This is meant for DB/API testing, and may not be fully consistent with the protocol.
+func GenBids(nBids int, blocks []common.Block, coords []common.Coordinator) []common.Bid {
+	bids := []common.Bid{}
+	for i := 0; i < nBids; i++ {
+		bids = append(bids, common.Bid{
+			SlotNum:     common.SlotNum(i),
+			BidValue:    big.NewInt(int64(i)),
+			EthBlockNum: blocks[i%len(blocks)].EthBlockNum,
+			ForgerAddr:  coords[i%len(blocks)].Forger,
+		})
+	}
+	return bids
+}
+
+func randomAccount(seed int, userAccount bool, userAddr *ethCommon.Address, accs []common.Account) (common.Account, error) {
+	i := seed % len(accs)
+	firstI := i
+	for {
+		acc := accs[i]
+		if userAccount && *userAddr == acc.EthAddr {
+			return acc, nil
+		}
+		if !userAccount && (userAddr == nil || *userAddr != acc.EthAddr) {
+			return acc, nil
+		}
+		i++
+		i = i % len(accs)
+		if i == firstI {
+			return acc, errors.New("Didnt found any account matchinng the criteria")
+		}
+	}
+}
+
+func randomTxType(seed int) common.TxType {
+	//nolint:gomnd
+	switch seed % 11 {
+	case 0:
+		return common.TxTypeExit
+	//nolint:gomnd
+	case 1:
+		return common.TxTypeWithdrawn
+	//nolint:gomnd
+	case 2:
+		return common.TxTypeTransfer
+	//nolint:gomnd
+	case 3:
+		return common.TxTypeDeposit
+	//nolint:gomnd
+	case 4:
+		return common.TxTypeCreateAccountDeposit
+	//nolint:gomnd
+	case 5:
+		return common.TxTypeCreateAccountDepositTransfer
+	//nolint:gomnd
+	case 6:
+		return common.TxTypeDepositTransfer
+	//nolint:gomnd
+	case 7:
+		return common.TxTypeForceTransfer
+	//nolint:gomnd
+	case 8:
+		return common.TxTypeForceExit
+	//nolint:gomnd
+	case 9:
+		return common.TxTypeTransferToEthAddr
+	//nolint:gomnd
+	case 10:
+		return common.TxTypeTransferToBJJ
+	default:
+		return common.TxTypeTransfer
+	}
+}