Use til for batches and blocks at historydb

api/api_test.go

@@ -71,6 +71,16 @@ var api *API
 // emulating the task of the synchronizer in order to have data to be returned
 // by the API endpoints that will be tested
 func TestMain(m *testing.M) {
+	/*
+		til update considerations:
+		1. Two instructions sets should be enough (one for L2 another for historydb)
+		2. FillBlocksExtra function must be used, there is a coment on top of the function that explains which data is setted
+		3. Some data will not be generated by til nor FillBlocksExtra, test.GenXXX will still be required to cover this cases
+		4. Most of the historydb inserts should be replaced with nBlocks calls to AddBlockSCData
+		5. When defining til instructions, there is no need to have 100s of entries for each table, but it's interesting to
+		   cover all different cases (for instance all tx types)
+	*/
+
 	// Initializations
 	// Swagger
 	router := swagger.NewRouter().WithSwaggerFromFile("./swagger.yml")
@@ -132,6 +142,7 @@ func TestMain(m *testing.M) {
 	// Fill HistoryDB and StateDB with fake data
 	// Gen blocks and add them to DB
 	const nBlocks = 5
+	// TODO: UPDATE with til
 	blocks := test.GenBlocks(1, nBlocks+1)
 	err = api.h.AddBlocks(blocks)
 	if err != nil {
@@ -141,6 +152,7 @@ func TestMain(m *testing.M) {
 
 	// Gen tokens and add them to DB
 	const nTokens = 10
+	// TODO: UPDATE with til
 	tokens, ethToken := test.GenTokens(nTokens, blocks)
 	err = api.h.AddTokens(tokens)
 	if err != nil {
@@ -173,6 +185,7 @@ func TestMain(m *testing.M) {
 	}
 	// Gen batches and add them to DB
 	const nBatches = 10
+	// TODO: UPDATE with til
 	batches := test.GenBatches(nBatches, blocks)
 	err = api.h.AddBatches(batches)
 	if err != nil {
@@ -184,6 +197,7 @@ func TestMain(m *testing.M) {
 	usrAddr := ethCommon.BigToAddress(big.NewInt(4896847))
 	privK := babyjub.NewRandPrivKey()
 	usrBjj := privK.Public()
+	// TODO: UPDATE with til
 	accs := test.GenAccounts(totalAccounts, userAccounts, tokens, &usrAddr, usrBjj, batches)
 	err = api.h.AddAccounts(accs)
 	if err != nil {
@@ -207,6 +221,7 @@ func TestMain(m *testing.M) {
 	}
 	// Gen exits and add them to DB
 	const totalExits = 40
+	// TODO: UPDATE with til
 	exits := test.GenExitTree(totalExits, batches, accs)
 	err = api.h.AddExitTree(exits)
 	if err != nil {
@@ -217,10 +232,12 @@ func TestMain(m *testing.M) {
 	// Gen L1Txs
 	const totalL1Txs = 40
 	const userL1Txs = 4
+	// TODO: UPDATE with til
 	usrL1Txs, othrL1Txs := test.GenL1Txs(256, totalL1Txs, userL1Txs, &usrAddr, accs, tokens, blocks, batches)
 	// Gen L2Txs
 	const totalL2Txs = 20
 	const userL2Txs = 4
+	// TODO: UPDATE with til
 	usrL2Txs, othrL2Txs := test.GenL2Txs(256+totalL1Txs, totalL2Txs, userL2Txs, &usrAddr, accs, tokens, blocks, batches)
 	// Sort txs
 	sortedTxs := []txSortFielder{}

db/historydb/historydb.go

@@ -1109,16 +1109,6 @@ func (hdb *HistoryDB) GetExitsAPI(
 	return db.SlicePtrsToSlice(exits).([]ExitAPI), exits[0].TotalItems - uint64(len(exits)), nil
 }
 
-// // GetTx returns a tx from the DB
-// func (hdb *HistoryDB) GetTx(txID common.TxID) (*common.Tx, error) {
-// 	tx := new(common.Tx)
-// 	return tx, meddler.QueryRow(
-// 		hdb.db, tx,
-// 		"SELECT * FROM tx WHERE id = $1;",
-// 		txID,
-// 	)
-// }
-
 // GetAllL1UserTxs returns all L1UserTxs from the DB
 func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) {
 	var txs []*common.L1Tx

db/historydb/historydb_test.go

@@ -50,27 +50,44 @@ func TestMain(m *testing.M) {
 
 func TestBlocks(t *testing.T) {
 	var fromBlock, toBlock int64
-	fromBlock = 1
+	fromBlock = 0
-	toBlock = 5
+	toBlock = 7
-	// Delete peviously created rows (clean previous test execs)
+	// Reset DB
 	test.WipeDB(historyDB.DB())
-	// Generate fake blocks
+	// Generate blocks using til
-	blocks := test.GenBlocks(fromBlock, toBlock)
+	set1 := `
+		Type: Blockchain
+		// block 0 is stored as default in the DB
+		// block 1 does not exist
+		> block // blockNum=2
+		> block // blockNum=3 
+		> block // blockNum=4
+		> block // blockNum=5
+		> block // blockNum=6
+	`
+	tc := til.NewContext(1)
+	blocks, err := tc.GenerateBlocks(set1)
+	require.NoError(t, err)
 	// Save timestamp of a block with UTC and change it without UTC
 	timestamp := time.Now().Add(time.Second * 13)
-	blocks[fromBlock].Timestamp = timestamp
+	blocks[fromBlock].Block.Timestamp = timestamp
 	// Insert blocks into DB
 	for i := 0; i < len(blocks); i++ {
-		err := historyDB.AddBlock(&blocks[i])
+		err := historyDB.AddBlock(&blocks[i].Block)
 		assert.NoError(t, err)
 	}
+	// Add block 0 to the generated blocks
+	blocks = append(
+		[]common.BlockData{common.BlockData{Block: test.Block0}}, //nolint:gofmt
+		blocks...,
+	)
 	// Get all blocks from DB
 	fetchedBlocks, err := historyDB.GetBlocks(fromBlock, toBlock)
 	assert.Equal(t, len(blocks), len(fetchedBlocks))
 	// Compare generated vs getted blocks
 	assert.NoError(t, err)
 	for i := range fetchedBlocks {
-		assertEqualBlock(t, &blocks[i], &fetchedBlocks[i])
+		assertEqualBlock(t, &blocks[i].Block, &fetchedBlocks[i])
 	}
 	// Compare saved timestamp vs getted
 	nameZoneUTC, offsetUTC := timestamp.UTC().Zone()
@@ -78,15 +95,15 @@ func TestBlocks(t *testing.T) {
 	assert.Equal(t, nameZoneUTC, zoneFetchedBlock)
 	assert.Equal(t, offsetUTC, offsetFetchedBlock)
 	// Get blocks from the DB one by one
-	for i := fromBlock; i < toBlock; i++ {
+	for i := int64(2); i < toBlock; i++ { // avoid block 0 for simplicity
 		fetchedBlock, err := historyDB.GetBlock(i)
 		assert.NoError(t, err)
-		assertEqualBlock(t, &blocks[i-1], fetchedBlock)
+		assertEqualBlock(t, &blocks[i-1].Block, fetchedBlock)
 	}
 	// Get last block
 	lastBlock, err := historyDB.GetLastBlock()
 	assert.NoError(t, err)
-	assertEqualBlock(t, &blocks[len(blocks)-1], lastBlock)
+	assertEqualBlock(t, &blocks[len(blocks)-1].Block, lastBlock)
 }
 
 func assertEqualBlock(t *testing.T, expected *common.Block, actual *common.Block) {
@@ -96,23 +113,81 @@ func assertEqualBlock(t *testing.T, expected *common.Block, actual *common.Block
 }
 
 func TestBatches(t *testing.T) {
-	const fromBlock int64 = 1
+	// Reset DB
-	const toBlock int64 = 3
+	test.WipeDB(historyDB.DB())
-	// Prepare blocks in the DB
+	// Generate batches using til (and blocks for foreign key)
-	blocks := setTestBlocks(fromBlock, toBlock)
+	set := `
-	// Generate fake batches
+		Type: Blockchain
-	const nBatches = 9
+		
-	batches := test.GenBatches(nBatches, blocks)
+		AddToken(1) // Will have value in USD
-	// Test GetLastL1TxsNum with no batches
+		AddToken(2) // Will NOT have value in USD
-	fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum()
+		CreateAccountDeposit(1) A: 2000
-	assert.NoError(t, err)
+		CreateAccountDeposit(2) A: 2000
-	assert.Nil(t, fetchedLastL1TxsNum)
+		CreateAccountDeposit(1) B: 1000
-	// Add batches to the DB
+		CreateAccountDeposit(2) B: 1000
-	err = historyDB.AddBatches(batches)
+		> batchL1 
-	assert.NoError(t, err)
+		> batchL1 
+		Transfer(1) A-B: 100 (5)
+		Transfer(2) B-A: 100 (199)
+		> batch   // batchNum=2, L2 only batch, forges transfers (mixed case of with(out) USD value)
+		> block
+		Transfer(1) A-B: 100 (5)
+		> batch   // batchNum=3, L2 only batch, forges transfer (with USD value)
+		Transfer(2) B-A: 100 (199)
+		> batch   // batchNum=4, L2 only batch, forges transfer (without USD value)
+		> block
+	`
+	tc := til.NewContext(common.RollupConstMaxL1UserTx)
+	tilCfgExtra := til.ConfigExtra{
+		BootCoordAddr: ethCommon.HexToAddress("0xE39fEc6224708f0772D2A74fd3f9055A90E0A9f2"),
+		CoordUser:     "A",
+	}
+	blocks, err := tc.GenerateBlocks(set)
+	require.Nil(t, err)
+	err = tc.FillBlocksExtra(blocks, &tilCfgExtra)
+	assert.Nil(t, err)
+	// Insert to DB
+	batches := []common.Batch{}
+	tokensValue := make(map[common.TokenID]float64)
+	lastL1TxsNum := new(int64)
+	for _, block := range blocks {
+		// Insert block
+		assert.NoError(t, historyDB.AddBlock(&block.Block))
+		// Insert tokens
+		for i, token := range block.Rollup.AddedTokens {
+			assert.NoError(t, historyDB.AddToken(&token)) //nolint:gosec
+			if i%2 != 0 {
+				// Set value to the token
+				value := (float64(i) + 5) * 5.389329
+				assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
+				tokensValue[token.TokenID] = value / math.Pow(10, float64(token.Decimals))
+			}
+		}
+		// Combine all generated batches into single array
+		for _, batch := range block.Rollup.Batches {
+			batches = append(batches, batch.Batch)
+			forgeTxsNum := batch.Batch.ForgeL1TxsNum
+			if forgeTxsNum != nil && (lastL1TxsNum == nil || *lastL1TxsNum < *forgeTxsNum) {
+				*lastL1TxsNum = *forgeTxsNum
+			}
+		}
+	}
+	// Insert batches
+	assert.NoError(t, historyDB.AddBatches(batches))
+	// Set expected total fee
+	for _, batch := range batches {
+		total := .0
+		for tokenID, amount := range batch.CollectedFees {
+			af := new(big.Float).SetInt(amount)
+			amountFloat, _ := af.Float64()
+			total += tokensValue[tokenID] * amountFloat
+		}
+		batch.TotalFeesUSD = &total
+	}
 	// Get batches from the DB
-	fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(nBatches))
+	fetchedBatches, err := historyDB.GetBatches(0, common.BatchNum(len(batches)+1))
 	assert.NoError(t, err)
+	assert.Equal(t, len(batches), len(fetchedBatches))
 	for i, fetchedBatch := range fetchedBatches {
 		assert.Equal(t, batches[i], fetchedBatch)
 	}
@@ -121,38 +196,9 @@ func TestBatches(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, batches[len(batches)-1].BatchNum, fetchedLastBatchNum)
 	// Test GetLastL1TxsNum
-	fetchedLastL1TxsNum, err = historyDB.GetLastL1TxsNum()
+	fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum()
 	assert.NoError(t, err)
-	assert.Equal(t, *batches[nBatches-1].ForgeL1TxsNum, *fetchedLastL1TxsNum)
+	assert.Equal(t, lastL1TxsNum, fetchedLastL1TxsNum)
-	// Test total fee
-	// Generate fake tokens
-	const nTokens = 5
-	tokens, ethToken := test.GenTokens(nTokens, blocks)
-	err = historyDB.AddTokens(tokens)
-	assert.NoError(t, err)
-	tokens = append([]common.Token{ethToken}, tokens...)
-	feeBatch := batches[0]
-	feeBatch.BatchNum = 9999
-	feeBatch.CollectedFees = make(map[common.TokenID]*big.Int)
-	var total float64
-	for i, token := range tokens {
-		value := 3.019237 * float64(i)
-		assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
-		bigAmount := big.NewInt(345000000)
-		feeBatch.CollectedFees[token.TokenID] = bigAmount
-		f := new(big.Float).SetInt(bigAmount)
-		amount, _ := f.Float64()
-		total += value * (amount / math.Pow(10, float64(token.Decimals)))
-	}
-	err = historyDB.AddBatch(&feeBatch)
-	assert.NoError(t, err)
-	fetchedBatches, err = historyDB.GetBatches(feeBatch.BatchNum-1, feeBatch.BatchNum+1)
-	assert.NoError(t, err)
-	for _, fetchedBatch := range fetchedBatches {
-		if fetchedBatch.BatchNum == feeBatch.BatchNum {
-			assert.Equal(t, total, *fetchedBatch.TotalFeesUSD)
-		}
-	}
 }
 
 func TestBids(t *testing.T) {
@@ -257,6 +303,32 @@ func TestAccounts(t *testing.T) {
 }
 
 func TestTxs(t *testing.T) {
+	// Reset DB
+	test.WipeDB(historyDB.DB())
+	// TODO: Generate batches using til (and blocks for foreign key)
+	set := `
+		Type: Blockchain
+
+		// Things to test:
+		// One tx of each type
+		// batches that forge user L1s
+		// historic USD is not set if USDUpdate is too old (24h)
+	`
+	tc := til.NewContext(common.RollupConstMaxL1UserTx)
+	tilCfgExtra := til.ConfigExtra{
+		BootCoordAddr: ethCommon.HexToAddress("0xE39fEc6224708f0772D2A74fd3f9055A90E0A9f2"),
+		CoordUser:     "A",
+	}
+	blocks, err := tc.GenerateBlocks(set)
+	require.Nil(t, err)
+	err = tc.FillBlocksExtra(blocks, &tilCfgExtra)
+	assert.Nil(t, err)
+
+	/*
+
+		OLD TEST
+
+
 		const fromBlock int64 = 1
 		const toBlock int64 = 5
 		// Prepare blocks in the DB
@@ -278,10 +350,9 @@ func TestTxs(t *testing.T) {
 		err = historyDB.AddAccounts(accs)
 		assert.NoError(t, err)
 
-	/*
 			Uncomment once the transaction generation is fixed
 			!! test that batches that forge user L1s !!
-		!! Missing tests to check that historic USD is not set if USDUpdate is too old (24h) !!
+			!! Missing tests to check that  !!
 
 			// Generate fake L1 txs
 			const nL1s = 64

db/l2db/l2db_test.go

@@ -84,6 +84,7 @@ func TestAddTxTest(t *testing.T) {
 	test.WipeDB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts, tokens)
 	for _, tx := range txs {
+		// TODO: UPDATE with til
 		err := l2DB.AddTxTest(tx)
 		assert.NoError(t, err)
 		fetchedTx, err := l2DB.GetTx(tx.TxID)
@@ -123,21 +124,23 @@ func assertTx(t *testing.T, expected, actual *common.PoolL2Tx) {
 	assert.Equal(t, expected, actual)
 }
 
-func BenchmarkAddTxTest(b *testing.B) {
+// NO UPDATE: benchmarks will be done after impl is finished
-	const nInserts = 20
+// func BenchmarkAddTxTest(b *testing.B) {
-	test.WipeDB(l2DB.DB())
+// 	const nInserts = 20
-	txs := test.GenPoolTxs(nInserts, tokens)
+// 	test.WipeDB(l2DB.DB())
-	now := time.Now()
+// 	txs := test.GenPoolTxs(nInserts, tokens)
-	for _, tx := range txs {
+// 	now := time.Now()
-		_ = l2DB.AddTxTest(tx)
+// 	for _, tx := range txs {
-	}
+// 		_ = l2DB.AddTxTest(tx)
-	elapsedTime := time.Since(now)
+// 	}
-	log.Info("Time to insert 2048 txs:", elapsedTime)
+// 	elapsedTime := time.Since(now)
-}
+// 	log.Info("Time to insert 2048 txs:", elapsedTime)
+// }
 
 func TestGetPending(t *testing.T) {
 	const nInserts = 20
 	test.WipeDB(l2DB.DB())
+	// TODO: UPDATE with til
 	txs := test.GenPoolTxs(nInserts, tokens)
 	var pendingTxs []*common.PoolL2Tx
 	for _, tx := range txs {
@@ -156,7 +159,7 @@ func TestGetPending(t *testing.T) {
 }
 
 /*
-	WARNING: this should be fixed once transaktio is ready
+		TODO: update with til
 func TestStartForging(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
@@ -188,7 +191,7 @@ func TestStartForging(t *testing.T) {
 */
 
 /*
-	WARNING: this should be fixed once transaktio is ready
+		TODO: update with til
 func TestDoneForging(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
@@ -220,7 +223,7 @@ func TestDoneForging(t *testing.T) {
 */
 
 /*
-	WARNING: this should be fixed once transaktio is ready
+		TODO: update with til
 func TestInvalidate(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
@@ -252,7 +255,10 @@ func TestInvalidate(t *testing.T) {
 */
 
 /*
-	WARNING: this should be fixed once transaktio is ready
+
+		TODO: update with til
+
+
 func TestCheckNonces(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
@@ -303,6 +309,7 @@ func TestReorg(t *testing.T) {
 	const lastValidBatch common.BatchNum = 20
 	const reorgBatch common.BatchNum = lastValidBatch + 1
 	test.WipeDB(l2DB.DB())
+	// TODO: update with til
 	txs := test.GenPoolTxs(nInserts, tokens)
 	// Add txs to the DB
 	reorgedTxIDs := []common.TxID{}
@@ -341,6 +348,9 @@ func TestReorg(t *testing.T) {
 
 func TestPurge(t *testing.T) {
 	/*
+
+		TODO: update with til
+
 		WARNING: this should be fixed once transaktio is ready
 		// Generate txs
 		nInserts := l2DB.maxTxs + 20

db/migrations/0001.sql

@@ -453,12 +453,12 @@ BEGIN
         NEW.load_amount IS NULL OR
         NEW.load_amount_f IS NULL OR
         (NOT NEW.user_origin AND NEW.batch_num IS NULL)  THEN -- If is Coordinator L1, must include batch_num
-            RAISE EXCEPTION 'Invalid L1 tx.';
+            RAISE EXCEPTION 'Invalid L1 tx: %', NEW;
         END IF;
     ELSE
         -- Validate
         IF NEW.batch_num IS NULL OR NEW.nonce IS NULL THEN
-            RAISE EXCEPTION 'Invalid L2 tx.';
+            RAISE EXCEPTION 'Invalid L2 tx: %', NEW;
         END IF;
         -- Set fee if it's null
         IF NEW.fee IS NULL THEN

test/historydb.go

@@ -12,6 +12,30 @@ import (
 	"github.com/iden3/go-merkletree"
 )
 
+// Block0 represents Ethereum's genesis block,
+// which is stored by default at HistoryDB
+var Block0 common.Block = common.Block{
+	EthBlockNum: 0,
+	Hash: ethCommon.Hash([32]byte{
+		212, 229, 103, 64, 248, 118, 174, 248,
+		192, 16, 184, 106, 64, 213, 245, 103,
+		69, 161, 24, 208, 144, 106, 52, 230,
+		154, 236, 140, 13, 177, 203, 143, 163,
+	}), // 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3
+	Timestamp: time.Date(2015, time.July, 30, 3, 26, 13, 0, time.UTC), // 2015-07-30 03:26:13
+}
+
+// EthToken represents the Ether coin, which is stored by default in the DB
+// with TokenID = 0
+var EthToken common.Token = common.Token{
+	TokenID:     0,
+	Name:        "Ether",
+	Symbol:      "ETH",
+	Decimals:    18, //nolint:gomnd
+	EthBlockNum: 0,
+	EthAddr:     ethCommon.BigToAddress(big.NewInt(0)),
+}
+
 // WARNING: the generators in this file doesn't necessary follow the protocol
 // they are intended to check that the parsers between struct <==> DB are correct