Integrate til into api

This commit is contained in:
Arnau B
2020-11-16 18:10:43 +01:00
parent b3b23473b6
commit ce1218e28a
19 changed files with 583 additions and 766 deletions

View File

@@ -10,7 +10,6 @@ import (
"math/big"
"net/http"
"os"
"sort"
"strconv"
"testing"
"time"
@@ -26,7 +25,6 @@ import (
"github.com/hermeznetwork/hermez-node/log"
"github.com/hermeznetwork/hermez-node/test"
"github.com/hermeznetwork/hermez-node/test/til"
"github.com/iden3/go-iden3-crypto/babyjub"
)
// Pendinger is an interface that allows getting last returned item ID and PendingItems to be used for building fromItem
@@ -34,6 +32,7 @@ import (
type Pendinger interface {
GetPending() (pendingItems, lastItemID uint64)
Len() int
New() Pendinger
}
const apiPort = ":4010"
@@ -46,6 +45,11 @@ var SetBlockchain = `
AddToken(2)
AddToken(3)
AddToken(4)
AddToken(5)
AddToken(6)
AddToken(7)
AddToken(8)
> block
// Coordinator accounts, Idxs: 256, 257
CreateAccountCoordinator(0) Coord
@@ -54,8 +58,8 @@ var SetBlockchain = `
// close Block:0, Batch:0
> batch
CreateAccountDeposit(0) A: 500
CreateAccountDeposit(1) C: 0
CreateAccountDeposit(0) A: 111111111
CreateAccountDeposit(1) C: 222222222
CreateAccountCoordinator(0) C
// close Block:0, Batch:1
@@ -64,7 +68,7 @@ var SetBlockchain = `
// Coord(0): 0, Coord(1): 0
// C(0): 0
CreateAccountDeposit(1) A: 500
CreateAccountDeposit(1) A: 333333333
// close Block:0, Batch:2
> batchL1
@@ -72,45 +76,40 @@ var SetBlockchain = `
// close Block:0, Batch:3
> batchL1
CreateAccountDepositTransfer(0) B-A: 500, 100
CreateAccountDepositTransfer(0) B-A: 444444444, 1234444444 // to_eth_addr is NULL
// close Block:0, Batch:4
> batchL1
CreateAccountDeposit(0) D: 800
CreateAccountDeposit(0) D: 555555555
// close Block:0, Batch:5
> batchL1
CreateAccountCoordinator(1) B
Transfer(1) A-B: 200 (126)
Transfer(0) B-C: 100 (126)
Transfer(1) A-B: 111111 (2) // to_eth_addr is NULL
Transfer(0) B-C: 222222 (3)
// close Block:0, Batch:6
> batchL1 // forge L1User{1}, forge L1Coord{2}, forge L2{2}
Deposit(0) C: 500
DepositTransfer(0) C-D: 400, 100
Deposit(0) C: 666666666
DepositTransfer(0) C-D: 777777777, 123777777 // to_eth_addr is NULL
Transfer(0) A-B: 100 (126)
Transfer(0) C-A: 50 (126)
Transfer(1) B-C: 100 (126)
Exit(0) A: 100 (126)
Transfer(0) A-B: 333333 (111)
Transfer(0) C-A: 444444 (222)
Transfer(1) B-C: 555555 (123)
Exit(0) A: 666666 (44)
ForceTransfer(0) D-B: 200
ForceExit(0) B: 100
ForceTransfer(0) D-B: 777777 // to_eth_addr is NULL
ForceExit(0) B: 888888
// close Block:0, Batch:7
> batchL1
> block
AddToken(5)
AddToken(6)
AddToken(7)
AddToken(8)
Transfer(0) D-A: 300 (126)
Transfer(0) B-D: 100 (126)
Transfer(0) D-A: 999999 (77)
Transfer(0) B-D: 123123 (55)
// close Block:1, Batch:0
> batchL1
@@ -166,13 +165,8 @@ type testCommon struct {
fullBatches []testFullBatch
coordinators []historydb.CoordinatorAPI
accounts []testAccount
usrAddr string
usrBjj string
accs []common.Account
usrTxs []testTx
allTxs []testTx
txs []testTx
exits []testExit
usrExits []testExit
poolTxsToSend []testPoolTxSend
poolTxsToReceive []testPoolTxReceive
auths []testAuth
@@ -192,16 +186,6 @@ var api *API
// emulating the task of the synchronizer in order to have data to be returned
// by the API endpoints that will be tested
func TestMain(m *testing.M) {
/*
til update considerations:
1. Two instructions sets should be enough (one for L2 another for historydb)
2. FillBlocksExtra function must be used, there is a coment on top of the function that explains which data is setted
3. Some data will not be generated by til nor FillBlocksExtra, test.GenXXX will still be required to cover this cases
4. Most of the historydb inserts should be replaced with nBlocks calls to AddBlockSCData
5. When defining til instructions, there is no need to have 100s of entries for each table, but it's interesting to
cover all different cases (for instance all tx types)
*/
// Initializations
// Swagger
router := swagger.NewRouter().WithSwaggerFromFile("./swagger.yml")
@@ -260,171 +244,117 @@ func TestMain(m *testing.M) {
}
}()
// Fill HistoryDB and StateDB with fake data
// Gen blocks and add them to DB
const nBlocks = 5
// TODO: UPDATE with til
blocks := test.GenBlocks(1, nBlocks+1)
err = api.h.AddBlocks(blocks)
if err != nil {
panic(err)
}
lastBlockNum := blocks[nBlocks-1].EthBlockNum
// Reset DB
test.WipeDB(api.h.DB())
// Gen tokens and add them to DB
const nTokens = 10
// TODO: UPDATE with til
tokens, ethToken := test.GenTokens(nTokens, blocks)
err = api.h.AddTokens(tokens)
// Genratre blockchain data with til
tcc := til.NewContext(common.RollupConstMaxL1UserTx)
tilCfgExtra := til.ConfigExtra{
BootCoordAddr: ethCommon.HexToAddress("0xE39fEc6224708f0772D2A74fd3f9055A90E0A9f2"),
CoordUser: "Coord",
}
blocksData, err := tcc.GenerateBlocks(SetBlockchain)
if err != nil {
panic(err)
}
tokens = append([]common.Token{ethToken}, tokens...)
// Set token value
tokensUSD := []historydb.TokenWithUSD{}
for i, tkn := range tokens {
token := historydb.TokenWithUSD{
TokenID: tkn.TokenID,
EthBlockNum: tkn.EthBlockNum,
EthAddr: tkn.EthAddr,
Name: tkn.Name,
Symbol: tkn.Symbol,
Decimals: tkn.Decimals,
err = tcc.FillBlocksExtra(blocksData, &tilCfgExtra)
if err != nil {
panic(err)
}
AddAditionalInformation(blocksData)
// Generate L2 Txs with til
commonPoolTxs, err := tcc.GeneratePoolL2Txs(til.SetPoolL2MinimumFlow0)
if err != nil {
panic(err)
}
// Extract til generated data, and add it to HistoryDB
var commonBlocks []common.Block
var commonBatches []common.Batch
var commonAccounts []common.Account
var commonExitTree []common.ExitInfo
var commonL1Txs []common.L1Tx
var commonL2Txs []common.L2Tx
// Add ETH token at the beginning of the array
testTokens := []historydb.TokenWithUSD{}
ethUSD := float64(500)
ethNow := time.Now()
testTokens = append(testTokens, historydb.TokenWithUSD{
TokenID: test.EthToken.TokenID,
EthBlockNum: test.EthToken.EthBlockNum,
EthAddr: test.EthToken.EthAddr,
Name: test.EthToken.Name,
Symbol: test.EthToken.Symbol,
Decimals: test.EthToken.Decimals,
USD: &ethUSD,
USDUpdate: &ethNow,
})
err = api.h.UpdateTokenValue(test.EthToken.Symbol, ethUSD)
if err != nil {
panic(err)
}
for _, block := range blocksData {
// Insert block into HistoryDB
if err := api.h.AddBlockSCData(&block); err != nil { //nolint:gosec block is used as read only in the function
panic(err)
}
// Set value of 50% of the tokens
if i%2 != 0 {
value := float64(i) * 1.234567
// Extract data
commonBlocks = append(commonBlocks, block.Block)
for i, tkn := range block.Rollup.AddedTokens {
token := historydb.TokenWithUSD{
TokenID: tkn.TokenID,
EthBlockNum: tkn.EthBlockNum,
EthAddr: tkn.EthAddr,
Name: tkn.Name,
Symbol: tkn.Symbol,
Decimals: tkn.Decimals,
}
value := float64(i + 423)
now := time.Now().UTC()
token.USD = &value
token.USDUpdate = &now
// Set value in DB
err = api.h.UpdateTokenValue(token.Symbol, value)
if err != nil {
panic(err)
}
testTokens = append(testTokens, token)
}
// Set USD value for tokens in DB
commonL1Txs = append(commonL1Txs, block.Rollup.L1UserTxs...)
for _, batch := range block.Rollup.Batches {
commonL2Txs = append(commonL2Txs, batch.L2Txs...)
commonAccounts = append(commonAccounts, batch.CreatedAccounts...)
commonBatches = append(commonBatches, batch.Batch)
commonExitTree = append(commonExitTree, batch.ExitTree...)
commonL1Txs = append(commonL1Txs, batch.L1CoordinatorTxs...)
}
tokensUSD = append(tokensUSD, token)
}
// Gen batches and add them to DB
const nBatches = 10
// TODO: UPDATE with til
batches := test.GenBatches(nBatches, blocks)
err = api.h.AddBatches(batches)
if err != nil {
panic(err)
}
// Gen accounts and add them to HistoryDB and StateDB
const totalAccounts = 40
const userAccounts = 4
usrAddr := ethCommon.BigToAddress(big.NewInt(4896847))
privK := babyjub.NewRandPrivKey()
usrBjj := privK.Public()
// TODO: UPDATE with til
accs := test.GenAccounts(totalAccounts, userAccounts, tokens, &usrAddr, usrBjj, batches)
err = api.h.AddAccounts(accs)
if err != nil {
panic(err)
}
// api.s.CreateAccount called in new part with til
/* for i := 0; i < len(accs); i++ {
if _, err := api.s.CreateAccount(accs[i].Idx, &accs[i]); err != nil {
// lastBlockNum2 := blocksData[len(blocksData)-1].Block.EthBlockNum
// Add accounts to StateDB
for i := 0; i < len(commonAccounts); i++ {
if _, err := api.s.CreateAccount(commonAccounts[i].Idx, &commonAccounts[i]); err != nil {
panic(err)
}
} */
// helper to vinculate user related resources
usrIdxs := []string{}
for _, acc := range accs {
if acc.EthAddr == usrAddr || acc.PublicKey == usrBjj {
for _, token := range tokens {
if token.TokenID == acc.TokenID {
usrIdxs = append(usrIdxs, idxToHez(acc.Idx, token.Symbol))
}
}
}
}
// Gen exits and add them to DB
const totalExits = 40
// TODO: UPDATE with til
exits := test.GenExitTree(totalExits, batches, accs, blocks)
err = api.h.AddExitTree(exits)
if err != nil {
panic(err)
}
// L1 and L2 txs need to be sorted in a combined way
// Gen L1Txs
const totalL1Txs = 40
const userL1Txs = 4
// TODO: UPDATE with til
usrL1Txs, othrL1Txs := test.GenL1Txs(256, totalL1Txs, userL1Txs, &usrAddr, accs, tokens, blocks, batches)
// Gen L2Txs
const totalL2Txs = 20
const userL2Txs = 4
// TODO: UPDATE with til
usrL2Txs, othrL2Txs := test.GenL2Txs(256+totalL1Txs, totalL2Txs, userL2Txs, &usrAddr, accs, tokens, blocks, batches)
// Sort txs
sortedTxs := []txSortFielder{}
for i := 0; i < len(usrL1Txs); i++ {
wL1 := wrappedL1(usrL1Txs[i])
sortedTxs = append(sortedTxs, &wL1)
}
for i := 0; i < len(othrL1Txs); i++ {
wL1 := wrappedL1(othrL1Txs[i])
sortedTxs = append(sortedTxs, &wL1)
}
for i := 0; i < len(usrL2Txs); i++ {
wL2 := wrappedL2(usrL2Txs[i])
sortedTxs = append(sortedTxs, &wL2)
}
for i := 0; i < len(othrL2Txs); i++ {
wL2 := wrappedL2(othrL2Txs[i])
sortedTxs = append(sortedTxs, &wL2)
}
sort.Sort(txsSort(sortedTxs))
// Store txs to DB
for _, genericTx := range sortedTxs {
l1 := genericTx.L1()
l2 := genericTx.L2()
if l1 != nil {
err = api.h.AddL1Txs([]common.L1Tx{*l1})
if err != nil {
panic(err)
}
} else if l2 != nil {
err = api.h.AddL2Txs([]common.L2Tx{*l2})
if err != nil {
panic(err)
}
} else {
panic("should be l1 or l2")
}
}
// Coordinators
// Generate Coordinators and add them to HistoryDB
const nCoords = 10
coords := test.GenCoordinators(nCoords, blocks)
err = api.h.AddCoordinators(coords)
if err != nil {
panic(err)
}
fromItem := uint(0)
limit := uint(99999)
coordinators, _, err := api.h.GetCoordinatorsAPI(&fromItem, &limit, historydb.OrderAsc)
if err != nil {
commonCoords := test.GenCoordinators(nCoords, commonBlocks)
if err := api.h.AddCoordinators(commonCoords); err != nil {
panic(err)
}
// Bids
// Generate Bids and add them to HistoryDB
const nBids = 20
bids := test.GenBids(nBids, blocks, coords)
err = api.h.AddBids(bids)
if err != nil {
commonBids := test.GenBids(nBids, commonBlocks, commonCoords)
if err = api.h.AddBids(commonBids); err != nil {
panic(err)
}
testBids := genTestBids(blocks, coordinators, bids)
// Vars
// Generate SC vars and add them to HistoryDB (if needed)
var defaultSlotSetBid [6]*big.Int = [6]*big.Int{big.NewInt(10), big.NewInt(10), big.NewInt(10), big.NewInt(10), big.NewInt(10), big.NewInt(10)}
auctionVars := common.AuctionVariables{
EthBlockNum: int64(2),
@@ -453,189 +383,37 @@ func TestMain(m *testing.M) {
panic(err)
}
const nSlots = 20
// Set testCommon
usrTxs, allTxs := genTestTxs(sortedTxs, usrIdxs, accs, tokensUSD, blocks)
poolTxsToSend, poolTxsToReceive := genTestPoolTx(accs, []babyjub.PrivateKey{privK}, tokensUSD) // NOTE: pool txs are not inserted to the DB here. In the test they will be posted and getted.
testBatches, fullBatches := genTestBatches(blocks, batches, allTxs)
/* usrExits, allExits*/ _, _ = genTestExits(exits, tokensUSD, accs, usrIdxs)
// NEW WITH TIL
// Reset DB
test.WipeDB(api.h.DB())
tcc := til.NewContext(common.RollupConstMaxL1UserTx)
tilCfgExtra := til.ConfigExtra{
BootCoordAddr: ethCommon.HexToAddress("0xE39fEc6224708f0772D2A74fd3f9055A90E0A9f2"),
CoordUser: "A",
}
blocksData, err := tcc.GenerateBlocks(SetBlockchain)
if err != nil {
panic(err)
}
err = tcc.FillBlocksExtra(blocksData, &tilCfgExtra)
if err != nil {
panic(err)
}
// poolL2Txs, err := tcc.GeneratePoolL2Txs(til.SetPoolL2MinimumFlow0)
var blocksTc []common.Block
var batchesTc []common.Batch
var tokensTc []common.Token
var accountsTc []common.Account
var exitTreeTc []common.ExitInfo
var allL1TxsTc []common.L1Tx
var allL2TxsTc []common.L2Tx
AddAditionalInformation(blocksData)
for _, block := range blocksData {
// Insert block
err := api.h.AddBlockSCData(&block)
if err != nil {
panic(err)
}
blocksTc = append(blocksTc, block.Block)
tokensTc = append(tokensTc, block.Rollup.AddedTokens...)
allL1TxsTc = append(allL1TxsTc, block.Rollup.L1UserTxs...)
for _, batch := range block.Rollup.Batches {
allL2TxsTc = append(allL2TxsTc, batch.L2Txs...)
accountsTc = append(accountsTc, batch.CreatedAccounts...)
batchesTc = append(batchesTc, batch.Batch)
exitTreeTc = append(exitTreeTc, batch.ExitTree...)
allL1TxsTc = append(allL1TxsTc, batch.L1CoordinatorTxs...)
}
}
// lastBlockNum2 := blocksData[len(blocksData)-1].Block.EthBlockNum
tokensTc = append([]common.Token{ethToken}, tokensTc...)
tokensUSDTc := []historydb.TokenWithUSD{}
for i, tkn := range tokensTc {
token := historydb.TokenWithUSD{
TokenID: tkn.TokenID,
EthBlockNum: tkn.EthBlockNum,
EthAddr: tkn.EthAddr,
Name: tkn.Name,
Symbol: tkn.Symbol,
Decimals: tkn.Decimals,
}
// Set value of 50% of the tokens
if i%2 != 0 {
value := float64(i) * 1.234567
now := time.Now().UTC()
token.USD = &value
token.USDUpdate = &now
err = api.h.UpdateTokenValue(token.Symbol, value)
if err != nil {
panic(err)
}
}
tokensUSDTc = append(tokensUSDTc, token)
}
for i := 0; i < len(accountsTc); i++ {
if _, err := api.s.CreateAccount(accountsTc[i].Idx, &accountsTc[i]); err != nil {
panic(err)
}
}
usrIdxsTc := []string{}
for _, acc := range accountsTc {
for _, token := range tokensTc {
if token.TokenID == acc.TokenID {
usrIdxsTc = append(usrIdxsTc, idxToHez(acc.Idx, token.Symbol))
}
}
}
// Sort txs
sortedTxsTc := []txSortFielder{}
for i := 0; i < len(allL1TxsTc); i++ {
wL1 := wrappedL1(allL1TxsTc[i])
sortedTxsTc = append(sortedTxsTc, &wL1)
}
for i := 0; i < len(allL2TxsTc); i++ {
wL2 := wrappedL2(allL2TxsTc[i])
sortedTxsTc = append(sortedTxsTc, &wL2)
}
sort.Sort(txsSort(sortedTxsTc))
// Coordinators
const nCoordsTc = 10
coordsTc := test.GenCoordinators(nCoordsTc, blocksTc)
err = api.h.AddCoordinators(coordsTc)
if err != nil {
panic(err)
}
coordinatorsTc, _, err := api.h.GetCoordinatorsAPI(&fromItem, &limit, historydb.OrderAsc)
if err != nil {
panic(err)
}
// Bids
const nBidsTc = 20
bidsTc := test.GenBids(nBidsTc, blocksTc, coordsTc)
err = api.h.AddBids(bidsTc)
if err != nil {
panic(err)
}
testBidsTc := genTestBids(blocksTc, coordinatorsTc, bidsTc)
usrExitsTc, allExitsTc := genTestExits(exitTreeTc, tokensUSDTc, accountsTc, usrIdxsTc)
_, allTxsTc := genTestTxs(sortedTxsTc, usrIdxsTc, accountsTc, tokensUSDTc, blocksTc)
fmt.Println(allTxsTc)
// allTxsTc == allTxs
// testBatchesTc, fullBatchesTc := genTestBatches(blocksTc, batchesTc, allTxsTc)
// Generate test data, as expected to be received/sended from/to the API
testCoords := genTestCoordinators(commonCoords)
testBids := genTestBids(commonBlocks, testCoords, commonBids)
testExits := genTestExits(commonExitTree, testTokens, commonAccounts)
testTxs := genTestTxs(commonL1Txs, commonL2Txs, commonAccounts, testTokens, commonBlocks)
testBatches, testFullBatches := genTestBatches(commonBlocks, commonBatches, testTxs)
poolTxsToSend, poolTxsToReceive := genTestPoolTxs(commonPoolTxs, testTokens, commonAccounts)
tc = testCommon{
blocks: blocksTc,
tokens: tokensUSDTc,
blocks: commonBlocks,
tokens: testTokens,
batches: testBatches,
fullBatches: fullBatches,
coordinators: coordinatorsTc,
accounts: genTestAccounts(accountsTc, tokensUSDTc),
usrAddr: ethAddrToHez(usrAddr),
usrBjj: bjjToString(usrBjj),
accs: accountsTc,
usrTxs: usrTxs,
allTxs: allTxs,
exits: allExitsTc,
usrExits: usrExitsTc,
poolTxsToSend: poolTxsToSend,
poolTxsToReceive: poolTxsToReceive,
auths: genTestAuths(test.GenAuths(5)),
router: router,
bids: testBidsTc,
slots: api.genTestSlots(nSlots, lastBlockNum, testBids, auctionVars),
auctionVars: auctionVars,
rollupVars: rollupVars,
wdelayerVars: wdelayerVars,
}
/* tc = testCommon{
blocks: blocks,
tokens: tokensUSD,
batches: testBatches,
fullBatches: fullBatches,
coordinators: coordinators,
accounts: genTestAccounts(accs, tokensUSD),
usrAddr: ethAddrToHez(usrAddr),
usrBjj: bjjToString(usrBjj),
accs: accs,
usrTxs: usrTxs,
allTxs: allTxs,
exits: allExits,
usrExits: usrExits,
fullBatches: testFullBatches,
coordinators: testCoords,
accounts: genTestAccounts(commonAccounts, testTokens),
txs: testTxs,
exits: testExits,
poolTxsToSend: poolTxsToSend,
poolTxsToReceive: poolTxsToReceive,
auths: genTestAuths(test.GenAuths(5)),
router: router,
bids: testBids,
slots: api.genTestSlots(nSlots, lastBlockNum, testBids, auctionVars),
auctionVars: auctionVars,
rollupVars: rollupVars,
wdelayerVars: wdelayerVars,
} */
slots: api.genTestSlots(
20,
commonBlocks[len(commonBlocks)-1].EthBlockNum,
testBids,
auctionVars,
),
auctionVars: auctionVars,
rollupVars: rollupVars,
wdelayerVars: wdelayerVars,
}
// Fake server
if os.Getenv("FAKE_SERVER") == "yes" {
@@ -682,7 +460,11 @@ func doGoodReqPaginated(
iterPath += strconv.Itoa(int(next))
}
// Call API to get this iteration items
if err := doGoodReq("GET", iterPath+"&order="+order, nil, iterStruct); err != nil {
iterStruct = iterStruct.New()
if err := doGoodReq(
"GET", iterPath+"&order="+order, nil,
iterStruct,
); err != nil {
return err
}
appendIter(iterStruct)
@@ -758,8 +540,10 @@ func doGoodReq(method, path string, reqBody io.Reader, returnStruct interface{})
// Unmarshal body into return struct
if err := json.Unmarshal(body, returnStruct); err != nil {
log.Error("invalid json: " + string(body))
log.Error(err)
return err
}
// log.Info(string(body))
// Validate response against swagger spec
responseValidationInput := &swagger.ResponseValidationInput{
RequestValidationInput: requestValidationInput,