hermez-node/api/api_test.go

package api

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/big"
	"net/http"
	"os"
	"sort"
	"strconv"
	"testing"
	"time"

	ethCommon "github.com/ethereum/go-ethereum/common"
	swagger "github.com/getkin/kin-openapi/openapi3filter"
	"github.com/gin-gonic/gin"
	"github.com/hermeznetwork/hermez-node/common"
	"github.com/hermeznetwork/hermez-node/db"
	"github.com/hermeznetwork/hermez-node/db/historydb"
	"github.com/hermeznetwork/hermez-node/db/l2db"
	"github.com/hermeznetwork/hermez-node/db/statedb"
	"github.com/hermeznetwork/hermez-node/log"
	"github.com/hermeznetwork/hermez-node/test"
	"github.com/hermeznetwork/hermez-node/test/til"
	"github.com/iden3/go-iden3-crypto/babyjub"
)

// Pendinger is an interface that allows getting last returned item ID and PendingItems to be used for building fromItem
// when testing paginated endpoints.
type Pendinger interface {
	GetPending() (pendingItems, lastItemID uint64)
	Len() int
}

const apiPort = ":4010"
const apiURL = "http://localhost" + apiPort + "/"

var SetBlockchain = `
	Type: Blockchain

	AddToken(1)
	AddToken(2)
	AddToken(3)
	AddToken(4)

	// Coordinator accounts, Idxs: 256, 257
	CreateAccountCoordinator(0) Coord
	CreateAccountCoordinator(1) Coord

	// close Block:0, Batch:0
	> batch

	CreateAccountDeposit(0) A: 500
	CreateAccountDeposit(1) C: 0
	CreateAccountCoordinator(0) C

	// close Block:0, Batch:1
	> batchL1
	// Expected balances:
	//     Coord(0): 0, Coord(1): 0
	//     C(0): 0

	CreateAccountDeposit(1) A: 500

	// close Block:0, Batch:2
	> batchL1

	// close Block:0, Batch:3
	> batchL1

	CreateAccountDepositTransfer(0) B-A: 500, 100

	// close Block:0, Batch:4
	> batchL1
	CreateAccountDeposit(0) D: 800

	// close Block:0, Batch:5
	> batchL1

	CreateAccountCoordinator(1) B

	Transfer(1) A-B: 200 (126)
	Transfer(0) B-C: 100 (126)

	// close Block:0, Batch:6
	> batchL1 // forge L1User{1}, forge L1Coord{2}, forge L2{2}

	Deposit(0) C: 500
	DepositTransfer(0) C-D: 400, 100

	Transfer(0) A-B: 100 (126)
	Transfer(0) C-A: 50 (126)
	Transfer(1) B-C: 100 (126)
	Exit(0) A: 100 (126)

	ForceTransfer(0) D-B: 200
	ForceExit(0) B: 100

	// close Block:0, Batch:7
	> batchL1
	> block

	AddToken(5)
	AddToken(6)
	AddToken(7)
	AddToken(8)

	Transfer(0) D-A: 300 (126)
	Transfer(0) B-D: 100 (126)

	// close Block:1, Batch:0
	> batchL1

	CreateAccountCoordinator(0) F

	CreateAccountCoordinator(0) G
	CreateAccountCoordinator(0) H
	CreateAccountCoordinator(0) I
	CreateAccountCoordinator(0) J
	CreateAccountCoordinator(0) K
	CreateAccountCoordinator(0) L
	CreateAccountCoordinator(0) M
	CreateAccountCoordinator(0) N
	CreateAccountCoordinator(0) O
	CreateAccountCoordinator(0) P

	CreateAccountCoordinator(5) G
	CreateAccountCoordinator(5) H
	CreateAccountCoordinator(5) I
	CreateAccountCoordinator(5) J
	CreateAccountCoordinator(5) K
	CreateAccountCoordinator(5) L
	CreateAccountCoordinator(5) M
	CreateAccountCoordinator(5) N
	CreateAccountCoordinator(5) O
	CreateAccountCoordinator(5) P

	CreateAccountCoordinator(2) G
	CreateAccountCoordinator(2) H
	CreateAccountCoordinator(2) I
	CreateAccountCoordinator(2) J
	CreateAccountCoordinator(2) K
	CreateAccountCoordinator(2) L
	CreateAccountCoordinator(2) M
	CreateAccountCoordinator(2) N
	CreateAccountCoordinator(2) O
	CreateAccountCoordinator(2) P


	> batch
	> block
	> batch
	> block
	> batch
	> block
`

type testCommon struct {
	blocks           []common.Block
	tokens           []historydb.TokenWithUSD
	batches          []testBatch
	fullBatches      []testFullBatch
	coordinators     []historydb.CoordinatorAPI
	accounts         []testAccount
	usrAddr          string
	usrBjj           string
	accs             []common.Account
	usrTxs           []testTx
	allTxs           []testTx
	exits            []testExit
	usrExits         []testExit
	poolTxsToSend    []testPoolTxSend
	poolTxsToReceive []testPoolTxReceive
	auths            []testAuth
	router           *swagger.Router
	bids             []testBid
	slots            []testSlot
	auctionVars      common.AuctionVariables
	rollupVars       common.RollupVariables
	wdelayerVars     common.WDelayerVariables
}

var tc testCommon
var config configAPI
var api *API

// TestMain initializes the API server, and fill HistoryDB and StateDB with fake data,
// emulating the task of the synchronizer in order to have data to be returned
// by the API endpoints that will be tested
func TestMain(m *testing.M) {
	/*
		til update considerations:
		1. Two instructions sets should be enough (one for L2 another for historydb)
		2. FillBlocksExtra function must be used, there is a coment on top of the function that explains which data is setted
		3. Some data will not be generated by til nor FillBlocksExtra, test.GenXXX will still be required to cover this cases
		4. Most of the historydb inserts should be replaced with nBlocks calls to AddBlockSCData
		5. When defining til instructions, there is no need to have 100s of entries for each table, but it's interesting to
		   cover all different cases (for instance all tx types)
	*/

	// Initializations
	// Swagger
	router := swagger.NewRouter().WithSwaggerFromFile("./swagger.yml")
	// HistoryDB
	pass := os.Getenv("POSTGRES_PASS")

	database, err := db.InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
	if err != nil {
		panic(err)
	}
	hdb := historydb.NewHistoryDB(database)
	if err != nil {
		panic(err)
	}
	// StateDB
	dir, err := ioutil.TempDir("", "tmpdb")
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := os.RemoveAll(dir); err != nil {
			panic(err)
		}
	}()
	sdb, err := statedb.NewStateDB(dir, statedb.TypeTxSelector, 0)
	if err != nil {
		panic(err)
	}
	// L2DB
	l2DB := l2db.NewL2DB(database, 10, 100, 24*time.Hour)
	test.WipeDB(l2DB.DB()) // this will clean HistoryDB and L2DB

	// Config (smart contract constants)
	config = getConfigTest()

	// API
	apiGin := gin.Default()
	api, err = NewAPI(
		true,
		true,
		apiGin,
		hdb,
		sdb,
		l2DB,
		&config,
	)
	if err != nil {
		panic(err)
	}
	// Start server
	server := &http.Server{Addr: apiPort, Handler: apiGin}
	go func() {
		if err := server.ListenAndServe(); err != nil &&
			err != http.ErrServerClosed {
			panic(err)
		}
	}()

	// Fill HistoryDB and StateDB with fake data
	// Gen blocks and add them to DB
	const nBlocks = 5
	// TODO: UPDATE with til
	blocks := test.GenBlocks(1, nBlocks+1)
	err = api.h.AddBlocks(blocks)
	if err != nil {
		panic(err)
	}
	lastBlockNum := blocks[nBlocks-1].EthBlockNum

	// Gen tokens and add them to DB
	const nTokens = 10
	// TODO: UPDATE with til
	tokens, ethToken := test.GenTokens(nTokens, blocks)
	err = api.h.AddTokens(tokens)
	if err != nil {
		panic(err)
	}
	tokens = append([]common.Token{ethToken}, tokens...)
	// Set token value
	tokensUSD := []historydb.TokenWithUSD{}
	for i, tkn := range tokens {
		token := historydb.TokenWithUSD{
			TokenID:     tkn.TokenID,
			EthBlockNum: tkn.EthBlockNum,
			EthAddr:     tkn.EthAddr,
			Name:        tkn.Name,
			Symbol:      tkn.Symbol,
			Decimals:    tkn.Decimals,
		}
		// Set value of 50% of the tokens
		if i%2 != 0 {
			value := float64(i) * 1.234567
			now := time.Now().UTC()
			token.USD = &value
			token.USDUpdate = &now
			err = api.h.UpdateTokenValue(token.Symbol, value)
			if err != nil {
				panic(err)
			}
		}
		tokensUSD = append(tokensUSD, token)
	}
	// Gen batches and add them to DB
	const nBatches = 10
	// TODO: UPDATE with til
	batches := test.GenBatches(nBatches, blocks)
	err = api.h.AddBatches(batches)
	if err != nil {
		panic(err)
	}
	// Gen accounts and add them to HistoryDB and StateDB
	const totalAccounts = 40
	const userAccounts = 4
	usrAddr := ethCommon.BigToAddress(big.NewInt(4896847))
	privK := babyjub.NewRandPrivKey()
	usrBjj := privK.Public()
	// TODO: UPDATE with til
	accs := test.GenAccounts(totalAccounts, userAccounts, tokens, &usrAddr, usrBjj, batches)
	err = api.h.AddAccounts(accs)
	if err != nil {
		panic(err)
	}

	// api.s.CreateAccount called in new part with til
	/* for i := 0; i < len(accs); i++ {
		if _, err := api.s.CreateAccount(accs[i].Idx, &accs[i]); err != nil {
			panic(err)
		}
	} */

	// helper to vinculate user related resources
	usrIdxs := []string{}
	for _, acc := range accs {
		if acc.EthAddr == usrAddr || acc.PublicKey == usrBjj {
			for _, token := range tokens {
				if token.TokenID == acc.TokenID {
					usrIdxs = append(usrIdxs, idxToHez(acc.Idx, token.Symbol))
				}
			}
		}
	}
	// Gen exits and add them to DB
	const totalExits = 40
	// TODO: UPDATE with til
	exits := test.GenExitTree(totalExits, batches, accs, blocks)
	err = api.h.AddExitTree(exits)
	if err != nil {
		panic(err)
	}

	// L1 and L2 txs need to be sorted in a combined way
	// Gen L1Txs
	const totalL1Txs = 40
	const userL1Txs = 4
	// TODO: UPDATE with til
	usrL1Txs, othrL1Txs := test.GenL1Txs(256, totalL1Txs, userL1Txs, &usrAddr, accs, tokens, blocks, batches)
	// Gen L2Txs
	const totalL2Txs = 20
	const userL2Txs = 4
	// TODO: UPDATE with til
	usrL2Txs, othrL2Txs := test.GenL2Txs(256+totalL1Txs, totalL2Txs, userL2Txs, &usrAddr, accs, tokens, blocks, batches)
	// Sort txs
	sortedTxs := []txSortFielder{}
	for i := 0; i < len(usrL1Txs); i++ {
		wL1 := wrappedL1(usrL1Txs[i])
		sortedTxs = append(sortedTxs, &wL1)
	}
	for i := 0; i < len(othrL1Txs); i++ {
		wL1 := wrappedL1(othrL1Txs[i])
		sortedTxs = append(sortedTxs, &wL1)
	}
	for i := 0; i < len(usrL2Txs); i++ {
		wL2 := wrappedL2(usrL2Txs[i])
		sortedTxs = append(sortedTxs, &wL2)
	}
	for i := 0; i < len(othrL2Txs); i++ {
		wL2 := wrappedL2(othrL2Txs[i])
		sortedTxs = append(sortedTxs, &wL2)
	}
	sort.Sort(txsSort(sortedTxs))
	// Store txs to DB
	for _, genericTx := range sortedTxs {
		l1 := genericTx.L1()
		l2 := genericTx.L2()
		if l1 != nil {
			err = api.h.AddL1Txs([]common.L1Tx{*l1})
			if err != nil {
				panic(err)
			}
		} else if l2 != nil {
			err = api.h.AddL2Txs([]common.L2Tx{*l2})
			if err != nil {
				panic(err)
			}
		} else {
			panic("should be l1 or l2")
		}
	}

	// Coordinators
	const nCoords = 10
	coords := test.GenCoordinators(nCoords, blocks)
	err = api.h.AddCoordinators(coords)
	if err != nil {
		panic(err)
	}
	fromItem := uint(0)
	limit := uint(99999)
	coordinators, _, err := api.h.GetCoordinatorsAPI(&fromItem, &limit, historydb.OrderAsc)
	if err != nil {
		panic(err)
	}

	// Bids
	const nBids = 20
	bids := test.GenBids(nBids, blocks, coords)
	err = api.h.AddBids(bids)
	if err != nil {
		panic(err)
	}
	testBids := genTestBids(blocks, coordinators, bids)

	// Vars
	var defaultSlotSetBid [6]*big.Int = [6]*big.Int{big.NewInt(10), big.NewInt(10), big.NewInt(10), big.NewInt(10), big.NewInt(10), big.NewInt(10)}
	auctionVars := common.AuctionVariables{
		EthBlockNum:        int64(2),
		DonationAddress:    ethCommon.HexToAddress("0x1111111111111111111111111111111111111111"),
		DefaultSlotSetBid:  defaultSlotSetBid,
		Outbidding:         uint16(1),
		SlotDeadline:       uint8(20),
		BootCoordinator:    ethCommon.HexToAddress("0x1111111111111111111111111111111111111111"),
		ClosedAuctionSlots: uint16(2),
		OpenAuctionSlots:   uint16(5),
	}

	rollupVars := common.RollupVariables{
		EthBlockNum:           int64(3),
		FeeAddToken:           big.NewInt(100),
		ForgeL1L2BatchTimeout: int64(44),
		WithdrawalDelay:       uint64(3000),
	}

	wdelayerVars := common.WDelayerVariables{
		WithdrawalDelay: uint64(3000),
	}

	err = api.h.AddAuctionVars(&auctionVars)
	if err != nil {
		panic(err)
	}

	const nSlots = 20

	// Set testCommon
	usrTxs, allTxs := genTestTxs(sortedTxs, usrIdxs, accs, tokensUSD, blocks)
	poolTxsToSend, poolTxsToReceive := genTestPoolTx(accs, []babyjub.PrivateKey{privK}, tokensUSD) // NOTE: pool txs are not inserted to the DB here. In the test they will be posted and getted.
	testBatches, fullBatches := genTestBatches(blocks, batches, allTxs)
	/* usrExits, allExits*/ _, _ = genTestExits(exits, tokensUSD, accs, usrIdxs)

	// NEW WITH TIL
	// Reset DB
	test.WipeDB(api.h.DB())

	tcc := til.NewContext(common.RollupConstMaxL1UserTx)
	tilCfgExtra := til.ConfigExtra{
		BootCoordAddr: ethCommon.HexToAddress("0xE39fEc6224708f0772D2A74fd3f9055A90E0A9f2"),
		CoordUser:     "A",
	}
	blocksData, err := tcc.GenerateBlocks(SetBlockchain)
	if err != nil {
		panic(err)
	}
	err = tcc.FillBlocksExtra(blocksData, &tilCfgExtra)
	if err != nil {
		panic(err)
	}
	// poolL2Txs, err := tcc.GeneratePoolL2Txs(til.SetPoolL2MinimumFlow0)

	var blocksTc []common.Block
	var batchesTc []common.Batch
	var tokensTc []common.Token
	var accountsTc []common.Account
	var exitTreeTc []common.ExitInfo
	var allL1TxsTc []common.L1Tx
	var allL2TxsTc []common.L2Tx

	AddAditionalInformation(blocksData)

	for _, block := range blocksData {
		// Insert block
		err := api.h.AddBlockSCData(&block)
		if err != nil {
			panic(err)
		}
		blocksTc = append(blocksTc, block.Block)
		tokensTc = append(tokensTc, block.Rollup.AddedTokens...)
		allL1TxsTc = append(allL1TxsTc, block.Rollup.L1UserTxs...)
		for _, batch := range block.Rollup.Batches {
			allL2TxsTc = append(allL2TxsTc, batch.L2Txs...)
			accountsTc = append(accountsTc, batch.CreatedAccounts...)
			batchesTc = append(batchesTc, batch.Batch)
			exitTreeTc = append(exitTreeTc, batch.ExitTree...)
			allL1TxsTc = append(allL1TxsTc, batch.L1CoordinatorTxs...)
		}
	}
	// lastBlockNum2 := blocksData[len(blocksData)-1].Block.EthBlockNum

	tokensTc = append([]common.Token{ethToken}, tokensTc...)
	tokensUSDTc := []historydb.TokenWithUSD{}
	for i, tkn := range tokensTc {
		token := historydb.TokenWithUSD{
			TokenID:     tkn.TokenID,
			EthBlockNum: tkn.EthBlockNum,
			EthAddr:     tkn.EthAddr,
			Name:        tkn.Name,
			Symbol:      tkn.Symbol,
			Decimals:    tkn.Decimals,
		}
		// Set value of 50% of the tokens
		if i%2 != 0 {
			value := float64(i) * 1.234567
			now := time.Now().UTC()
			token.USD = &value
			token.USDUpdate = &now
			err = api.h.UpdateTokenValue(token.Symbol, value)
			if err != nil {
				panic(err)
			}
		}
		tokensUSDTc = append(tokensUSDTc, token)
	}

	for i := 0; i < len(accountsTc); i++ {
		if _, err := api.s.CreateAccount(accountsTc[i].Idx, &accountsTc[i]); err != nil {
			panic(err)
		}
	}

	usrIdxsTc := []string{}
	for _, acc := range accountsTc {
		for _, token := range tokensTc {
			if token.TokenID == acc.TokenID {
				usrIdxsTc = append(usrIdxsTc, idxToHez(acc.Idx, token.Symbol))
			}
		}
	}

	// Sort txs
	sortedTxsTc := []txSortFielder{}
	for i := 0; i < len(allL1TxsTc); i++ {
		wL1 := wrappedL1(allL1TxsTc[i])
		sortedTxsTc = append(sortedTxsTc, &wL1)
	}
	for i := 0; i < len(allL2TxsTc); i++ {
		wL2 := wrappedL2(allL2TxsTc[i])
		sortedTxsTc = append(sortedTxsTc, &wL2)
	}
	sort.Sort(txsSort(sortedTxsTc))

	// Coordinators
	const nCoordsTc = 10
	coordsTc := test.GenCoordinators(nCoordsTc, blocksTc)
	err = api.h.AddCoordinators(coordsTc)
	if err != nil {
		panic(err)
	}
	coordinatorsTc, _, err := api.h.GetCoordinatorsAPI(&fromItem, &limit, historydb.OrderAsc)
	if err != nil {
		panic(err)
	}

	// Bids
	const nBidsTc = 20
	bidsTc := test.GenBids(nBidsTc, blocksTc, coordsTc)
	err = api.h.AddBids(bidsTc)
	if err != nil {
		panic(err)
	}
	testBidsTc := genTestBids(blocksTc, coordinatorsTc, bidsTc)
	usrExitsTc, allExitsTc := genTestExits(exitTreeTc, tokensUSDTc, accountsTc, usrIdxsTc)
	_, allTxsTc := genTestTxs(sortedTxsTc, usrIdxsTc, accountsTc, tokensUSDTc, blocksTc)
	fmt.Println(allTxsTc)
	// allTxsTc == allTxs
	// testBatchesTc, fullBatchesTc := genTestBatches(blocksTc, batchesTc, allTxsTc)

	tc = testCommon{
		blocks:           blocksTc,
		tokens:           tokensUSDTc,
		batches:          testBatches,
		fullBatches:      fullBatches,
		coordinators:     coordinatorsTc,
		accounts:         genTestAccounts(accountsTc, tokensUSDTc),
		usrAddr:          ethAddrToHez(usrAddr),
		usrBjj:           bjjToString(usrBjj),
		accs:             accountsTc,
		usrTxs:           usrTxs,
		allTxs:           allTxs,
		exits:            allExitsTc,
		usrExits:         usrExitsTc,
		poolTxsToSend:    poolTxsToSend,
		poolTxsToReceive: poolTxsToReceive,
		auths:            genTestAuths(test.GenAuths(5)),
		router:           router,
		bids:             testBidsTc,
		slots:            api.genTestSlots(nSlots, lastBlockNum, testBids, auctionVars),
		auctionVars:      auctionVars,
		rollupVars:       rollupVars,
		wdelayerVars:     wdelayerVars,
	}

	/* tc = testCommon{
		blocks:           blocks,
		tokens:           tokensUSD,
		batches:          testBatches,
		fullBatches:      fullBatches,
		coordinators:     coordinators,
		accounts:         genTestAccounts(accs, tokensUSD),
		usrAddr:          ethAddrToHez(usrAddr),
		usrBjj:           bjjToString(usrBjj),
		accs:             accs,
		usrTxs:           usrTxs,
		allTxs:           allTxs,
		exits:            allExits,
		usrExits:         usrExits,
		poolTxsToSend:    poolTxsToSend,
		poolTxsToReceive: poolTxsToReceive,
		auths:            genTestAuths(test.GenAuths(5)),
		router:           router,
		bids:             testBids,
		slots:            api.genTestSlots(nSlots, lastBlockNum, testBids, auctionVars),
		auctionVars:      auctionVars,
		rollupVars:       rollupVars,
		wdelayerVars:     wdelayerVars,
	} */

	// Fake server
	if os.Getenv("FAKE_SERVER") == "yes" {
		for {
			log.Info("Running fake server at " + apiURL + " until ^C is received")
			time.Sleep(30 * time.Second)
		}
	}
	// Run tests
	result := m.Run()
	// Stop server
	if err := server.Shutdown(context.Background()); err != nil {
		panic(err)
	}
	if err := database.Close(); err != nil {
		panic(err)
	}
	if err := os.RemoveAll(dir); err != nil {
		panic(err)
	}
	os.Exit(result)
}

func doGoodReqPaginated(
	path, order string,
	iterStruct Pendinger,
	appendIter func(res interface{}),
) error {
	var next uint64
	firstIte := true
	expectedTotal := 0
	totalReceived := 0
	for {
		// Calculate fromItem
		iterPath := path
		if firstIte {
			if order == historydb.OrderDesc {
				// Fetch first item in reverse order
				iterPath += "99999" // Asumption that for testing there won't be any itemID > 99999
			} else {
				iterPath += "0"
			}
		} else {
			iterPath += strconv.Itoa(int(next))
		}
		// Call API to get this iteration items
		if err := doGoodReq("GET", iterPath+"&order="+order, nil, iterStruct); err != nil {
			return err
		}
		appendIter(iterStruct)
		// Keep iterating?
		remaining, lastID := iterStruct.GetPending()
		if remaining == 0 {
			break
		}
		if order == historydb.OrderDesc {
			next = lastID - 1
		} else {
			next = lastID + 1
		}
		// Check that the expected amount of items is consistent across iterations
		totalReceived += iterStruct.Len()
		if firstIte {
			firstIte = false
			expectedTotal = totalReceived + int(remaining)
		}
		if expectedTotal != totalReceived+int(remaining) {
			panic(fmt.Sprintf(
				"pagination error, totalReceived + remaining should be %d, but is %d",
				expectedTotal, totalReceived+int(remaining),
			))
		}
	}
	return nil
}

func doGoodReq(method, path string, reqBody io.Reader, returnStruct interface{}) error {
	ctx := context.Background()
	client := &http.Client{}
	httpReq, err := http.NewRequest(method, path, reqBody)
	if err != nil {
		return err
	}
	if reqBody != nil {
		httpReq.Header.Add("Content-Type", "application/json")
	}
	route, pathParams, err := tc.router.FindRoute(httpReq.Method, httpReq.URL)
	if err != nil {
		return err
	}
	// Validate request against swagger spec
	requestValidationInput := &swagger.RequestValidationInput{
		Request:    httpReq,
		PathParams: pathParams,
		Route:      route,
	}
	if err := swagger.ValidateRequest(ctx, requestValidationInput); err != nil {
		return err
	}
	// Do API call
	resp, err := client.Do(httpReq)
	if err != nil {
		return err
	}
	if resp.Body == nil && returnStruct != nil {
		return errors.New("Nil body")
	}
	//nolint
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("%d response. Body: %s", resp.StatusCode, string(body))
	}
	if returnStruct == nil {
		return nil
	}
	// Unmarshal body into return struct
	if err := json.Unmarshal(body, returnStruct); err != nil {
		log.Error("invalid json: " + string(body))
		return err
	}
	// Validate response against swagger spec
	responseValidationInput := &swagger.ResponseValidationInput{
		RequestValidationInput: requestValidationInput,
		Status:                 resp.StatusCode,
		Header:                 resp.Header,
	}
	responseValidationInput = responseValidationInput.SetBodyBytes(body)
	return swagger.ValidateResponse(ctx, responseValidationInput)
}

func doBadReq(method, path string, reqBody io.Reader, expectedResponseCode int) error {
	ctx := context.Background()
	client := &http.Client{}
	httpReq, _ := http.NewRequest(method, path, reqBody)
	route, pathParams, err := tc.router.FindRoute(httpReq.Method, httpReq.URL)
	if err != nil {
		return err
	}
	// Validate request against swagger spec
	requestValidationInput := &swagger.RequestValidationInput{
		Request:    httpReq,
		PathParams: pathParams,
		Route:      route,
	}
	if err := swagger.ValidateRequest(ctx, requestValidationInput); err != nil {
		if expectedResponseCode != 400 {
			return err
		}
		log.Warn("The request does not match the API spec")
	}
	// Do API call
	resp, err := client.Do(httpReq)
	if err != nil {
		return err
	}
	if resp.Body == nil {
		return errors.New("Nil body")
	}
	//nolint
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if resp.StatusCode != expectedResponseCode {
		return fmt.Errorf("Unexpected response code: %d. Body: %s", resp.StatusCode, string(body))
	}
	// Validate response against swagger spec
	responseValidationInput := &swagger.ResponseValidationInput{
		RequestValidationInput: requestValidationInput,
		Status:                 resp.StatusCode,
		Header:                 resp.Header,
	}
	responseValidationInput = responseValidationInput.SetBodyBytes(body)
	return swagger.ValidateResponse(ctx, responseValidationInput)
}

// test helpers

func getTimestamp(blockNum int64, blocks []common.Block) time.Time {
	for i := 0; i < len(blocks); i++ {
		if blocks[i].EthBlockNum == blockNum {
			return blocks[i].Timestamp
		}
	}
	panic("timesamp not found")
}

func getTokenByID(id common.TokenID, tokens []historydb.TokenWithUSD) historydb.TokenWithUSD {
	for i := 0; i < len(tokens); i++ {
		if tokens[i].TokenID == id {
			return tokens[i]
		}
	}
	panic("token not found")
}

func getTokenByIdx(idx common.Idx, tokens []historydb.TokenWithUSD, accs []common.Account) historydb.TokenWithUSD {
	for _, acc := range accs {
		if idx == acc.Idx {
			return getTokenByID(acc.TokenID, tokens)
		}
	}
	panic("token not found")
}

func getAccountByIdx(idx common.Idx, accs []common.Account) *common.Account {
	for _, acc := range accs {
		if acc.Idx == idx {
			return &acc
		}
	}
	panic("account not found")
}

func getBlockByNum(ethBlockNum int64, blocks []common.Block) common.Block {
	for _, b := range blocks {
		if b.EthBlockNum == ethBlockNum {
			return b
		}
	}
	panic("block not found")
}

func getCoordinatorByBidder(bidder ethCommon.Address, coordinators []historydb.CoordinatorAPI) historydb.CoordinatorAPI {
	for _, c := range coordinators {
		if c.Bidder == bidder {
			return c
		}
	}
	panic("coordinator not found")
}