From aa6cb6f818da4678b19df781a83bbacd1d98a19a Mon Sep 17 00:00:00 2001 From: laisolizq Date: Tue, 27 Oct 2020 18:27:05 +0100 Subject: [PATCH] API add bids endpoint --- api/api_test.go | 30 ++++++ api/bids.go | 47 ++++++++++ api/bids_test.go | 165 +++++++++++++++++++++++++++++++++ api/handlers.go | 4 - api/parsers.go | 12 +++ api/parsers_test.go | 20 ++++ api/swagger.yml | 23 ++++- db/historydb/historydb.go | 79 +++++++++++++++- db/historydb/historydb_test.go | 2 +- db/historydb/views.go | 16 ++++ db/migrations/0001.sql | 4 +- 11 files changed, 388 insertions(+), 14 deletions(-) create mode 100644 api/bids.go create mode 100644 api/bids_test.go diff --git a/api/api_test.go b/api/api_test.go index 42b2252..5e813ed 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -52,6 +52,7 @@ type testCommon struct { poolTxsToReceive []testPoolTxReceive auths []accountCreationAuthAPI router *swagger.Router + bids []testBid } var tc testCommon @@ -348,6 +349,15 @@ func TestMain(m *testing.M) { apiAuth := accountCreationAuthToAPI(auth) apiAuths = append(apiAuths, *apiAuth) } + + // Bids + const nBids = 10 + bids := test.GenBids(nBids, blocks, coords) + err = hdb.AddBids(bids) + if err != nil { + panic(err) + } + // Set testCommon usrTxs, allTxs := genTestTxs(sortedTxs, usrIdxs, accs, tokensUSD, blocks) poolTxsToSend, poolTxsToReceive := genTestPoolTx(accs, []babyjub.PrivateKey{privK}, tokensUSD) // NOTE: pool txs are not inserted to the DB here. In the test they will be posted and getted. @@ -367,7 +377,9 @@ func TestMain(m *testing.M) { poolTxsToReceive: poolTxsToReceive, auths: apiAuths, router: router, + bids: genTestBids(blocks, coordinators, bids), } + // Fake server if os.Getenv("FAKE_SERVER") == "yes" { for { @@ -816,3 +828,21 @@ func getAccountByIdx(idx common.Idx, accs []common.Account) *common.Account { } panic("account not found") } + +func getBlockByNum(ethBlockNum int64, blocks []common.Block) common.Block { + for _, b := range blocks { + if b.EthBlockNum == ethBlockNum { + return b + } + } + panic("block not found") +} + +func getCoordinatorByBidder(bidder ethCommon.Address, coordinators []historydb.CoordinatorAPI) historydb.CoordinatorAPI { + for _, c := range coordinators { + if c.Bidder == bidder { + return c + } + } + panic("coordinator not found") +} diff --git a/api/bids.go b/api/bids.go new file mode 100644 index 0000000..2c02ab1 --- /dev/null +++ b/api/bids.go @@ -0,0 +1,47 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/hermeznetwork/hermez-node/db" + "github.com/hermeznetwork/hermez-node/db/historydb" +) + +func getBids(c *gin.Context) { + slotNum, bidderAddr, err := parseBidFilters(c) + if err != nil { + retBadReq(err, c) + return + } + if slotNum == nil && bidderAddr == nil { + retBadReq(errors.New("It is necessary to add at least one filter: slotNum or/and bidderAddr"), c) + return + } + // Pagination + fromItem, order, limit, err := parsePagination(c) + if err != nil { + retBadReq(err, c) + return + } + + bids, pagination, err := h.GetBidsAPI( + slotNum, bidderAddr, fromItem, limit, order, + ) + + if err != nil { + retSQLErr(err, c) + return + } + + // Build succesfull response + type bidsResponse struct { + Bids []historydb.BidAPI `json:"bids"` + Pagination *db.Pagination `json:"pagination"` + } + c.JSON(http.StatusOK, &bidsResponse{ + Bids: bids, + Pagination: pagination, + }) +} diff --git a/api/bids_test.go b/api/bids_test.go new file mode 100644 index 0000000..2517fbe --- /dev/null +++ b/api/bids_test.go @@ -0,0 +1,165 @@ +package api + +import ( + "fmt" + "testing" + "time" + + ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/common" + "github.com/hermeznetwork/hermez-node/db" + "github.com/hermeznetwork/hermez-node/db/historydb" + "github.com/mitchellh/copystructure" + "github.com/stretchr/testify/assert" +) + +type testBid struct { + ItemID int `json:"itemId"` + SlotNum int64 `json:"slotNum"` + BidValue string `json:"bidValue"` + EthBlockNum int64 `json:"ethereumBlockNum"` + Bidder ethCommon.Address `json:"bidderAddr"` + Forger ethCommon.Address `json:"forgerAddr"` + URL string `json:"URL"` + Timestamp time.Time `json:"timestamp"` +} + +type testBidsResponse struct { + Bids []testBid `json:"bids"` + Pagination *db.Pagination `json:"pagination"` +} + +func (t testBidsResponse) GetPagination() *db.Pagination { + if t.Bids[0].ItemID < t.Bids[len(t.Bids)-1].ItemID { + t.Pagination.FirstReturnedItem = t.Bids[0].ItemID + t.Pagination.LastReturnedItem = t.Bids[len(t.Bids)-1].ItemID + } else { + t.Pagination.LastReturnedItem = t.Bids[0].ItemID + t.Pagination.FirstReturnedItem = t.Bids[len(t.Bids)-1].ItemID + } + return t.Pagination +} + +func (t testBidsResponse) Len() int { + return len(t.Bids) +} + +func genTestBids(blocks []common.Block, coordinators []historydb.CoordinatorAPI, bids []common.Bid) []testBid { + tBids := []testBid{} + for _, bid := range bids { + block := getBlockByNum(bid.EthBlockNum, blocks) + coordinator := getCoordinatorByBidder(bid.Bidder, coordinators) + tBid := testBid{ + SlotNum: bid.SlotNum, + BidValue: bid.BidValue.String(), + EthBlockNum: bid.EthBlockNum, + Bidder: bid.Bidder, + Forger: coordinator.Forger, + URL: coordinator.URL, + Timestamp: block.Timestamp, + } + tBids = append(tBids, tBid) + } + return tBids +} + +func TestGetBids(t *testing.T) { + endpoint := apiURL + "bids" + fetchedBids := []testBid{} + appendIter := func(intr interface{}) { + for i := 0; i < len(intr.(*testBidsResponse).Bids); i++ { + tmp, err := copystructure.Copy(intr.(*testBidsResponse).Bids[i]) + if err != nil { + panic(err) + } + fetchedBids = append(fetchedBids, tmp.(testBid)) + } + } + + limit := 3 + // bidderAddress + fetchedBids = []testBid{} + bidderAddress := tc.bids[3].Bidder + path := fmt.Sprintf("%s?bidderAddr=%s&limit=%d&fromItem=", endpoint, bidderAddress.String(), limit) + err := doGoodReqPaginated(path, historydb.OrderAsc, &testBidsResponse{}, appendIter) + assert.NoError(t, err) + bidderAddrBids := []testBid{} + for i := 0; i < len(tc.bids); i++ { + if tc.bids[i].Bidder == bidderAddress { + bidderAddrBids = append(bidderAddrBids, tc.bids[i]) + } + } + assertBids(t, bidderAddrBids, fetchedBids) + + // slotNum + fetchedBids = []testBid{} + slotNum := tc.bids[3].SlotNum + path = fmt.Sprintf("%s?slotNum=%d&limit=%d&fromItem=", endpoint, slotNum, limit) + err = doGoodReqPaginated(path, historydb.OrderAsc, &testBidsResponse{}, appendIter) + assert.NoError(t, err) + slotNumBids := []testBid{} + for i := 0; i < len(tc.bids); i++ { + if tc.bids[i].SlotNum == slotNum { + slotNumBids = append(slotNumBids, tc.bids[i]) + } + } + assertBids(t, slotNumBids, fetchedBids) + + // slotNum, in reverse order + fetchedBids = []testBid{} + path = fmt.Sprintf("%s?slotNum=%d&limit=%d&fromItem=", endpoint, slotNum, limit) + err = doGoodReqPaginated(path, historydb.OrderAsc, &testBidsResponse{}, appendIter) + assert.NoError(t, err) + flippedBids := []testBid{} + for i := len(slotNumBids) - 1; i >= 0; i-- { + flippedBids = append(flippedBids, slotNumBids[i]) + } + assertBids(t, flippedBids, fetchedBids) + + // Mixed filters + fetchedBids = []testBid{} + bidderAddress = tc.bids[9].Bidder + slotNum = tc.bids[4].SlotNum + path = fmt.Sprintf("%s?bidderAddr=%s&slotNum=%d&limit=%d&fromItem=", endpoint, bidderAddress.String(), slotNum, limit) + err = doGoodReqPaginated(path, historydb.OrderAsc, &testBidsResponse{}, appendIter) + assert.NoError(t, err) + slotNumBidderAddrBids := []testBid{} + for i := 0; i < len(tc.bids); i++ { + if tc.bids[i].Bidder == bidderAddress && tc.bids[i].SlotNum == slotNum { + slotNumBidderAddrBids = append(slotNumBidderAddrBids, tc.bids[i]) + } + } + assertBids(t, slotNumBidderAddrBids, fetchedBids) + + // 400 + // No filters + path = fmt.Sprintf("%s?limit=%d&fromItem=", endpoint, limit) + err = doBadReq("GET", path, nil, 400) + assert.NoError(t, err) + // Invalid slotNum + path = fmt.Sprintf("%s?slotNum=%d", endpoint, -2) + err = doBadReq("GET", path, nil, 400) + assert.NoError(t, err) + // Invalid bidderAddress + path = fmt.Sprintf("%s?bidderAddr=%s", endpoint, "0xG0000001") + err = doBadReq("GET", path, nil, 400) + assert.NoError(t, err) + // 404 + path = fmt.Sprintf("%s?slotNum=%d&bidderAddr=%s", endpoint, tc.bids[0].SlotNum, tc.bids[1].Bidder.String()) + err = doBadReq("GET", path, nil, 404) + assert.NoError(t, err) +} + +func assertBids(t *testing.T, expected, actual []testBid) { + assert.Equal(t, len(expected), len(actual)) + for i := 0; i < len(expected); i++ { + assertBid(t, expected[i], actual[i]) + } +} + +func assertBid(t *testing.T, expected, actual testBid) { + assert.Equal(t, expected.Timestamp.Unix(), actual.Timestamp.Unix()) + expected.Timestamp = actual.Timestamp + actual.ItemID = expected.ItemID + assert.Equal(t, expected, actual) +} diff --git a/api/handlers.go b/api/handlers.go index 2bd598b..d4da8ef 100644 --- a/api/handlers.go +++ b/api/handlers.go @@ -141,10 +141,6 @@ func getSlots(c *gin.Context) { } -func getBids(c *gin.Context) { - -} - func getNextForgers(c *gin.Context) { } diff --git a/api/parsers.go b/api/parsers.go index 7475785..6e63f9f 100644 --- a/api/parsers.go +++ b/api/parsers.go @@ -198,6 +198,18 @@ func parseTokenFilters(c querier) ([]common.TokenID, []string, string, error) { return tokensIDs, symbols, nameStr, nil } +func parseBidFilters(c querier) (*uint, *ethCommon.Address, error) { + slotNum, err := parseQueryUint("slotNum", nil, 0, maxUint32, c) + if err != nil { + return nil, nil, err + } + bidderAddr, err := parseQueryEthAddr("bidderAddr", c) + if err != nil { + return nil, nil, err + } + return slotNum, bidderAddr, nil +} + // Param parsers type paramer interface { diff --git a/api/parsers_test.go b/api/parsers_test.go index 2d01090..f9ba38b 100644 --- a/api/parsers_test.go +++ b/api/parsers_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "math/big" "strconv" + "strings" "testing" ethCommon "github.com/ethereum/go-ethereum/common" @@ -322,3 +323,22 @@ func TestParseEthAddr(t *testing.T) { assert.NoError(t, err) assert.Equal(t, ethAddr, *res) } + +func TestParseBidFilters(t *testing.T) { + slotNum := "slotNum" + bidderAddr := "bidderAddr" + slotNumValue := "2" + bidderAddrValue := "0xaa942cfcd25ad4d90a62358b0dd84f33b398262a" + c := &queryParser{} + c.m = make(map[string]string) + // Incorrect values + c.m[slotNum] = slotNumValue + c.m[bidderAddr] = bidderAddrValue + + slotNumParse, bidderAddrParse, err := parseBidFilters(c) + assert.NoError(t, err) + + // Correct values + assert.Equal(t, strings.ToLower(bidderAddrParse.Hex()), bidderAddrValue) + assert.Equal(t, slotNumValue, strconv.FormatUint(uint64(*slotNumParse), 10)) +} diff --git a/api/swagger.yml b/api/swagger.yml index 30515d7..dbc5a6f 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -892,9 +892,9 @@ paths: required: false schema: $ref: '#/components/schemas/SlotNum' - - name: forgerAddr + - name: bidderAddr in: query - description: Get only bids made by a coordinator identified by its forger address. + description: Get only bids made by a coordinator identified by its bidder address. required: false schema: $ref: '#/components/schemas/EthereumAddress' @@ -1918,12 +1918,14 @@ components: type: object description: Tokens placed in an auction by a coordinator to gain the right to forge batches during a specific slot. properties: + itemId: + $ref: '#/components/schemas/ItemId' + bidderAddr: + $ref: '#/components/schemas/EthereumAddress' forgerAddr: $ref: '#/components/schemas/EthereumAddress' slotNum: $ref: '#/components/schemas/SlotNum' - withdrawAddr: - $ref: '#/components/schemas/EthereumAddress' URL: $ref: '#/components/schemas/URL' bidValue: @@ -1933,6 +1935,15 @@ components: timestamp: type: string format: date-time + additionalProperties: false + require: + - bidderAddr + - forgerAddr + - slotNum + - URL + - bidValue + - ethereumBlockNum + - timestamp Bids: type: object properties: @@ -1943,6 +1954,10 @@ components: $ref: '#/components/schemas/Bid' pagination: $ref: '#/components/schemas/PaginationInfo' + additionalProperties: false + require: + - bids + - pagination RecommendedFee: type: object description: Fee that the coordinator recommends per transaction in USD. diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index ca8b435..0c61930 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -339,16 +339,89 @@ func (hdb *HistoryDB) addBids(d meddler.DB, bids []common.Bid) error { ) } -// GetBids return the bids -func (hdb *HistoryDB) GetBids() ([]common.Bid, error) { +// GetAllBids retrieve all bids from the DB +func (hdb *HistoryDB) GetAllBids() ([]common.Bid, error) { var bids []*common.Bid err := meddler.QueryAll( hdb.db, &bids, - "SELECT * FROM bid;", + `SELECT bid.slot_num, bid.bid_value, bid.eth_block_num, bid.bidder_addr FROM bid;`, ) return db.SlicePtrsToSlice(bids).([]common.Bid), err } +// GetBidsAPI return the bids applying the given filters +func (hdb *HistoryDB) GetBidsAPI(slotNum *uint, forgerAddr *ethCommon.Address, fromItem, limit *uint, order string) ([]BidAPI, *db.Pagination, error) { + var query string + var args []interface{} + queryStr := `SELECT bid.*, block.timestamp, coordinator.forger_addr, coordinator.url, + COUNT(*) OVER() AS total_items, MIN(bid.item_id) OVER() AS first_item, + MAX(bid.item_id) OVER() AS last_item FROM bid + INNER JOIN block ON bid.eth_block_num = block.eth_block_num + INNER JOIN coordinator ON bid.bidder_addr = coordinator.bidder_addr ` + // Apply filters + nextIsAnd := false + // slotNum filter + if slotNum != nil { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + queryStr += "bid.slot_num = ? " + args = append(args, slotNum) + nextIsAnd = true + } + // slotNum filter + if forgerAddr != nil { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + queryStr += "bid.bidder_addr = ? " + args = append(args, forgerAddr) + nextIsAnd = true + } + if fromItem != nil { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + if order == OrderAsc { + queryStr += "bid.item_id >= ? " + } else { + queryStr += "bid.item_id <= ? " + } + args = append(args, fromItem) + } + // pagination + queryStr += "ORDER BY bid.item_id " + if order == OrderAsc { + queryStr += "ASC " + } else { + queryStr += "DESC " + } + queryStr += fmt.Sprintf("LIMIT %d;", *limit) + query, argsQ, err := sqlx.In(queryStr, args...) + if err != nil { + return nil, nil, err + } + query = hdb.db.Rebind(query) + bids := []*BidAPI{} + if err := meddler.QueryAll(hdb.db, &bids, query, argsQ...); err != nil { + return nil, nil, err + } + if len(bids) == 0 { + return nil, nil, sql.ErrNoRows + } + return db.SlicePtrsToSlice(bids).([]BidAPI), &db.Pagination{ + TotalItems: bids[0].TotalItems, + FirstItem: bids[0].FirstItem, + LastItem: bids[0].LastItem, + }, nil +} + // AddCoordinators insert Coordinators into the DB func (hdb *HistoryDB) AddCoordinators(coordinators []common.Coordinator) error { return hdb.addCoordinators(hdb.db, coordinators) diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index 133920b..dacd536 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -159,7 +159,7 @@ func TestBids(t *testing.T) { err = historyDB.AddBids(bids) assert.NoError(t, err) // Fetch bids - fetchedBids, err := historyDB.GetBids() + fetchedBids, err := historyDB.GetAllBids() assert.NoError(t, err) // Compare fetched bids vs generated bids for i, bid := range fetchedBids { diff --git a/db/historydb/views.go b/db/historydb/views.go index 5cb57cd..80c0180 100644 --- a/db/historydb/views.go +++ b/db/historydb/views.go @@ -227,3 +227,19 @@ type Metrics struct { TotalBJJs int64 `json:"totalBJJs"` AvgTransactionFee float64 `json:"avgTransactionFee"` } + +// BidAPI is a representation of a bid with additional information +// required by the API +type BidAPI struct { + ItemID int `json:"itemId" meddler:"item_id"` + SlotNum int64 `json:"slotNum" meddler:"slot_num"` + BidValue apitypes.BigIntStr `json:"bidValue" meddler:"bid_value"` + EthBlockNum int64 `json:"ethereumBlockNum" meddler:"eth_block_num"` + Bidder ethCommon.Address `json:"bidderAddr" meddler:"bidder_addr"` + Forger ethCommon.Address `json:"forgerAddr" meddler:"forger_addr"` + URL string `json:"URL" meddler:"url"` + Timestamp time.Time `json:"timestamp" meddler:"timestamp,utctime"` + TotalItems int `json:"-" meddler:"total_items"` + FirstItem int `json:"-" meddler:"first_item"` + LastItem int `json:"-" meddler:"last_item"` +} diff --git a/db/migrations/0001.sql b/db/migrations/0001.sql index 8f8847a..71f4d55 100644 --- a/db/migrations/0001.sql +++ b/db/migrations/0001.sql @@ -31,11 +31,11 @@ CREATE TABLE batch ( ); CREATE TABLE bid ( + item_id SERIAL PRIMARY KEY, slot_num BIGINT NOT NULL, bid_value BYTEA NOT NULL, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, - bidder_addr BYTEA NOT NULL, -- fake foreign key for coordinator - PRIMARY KEY (slot_num, bid_value) + bidder_addr BYTEA NOT NULL -- fake foreign key for coordinator ); CREATE TABLE token (