From 043710112df5b86f960be3ac4a02bbda980f4e5c Mon Sep 17 00:00:00 2001 From: laisolizq Date: Thu, 15 Oct 2020 18:58:24 +0200 Subject: [PATCH] Add get tokens endpoint --- api/api_test.go | 111 +++++++++++++++++++++++++++--- api/dbtoapistructs.go | 83 +++++++++++++++++----- api/handlers.go | 29 +++++++- api/parsers.go | 24 +++++++ api/parsers_test.go | 25 +++++++ api/swagger.yml | 3 + db/historydb/historydb.go | 76 ++++++++++++++++++-- db/historydb/historydb_test.go | 5 +- db/historydb/views.go | 20 +++--- db/migrations/0001.sql | 3 +- priceupdater/priceupdater_test.go | 3 +- test/historydb.go | 2 +- 12 files changed, 338 insertions(+), 46 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 220c560..5194a57 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -37,7 +37,7 @@ const apiURL = "http://localhost" + apiPort + "/" type testCommon struct { blocks []common.Block - tokens []historydb.TokenRead + tokens []tokenAPI batches []common.Batch usrAddr string usrBjj string @@ -201,9 +201,9 @@ func TestMain(m *testing.M) { panic(err) } // Set token value - tokensUSD := []historydb.TokenRead{} + tokensUSD := []tokenAPI{} for i, tkn := range tokens { - token := historydb.TokenRead{ + token := tokenAPI{ TokenID: tkn.TokenID, EthBlockNum: tkn.EthBlockNum, EthAddr: tkn.EthAddr, @@ -287,7 +287,7 @@ func TestMain(m *testing.M) { } panic("timesamp not found") } - getToken := func(id common.TokenID) historydb.TokenRead { + getToken := func(id common.TokenID) tokenAPI { for i := 0; i < len(tokensUSD); i++ { if tokensUSD[i].TokenID == id { return tokensUSD[i] @@ -295,7 +295,7 @@ func TestMain(m *testing.M) { } panic("token not found") } - getTokenByIdx := func(idx common.Idx) historydb.TokenRead { + getTokenByIdx := func(idx common.Idx) tokenAPI { for _, acc := range accs { if idx == acc.Idx { return getToken(acc.TokenID) @@ -874,18 +874,113 @@ func assertExitAPIs(t *testing.T, expected, actual []exitAPI) { func TestGetToken(t *testing.T) { // Get all txs by their ID endpoint := apiURL + "tokens/" - fetchedTokens := []historydb.TokenRead{} + fetchedTokens := []tokenAPI{} for _, token := range tc.tokens { - fetchedToken := historydb.TokenRead{} + fetchedToken := tokenAPI{} assert.NoError(t, doGoodReq("GET", endpoint+strconv.Itoa(int(token.TokenID)), nil, &fetchedToken)) fetchedTokens = append(fetchedTokens, fetchedToken) } assertTokensAPIs(t, tc.tokens, fetchedTokens) } -func assertTokensAPIs(t *testing.T, expected, actual []historydb.TokenRead) { +func TestGetTokens(t *testing.T) { + endpoint := apiURL + "tokens" + fetchedTokens := []tokenAPI{} + appendIter := func(intr interface{}) { + for i := 0; i < len(intr.(*tokensAPI).Tokens); i++ { + tmp, err := copystructure.Copy(intr.(*tokensAPI).Tokens[i]) + if err != nil { + panic(err) + } + fetchedTokens = append(fetchedTokens, tmp.(tokenAPI)) + } + } + // Get all (no filters) + limit := 8 + path := fmt.Sprintf("%s?limit=%d&fromItem=", endpoint, limit) + err := doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + assertTokensAPIs(t, tc.tokens, fetchedTokens) + + // Get by tokenIds + fetchedTokens = []tokenAPI{} + limit = 7 + stringIds := strconv.Itoa(int(tc.tokens[2].TokenID)) + "," + strconv.Itoa(int(tc.tokens[5].TokenID)) + "," + strconv.Itoa(int(tc.tokens[6].TokenID)) + path = fmt.Sprintf( + "%s?ids=%s&limit=%d&fromItem=", + endpoint, stringIds, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + var tokensFiltered []tokenAPI + tokensFiltered = append(tokensFiltered, tc.tokens[2]) + tokensFiltered = append(tokensFiltered, tc.tokens[5]) + tokensFiltered = append(tokensFiltered, tc.tokens[6]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // Get by symbols + fetchedTokens = []tokenAPI{} + limit = 7 + stringSymbols := tc.tokens[1].Symbol + "," + tc.tokens[3].Symbol + path = fmt.Sprintf( + "%s?symbols=%s&limit=%d&fromItem=", + endpoint, stringSymbols, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + tokensFiltered = nil + tokensFiltered = append(tokensFiltered, tc.tokens[1]) + tokensFiltered = append(tokensFiltered, tc.tokens[3]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // Get by name + fetchedTokens = []tokenAPI{} + limit = 5 + stringName := tc.tokens[8].Name[4:5] + path = fmt.Sprintf( + "%s?name=%s&limit=%d&fromItem=", + endpoint, stringName, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + tokensFiltered = nil + tokensFiltered = append(tokensFiltered, tc.tokens[8]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // Multiple filters + fetchedTokens = []tokenAPI{} + limit = 5 + stringSymbols = tc.tokens[2].Symbol + "," + tc.tokens[6].Symbol + stringIds = strconv.Itoa(int(tc.tokens[2].TokenID)) + "," + strconv.Itoa(int(tc.tokens[5].TokenID)) + "," + strconv.Itoa(int(tc.tokens[6].TokenID)) + path = fmt.Sprintf( + "%s?symbols=%s&ids=%s&limit=%d&fromItem=", + endpoint, stringSymbols, stringIds, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + + tokensFiltered = nil + tokensFiltered = append(tokensFiltered, tc.tokens[2]) + tokensFiltered = append(tokensFiltered, tc.tokens[6]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // All, in reverse order + fetchedTokens = []tokenAPI{} + limit = 5 + path = fmt.Sprintf("%s?limit=%d&fromItem=", endpoint, limit) + err = doGoodReqPaginated(path, historydb.OrderDesc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + flipedTokens := []tokenAPI{} + for i := 0; i < len(tc.tokens); i++ { + flipedTokens = append(flipedTokens, tc.tokens[len(tc.tokens)-1-i]) + } + assertTokensAPIs(t, flipedTokens, fetchedTokens) +} + +func assertTokensAPIs(t *testing.T, expected, actual []tokenAPI) { require.Equal(t, len(expected), len(actual)) for i := 0; i < len(actual); i++ { //nolint len(actual) won't change within the loop + actual[i].ItemID = 0 if expected[i].USDUpdate == nil { assert.Equal(t, expected[i].USDUpdate, actual[i].USDUpdate) } else { diff --git a/api/dbtoapistructs.go b/api/dbtoapistructs.go index e175604..fba787e 100644 --- a/api/dbtoapistructs.go +++ b/api/dbtoapistructs.go @@ -71,20 +71,20 @@ type l2Info struct { } type historyTxAPI struct { - IsL1 string `json:"L1orL2"` - TxID string `json:"id"` - ItemID int `json:"itemId"` - Type common.TxType `json:"type"` - Position int `json:"position"` - FromIdx *string `json:"fromAccountIndex"` - ToIdx string `json:"toAccountIndex"` - Amount string `json:"amount"` - BatchNum *common.BatchNum `json:"batchNum"` - HistoricUSD *float64 `json:"historicUSD"` - Timestamp time.Time `json:"timestamp"` - L1Info *l1Info `json:"L1Info"` - L2Info *l2Info `json:"L2Info"` - Token historydb.TokenRead `json:"token"` + IsL1 string `json:"L1orL2"` + TxID string `json:"id"` + ItemID int `json:"itemId"` + Type common.TxType `json:"type"` + Position int `json:"position"` + FromIdx *string `json:"fromAccountIndex"` + ToIdx string `json:"toAccountIndex"` + Amount string `json:"amount"` + BatchNum *common.BatchNum `json:"batchNum"` + HistoricUSD *float64 `json:"historicUSD"` + Timestamp time.Time `json:"timestamp"` + L1Info *l1Info `json:"L1Info"` + L2Info *l2Info `json:"L2Info"` + Token tokenAPI `json:"token"` } func historyTxsToAPI(dbTxs []historydb.HistoryTx) []historyTxAPI { @@ -100,7 +100,7 @@ func historyTxsToAPI(dbTxs []historydb.HistoryTx) []historyTxAPI { HistoricUSD: dbTxs[i].HistoricUSD, BatchNum: dbTxs[i].BatchNum, Timestamp: dbTxs[i].Timestamp, - Token: historydb.TokenRead{ + Token: tokenAPI{ TokenID: dbTxs[i].TokenID, EthBlockNum: dbTxs[i].TokenEthBlockNum, EthAddr: dbTxs[i].TokenEthAddr, @@ -170,7 +170,7 @@ type exitAPI struct { InstantWithdrawn *int64 `json:"instantWithdrawn"` DelayedWithdrawRequest *int64 `json:"delayedWithdrawRequest"` DelayedWithdrawn *int64 `json:"delayedWithdrawn"` - Token historydb.TokenRead `json:"token"` + Token tokenAPI `json:"token"` } func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI { @@ -185,7 +185,7 @@ func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI { InstantWithdrawn: dbExits[i].InstantWithdrawn, DelayedWithdrawRequest: dbExits[i].DelayedWithdrawRequest, DelayedWithdrawn: dbExits[i].DelayedWithdrawn, - Token: historydb.TokenRead{ + Token: tokenAPI{ TokenID: dbExits[i].TokenID, EthBlockNum: dbExits[i].TokenEthBlockNum, EthAddr: dbExits[i].TokenEthAddr, @@ -199,3 +199,52 @@ func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI { } return apiExits } + +// Tokens + +type tokensAPI struct { + Tokens []tokenAPI `json:"tokens"` + Pagination *db.Pagination `json:"pagination"` +} + +func (t *tokensAPI) GetPagination() *db.Pagination { + if t.Tokens[0].ItemID < t.Tokens[len(t.Tokens)-1].ItemID { + t.Pagination.FirstReturnedItem = t.Tokens[0].ItemID + t.Pagination.LastReturnedItem = t.Tokens[len(t.Tokens)-1].ItemID + } else { + t.Pagination.LastReturnedItem = t.Tokens[0].ItemID + t.Pagination.FirstReturnedItem = t.Tokens[len(t.Tokens)-1].ItemID + } + return t.Pagination +} +func (t *tokensAPI) Len() int { return len(t.Tokens) } + +type tokenAPI struct { + ItemID int `json:"itemId"` + TokenID common.TokenID `json:"id"` + EthBlockNum int64 `json:"ethereumBlockNum"` // Ethereum block number in which this token was registered + EthAddr ethCommon.Address `json:"ethereumAddress"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals uint64 `json:"decimals"` + USD *float64 `json:"USD"` + USDUpdate *time.Time `json:"fiatUpdate"` +} + +func tokensToAPI(dbTokens []historydb.TokenRead) []tokenAPI { + apiTokens := []tokenAPI{} + for i := 0; i < len(dbTokens); i++ { + apiTokens = append(apiTokens, tokenAPI{ + ItemID: dbTokens[i].ItemID, + TokenID: dbTokens[i].TokenID, + EthBlockNum: dbTokens[i].EthBlockNum, + EthAddr: dbTokens[i].EthAddr, + Name: dbTokens[i].Name, + Symbol: dbTokens[i].Symbol, + Decimals: dbTokens[i].Decimals, + USD: dbTokens[i].USD, + USDUpdate: dbTokens[i].USDUpdate, + }) + } + return apiTokens +} diff --git a/api/handlers.go b/api/handlers.go index a999339..14b2a94 100644 --- a/api/handlers.go +++ b/api/handlers.go @@ -202,7 +202,33 @@ func getConfig(c *gin.Context) { } func getTokens(c *gin.Context) { + // Account filters + tokenIDs, symbols, name, err := parseTokenFilters(c) + if err != nil { + retBadReq(err, c) + return + } + // Pagination + fromItem, order, limit, err := parsePagination(c) + if err != nil { + retBadReq(err, c) + return + } + // Fetch exits from historyDB + tokens, pagination, err := h.GetTokens( + tokenIDs, symbols, name, fromItem, limit, order, + ) + if err != nil { + retSQLErr(err, c) + return + } + // Build succesfull response + apiTokens := tokensToAPI(tokens) + c.JSON(http.StatusOK, &tokensAPI{ + Tokens: apiTokens, + Pagination: pagination, + }) } func getToken(c *gin.Context) { @@ -219,7 +245,8 @@ func getToken(c *gin.Context) { retSQLErr(err, c) return } - c.JSON(http.StatusOK, token) + apiToken := tokensToAPI([]historydb.TokenRead{*token}) + c.JSON(http.StatusOK, apiToken[0]) } func getRecommendedFee(c *gin.Context) { diff --git a/api/parsers.go b/api/parsers.go index 6b56808..b609411 100644 --- a/api/parsers.go +++ b/api/parsers.go @@ -219,6 +219,30 @@ func parseAccountFilters(c querier) (*common.TokenID, *ethCommon.Address, *babyj return tokenID, addr, bjj, idx, nil } +func parseTokenFilters(c querier) ([]common.TokenID, []string, string, error) { + idsStr := c.Query("ids") + symbolsStr := c.Query("symbols") + nameStr := c.Query("name") + var tokensIDs []common.TokenID + if idsStr != "" { + ids := strings.Split(idsStr, ",") + + for _, id := range ids { + idUint, err := strconv.Atoi(id) + if err != nil { + return nil, nil, "", err + } + tokenID := common.TokenID(idUint) + tokensIDs = append(tokensIDs, tokenID) + } + } + var symbols []string + if symbolsStr != "" { + symbols = strings.Split(symbolsStr, ",") + } + return tokensIDs, symbols, nameStr, nil +} + // Param parsers type paramer interface { diff --git a/api/parsers_test.go b/api/parsers_test.go index 3824c03..8ca069a 100644 --- a/api/parsers_test.go +++ b/api/parsers_test.go @@ -270,3 +270,28 @@ func TestParseQueryTxType(t *testing.T) { assert.NoError(t, err) assert.Equal(t, common.TxTypeTransferToBJJ, *res) } + +func TestParseTokenFilters(t *testing.T) { + ids := "ids" + symbols := "symbols" + name := "name" + nameValue := "1" + symbolsValue := "1,2,3" + idsValue := "2,3,4" + c := &queryParser{} + c.m = make(map[string]string) + // Incorrect values + c.m[name] = nameValue + c.m[ids] = idsValue + c.m[symbols] = symbolsValue + + idsParse, symbolsParse, nameParse, err := parseTokenFilters(c) + assert.NoError(t, err) + + // Correct values + var tokenIds []common.TokenID = []common.TokenID{2, 3, 4} + assert.Equal(t, tokenIds, idsParse) + var symbolsArray []string = []string{"1", "2", "3"} + assert.Equal(t, symbolsArray, symbolsParse) + assert.Equal(t, nameValue, nameParse) +} diff --git a/api/swagger.yml b/api/swagger.yml index b709c8e..35b6c5d 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -1917,6 +1917,8 @@ components: - $ref: '#/components/schemas/EthereumAddress' - description: Ethereum address in which the token is deployed. - example: "0xaa942cfcd25ad4d90a62358b0dd84f33b398262a" + itemId: + $ref: '#/components/schemas/ItemId' name: type: string description: full name of the token @@ -1947,6 +1949,7 @@ components: required: - id - ethereumAddress + - itemId - name - symbol - decimals diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index 21e22de..43ed180 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -298,13 +298,75 @@ func (hdb *HistoryDB) GetToken(tokenID common.TokenID) (*TokenRead, error) { } // GetTokens returns a list of tokens from the DB -func (hdb *HistoryDB) GetTokens() ([]TokenRead, error) { - var tokens []*TokenRead - err := meddler.QueryAll( - hdb.db, &tokens, - "SELECT * FROM token ORDER BY token_id;", - ) - return db.SlicePtrsToSlice(tokens).([]TokenRead), err +func (hdb *HistoryDB) GetTokens(ids []common.TokenID, symbols []string, name string, fromItem, limit *uint, order string) ([]TokenRead, *db.Pagination, error) { + var query string + var args []interface{} + queryStr := `SELECT * , COUNT(*) OVER() AS total_items, MIN(token.item_id) OVER() AS first_item, MAX(token.item_id) OVER() AS last_item FROM token ` + // Apply filters + nextIsAnd := false + if len(ids) > 0 { + queryStr += "WHERE token_id IN (?) " + nextIsAnd = true + args = append(args, ids) + } + if len(symbols) > 0 { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + queryStr += "symbol IN (?) " + args = append(args, symbols) + nextIsAnd = true + } + if name != "" { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + queryStr += "name ~ ? " + args = append(args, name) + nextIsAnd = true + } + if fromItem != nil { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + if order == OrderAsc { + queryStr += "item_id >= ? " + } else { + queryStr += "item_id <= ? " + } + args = append(args, fromItem) + } + // pagination + queryStr += "ORDER BY item_id " + if order == OrderAsc { + queryStr += "ASC " + } else { + queryStr += "DESC " + } + queryStr += fmt.Sprintf("LIMIT %d;", *limit) + query, argsQ, err := sqlx.In(queryStr, args...) + if err != nil { + return nil, nil, err + } + query = hdb.db.Rebind(query) + tokens := []*TokenRead{} + if err := meddler.QueryAll(hdb.db, &tokens, query, argsQ...); err != nil { + return nil, nil, err + } + if len(tokens) == 0 { + return nil, nil, sql.ErrNoRows + } + return db.SlicePtrsToSlice(tokens).([]TokenRead), &db.Pagination{ + TotalItems: tokens[0].TotalItems, + FirstItem: tokens[0].FirstItem, + LastItem: tokens[0].LastItem, + }, nil } // GetTokenSymbols returns all the token symbols from the DB diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index e907a49..95573f6 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -145,8 +145,9 @@ func TestTokens(t *testing.T) { tokens := test.GenTokens(nTokens, blocks) err := historyDB.AddTokens(tokens) assert.NoError(t, err) - // Fetch tokens - fetchedTokens, err := historyDB.GetTokens() + limit := uint(10) + // Fetch tokens6 + fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, OrderAsc) assert.NoError(t, err) // Compare fetched tokens vs generated tokens // All the tokens should have USDUpdate setted by the DB trigger diff --git a/db/historydb/views.go b/db/historydb/views.go index 5898b9d..814b31a 100644 --- a/db/historydb/views.go +++ b/db/historydb/views.go @@ -81,14 +81,18 @@ type txWrite struct { // TokenRead add USD info to common.Token type TokenRead struct { - TokenID common.TokenID `json:"id" meddler:"token_id"` - EthBlockNum int64 `json:"ethereumBlockNum" meddler:"eth_block_num"` // Ethereum block number in which this token was registered - EthAddr ethCommon.Address `json:"ethereumAddress" meddler:"eth_addr"` - Name string `json:"name" meddler:"name"` - Symbol string `json:"symbol" meddler:"symbol"` - Decimals uint64 `json:"decimals" meddler:"decimals"` - USD *float64 `json:"USD" meddler:"usd"` - USDUpdate *time.Time `json:"fiatUpdate" meddler:"usd_update,utctime"` + ItemID int `meddler:"item_id"` + TokenID common.TokenID `meddler:"token_id"` + EthBlockNum int64 `meddler:"eth_block_num"` // Ethereum block number in which this token was registered + EthAddr ethCommon.Address `meddler:"eth_addr"` + Name string `meddler:"name"` + Symbol string `meddler:"symbol"` + Decimals uint64 `meddler:"decimals"` + USD *float64 `meddler:"usd"` + USDUpdate *time.Time `meddler:"usd_update,utctime"` + TotalItems int `meddler:"total_items"` + FirstItem int `meddler:"first_item"` + LastItem int `meddler:"last_item"` } // HistoryExit is a representation of a exit with additional information diff --git a/db/migrations/0001.sql b/db/migrations/0001.sql index 63cc7b2..3bf215a 100644 --- a/db/migrations/0001.sql +++ b/db/migrations/0001.sql @@ -37,7 +37,8 @@ CREATE TABLE bid ( ); CREATE TABLE token ( - token_id INT PRIMARY KEY, + item_id SERIAL PRIMARY KEY, + token_id INT UNIQUE NOT NULL, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, eth_addr BYTEA UNIQUE NOT NULL, name VARCHAR(20) NOT NULL, diff --git a/priceupdater/priceupdater_test.go b/priceupdater/priceupdater_test.go index 4a05a61..fcff3f0 100644 --- a/priceupdater/priceupdater_test.go +++ b/priceupdater/priceupdater_test.go @@ -51,7 +51,8 @@ func TestPriceUpdater(t *testing.T) { // Update prices pu.UpdatePrices() // Check that prices have been updated - fetchedTokens, err := historyDB.GetTokens() + limit := uint(10) + fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, historydb.OrderAsc) assert.NoError(t, err) for _, token := range fetchedTokens { assert.NotNil(t, token.USD) diff --git a/test/historydb.go b/test/historydb.go index b0dd35f..d1c93e7 100644 --- a/test/historydb.go +++ b/test/historydb.go @@ -35,7 +35,7 @@ func GenTokens(nTokens int, blocks []common.Block) []common.Token { for i := 0; i < nTokens; i++ { token := common.Token{ TokenID: common.TokenID(i), - Name: fmt.Sprint(i), + Name: "NAME" + fmt.Sprint(i), Symbol: fmt.Sprint(i), Decimals: uint64(i), EthBlockNum: blocks[i%len(blocks)].EthBlockNum,