Add get tokens endpoint
laisolizq laisolizq@gmail.com
Thu, 15 Oct 2020 18:58:24 +0200
12 files changed,
338 insertions(+),
46 deletions(-)
M
api/api_test.go
→
api/api_test.go
@@ -37,7 +37,7 @@ const apiURL = "http://localhost" + apiPort + "/"
type testCommon struct { blocks []common.Block - tokens []historydb.TokenRead + tokens []tokenAPI batches []common.Batch usrAddr string usrBjj string@@ -201,9 +201,9 @@ if err != nil {
panic(err) } // Set token value - tokensUSD := []historydb.TokenRead{} + tokensUSD := []tokenAPI{} for i, tkn := range tokens { - token := historydb.TokenRead{ + token := tokenAPI{ TokenID: tkn.TokenID, EthBlockNum: tkn.EthBlockNum, EthAddr: tkn.EthAddr,@@ -287,7 +287,7 @@ }
} panic("timesamp not found") } - getToken := func(id common.TokenID) historydb.TokenRead { + getToken := func(id common.TokenID) tokenAPI { for i := 0; i < len(tokensUSD); i++ { if tokensUSD[i].TokenID == id { return tokensUSD[i]@@ -295,7 +295,7 @@ }
} panic("token not found") } - getTokenByIdx := func(idx common.Idx) historydb.TokenRead { + getTokenByIdx := func(idx common.Idx) tokenAPI { for _, acc := range accs { if idx == acc.Idx { return getToken(acc.TokenID)@@ -874,18 +874,113 @@
func TestGetToken(t *testing.T) { // Get all txs by their ID endpoint := apiURL + "tokens/" - fetchedTokens := []historydb.TokenRead{} + fetchedTokens := []tokenAPI{} for _, token := range tc.tokens { - fetchedToken := historydb.TokenRead{} + fetchedToken := tokenAPI{} assert.NoError(t, doGoodReq("GET", endpoint+strconv.Itoa(int(token.TokenID)), nil, &fetchedToken)) fetchedTokens = append(fetchedTokens, fetchedToken) } assertTokensAPIs(t, tc.tokens, fetchedTokens) } -func assertTokensAPIs(t *testing.T, expected, actual []historydb.TokenRead) { +func TestGetTokens(t *testing.T) { + endpoint := apiURL + "tokens" + fetchedTokens := []tokenAPI{} + appendIter := func(intr interface{}) { + for i := 0; i < len(intr.(*tokensAPI).Tokens); i++ { + tmp, err := copystructure.Copy(intr.(*tokensAPI).Tokens[i]) + if err != nil { + panic(err) + } + fetchedTokens = append(fetchedTokens, tmp.(tokenAPI)) + } + } + // Get all (no filters) + limit := 8 + path := fmt.Sprintf("%s?limit=%d&fromItem=", endpoint, limit) + err := doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + assertTokensAPIs(t, tc.tokens, fetchedTokens) + + // Get by tokenIds + fetchedTokens = []tokenAPI{} + limit = 7 + stringIds := strconv.Itoa(int(tc.tokens[2].TokenID)) + "," + strconv.Itoa(int(tc.tokens[5].TokenID)) + "," + strconv.Itoa(int(tc.tokens[6].TokenID)) + path = fmt.Sprintf( + "%s?ids=%s&limit=%d&fromItem=", + endpoint, stringIds, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + var tokensFiltered []tokenAPI + tokensFiltered = append(tokensFiltered, tc.tokens[2]) + tokensFiltered = append(tokensFiltered, tc.tokens[5]) + tokensFiltered = append(tokensFiltered, tc.tokens[6]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // Get by symbols + fetchedTokens = []tokenAPI{} + limit = 7 + stringSymbols := tc.tokens[1].Symbol + "," + tc.tokens[3].Symbol + path = fmt.Sprintf( + "%s?symbols=%s&limit=%d&fromItem=", + endpoint, stringSymbols, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + tokensFiltered = nil + tokensFiltered = append(tokensFiltered, tc.tokens[1]) + tokensFiltered = append(tokensFiltered, tc.tokens[3]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // Get by name + fetchedTokens = []tokenAPI{} + limit = 5 + stringName := tc.tokens[8].Name[4:5] + path = fmt.Sprintf( + "%s?name=%s&limit=%d&fromItem=", + endpoint, stringName, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + tokensFiltered = nil + tokensFiltered = append(tokensFiltered, tc.tokens[8]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // Multiple filters + fetchedTokens = []tokenAPI{} + limit = 5 + stringSymbols = tc.tokens[2].Symbol + "," + tc.tokens[6].Symbol + stringIds = strconv.Itoa(int(tc.tokens[2].TokenID)) + "," + strconv.Itoa(int(tc.tokens[5].TokenID)) + "," + strconv.Itoa(int(tc.tokens[6].TokenID)) + path = fmt.Sprintf( + "%s?symbols=%s&ids=%s&limit=%d&fromItem=", + endpoint, stringSymbols, stringIds, limit, + ) + err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + + tokensFiltered = nil + tokensFiltered = append(tokensFiltered, tc.tokens[2]) + tokensFiltered = append(tokensFiltered, tc.tokens[6]) + assertTokensAPIs(t, tokensFiltered, fetchedTokens) + + // All, in reverse order + fetchedTokens = []tokenAPI{} + limit = 5 + path = fmt.Sprintf("%s?limit=%d&fromItem=", endpoint, limit) + err = doGoodReqPaginated(path, historydb.OrderDesc, &tokensAPI{}, appendIter) + assert.NoError(t, err) + flipedTokens := []tokenAPI{} + for i := 0; i < len(tc.tokens); i++ { + flipedTokens = append(flipedTokens, tc.tokens[len(tc.tokens)-1-i]) + } + assertTokensAPIs(t, flipedTokens, fetchedTokens) +} + +func assertTokensAPIs(t *testing.T, expected, actual []tokenAPI) { require.Equal(t, len(expected), len(actual)) for i := 0; i < len(actual); i++ { //nolint len(actual) won't change within the loop + actual[i].ItemID = 0 if expected[i].USDUpdate == nil { assert.Equal(t, expected[i].USDUpdate, actual[i].USDUpdate) } else {
M
api/dbtoapistructs.go
→
api/dbtoapistructs.go
@@ -71,20 +71,20 @@ Nonce common.Nonce `json:"nonce"`
} type historyTxAPI struct { - IsL1 string `json:"L1orL2"` - TxID string `json:"id"` - ItemID int `json:"itemId"` - Type common.TxType `json:"type"` - Position int `json:"position"` - FromIdx *string `json:"fromAccountIndex"` - ToIdx string `json:"toAccountIndex"` - Amount string `json:"amount"` - BatchNum *common.BatchNum `json:"batchNum"` - HistoricUSD *float64 `json:"historicUSD"` - Timestamp time.Time `json:"timestamp"` - L1Info *l1Info `json:"L1Info"` - L2Info *l2Info `json:"L2Info"` - Token historydb.TokenRead `json:"token"` + IsL1 string `json:"L1orL2"` + TxID string `json:"id"` + ItemID int `json:"itemId"` + Type common.TxType `json:"type"` + Position int `json:"position"` + FromIdx *string `json:"fromAccountIndex"` + ToIdx string `json:"toAccountIndex"` + Amount string `json:"amount"` + BatchNum *common.BatchNum `json:"batchNum"` + HistoricUSD *float64 `json:"historicUSD"` + Timestamp time.Time `json:"timestamp"` + L1Info *l1Info `json:"L1Info"` + L2Info *l2Info `json:"L2Info"` + Token tokenAPI `json:"token"` } func historyTxsToAPI(dbTxs []historydb.HistoryTx) []historyTxAPI {@@ -100,7 +100,7 @@ Amount: dbTxs[i].Amount.String(),
HistoricUSD: dbTxs[i].HistoricUSD, BatchNum: dbTxs[i].BatchNum, Timestamp: dbTxs[i].Timestamp, - Token: historydb.TokenRead{ + Token: tokenAPI{ TokenID: dbTxs[i].TokenID, EthBlockNum: dbTxs[i].TokenEthBlockNum, EthAddr: dbTxs[i].TokenEthAddr,@@ -170,7 +170,7 @@ Balance string `json:"balance"`
InstantWithdrawn *int64 `json:"instantWithdrawn"` DelayedWithdrawRequest *int64 `json:"delayedWithdrawRequest"` DelayedWithdrawn *int64 `json:"delayedWithdrawn"` - Token historydb.TokenRead `json:"token"` + Token tokenAPI `json:"token"` } func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI {@@ -185,7 +185,7 @@ Balance: dbExits[i].Balance.String(),
InstantWithdrawn: dbExits[i].InstantWithdrawn, DelayedWithdrawRequest: dbExits[i].DelayedWithdrawRequest, DelayedWithdrawn: dbExits[i].DelayedWithdrawn, - Token: historydb.TokenRead{ + Token: tokenAPI{ TokenID: dbExits[i].TokenID, EthBlockNum: dbExits[i].TokenEthBlockNum, EthAddr: dbExits[i].TokenEthAddr,@@ -199,3 +199,52 @@ })
} return apiExits } + +// Tokens + +type tokensAPI struct { + Tokens []tokenAPI `json:"tokens"` + Pagination *db.Pagination `json:"pagination"` +} + +func (t *tokensAPI) GetPagination() *db.Pagination { + if t.Tokens[0].ItemID < t.Tokens[len(t.Tokens)-1].ItemID { + t.Pagination.FirstReturnedItem = t.Tokens[0].ItemID + t.Pagination.LastReturnedItem = t.Tokens[len(t.Tokens)-1].ItemID + } else { + t.Pagination.LastReturnedItem = t.Tokens[0].ItemID + t.Pagination.FirstReturnedItem = t.Tokens[len(t.Tokens)-1].ItemID + } + return t.Pagination +} +func (t *tokensAPI) Len() int { return len(t.Tokens) } + +type tokenAPI struct { + ItemID int `json:"itemId"` + TokenID common.TokenID `json:"id"` + EthBlockNum int64 `json:"ethereumBlockNum"` // Ethereum block number in which this token was registered + EthAddr ethCommon.Address `json:"ethereumAddress"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals uint64 `json:"decimals"` + USD *float64 `json:"USD"` + USDUpdate *time.Time `json:"fiatUpdate"` +} + +func tokensToAPI(dbTokens []historydb.TokenRead) []tokenAPI { + apiTokens := []tokenAPI{} + for i := 0; i < len(dbTokens); i++ { + apiTokens = append(apiTokens, tokenAPI{ + ItemID: dbTokens[i].ItemID, + TokenID: dbTokens[i].TokenID, + EthBlockNum: dbTokens[i].EthBlockNum, + EthAddr: dbTokens[i].EthAddr, + Name: dbTokens[i].Name, + Symbol: dbTokens[i].Symbol, + Decimals: dbTokens[i].Decimals, + USD: dbTokens[i].USD, + USDUpdate: dbTokens[i].USDUpdate, + }) + } + return apiTokens +}
M
api/handlers.go
→
api/handlers.go
@@ -202,7 +202,33 @@
} func getTokens(c *gin.Context) { + // Account filters + tokenIDs, symbols, name, err := parseTokenFilters(c) + if err != nil { + retBadReq(err, c) + return + } + // Pagination + fromItem, order, limit, err := parsePagination(c) + if err != nil { + retBadReq(err, c) + return + } + // Fetch exits from historyDB + tokens, pagination, err := h.GetTokens( + tokenIDs, symbols, name, fromItem, limit, order, + ) + if err != nil { + retSQLErr(err, c) + return + } + // Build succesfull response + apiTokens := tokensToAPI(tokens) + c.JSON(http.StatusOK, &tokensAPI{ + Tokens: apiTokens, + Pagination: pagination, + }) } func getToken(c *gin.Context) {@@ -219,7 +245,8 @@ if err != nil {
retSQLErr(err, c) return } - c.JSON(http.StatusOK, token) + apiToken := tokensToAPI([]historydb.TokenRead{*token}) + c.JSON(http.StatusOK, apiToken[0]) } func getRecommendedFee(c *gin.Context) {
M
api/parsers.go
→
api/parsers.go
@@ -219,6 +219,30 @@ }
return tokenID, addr, bjj, idx, nil } +func parseTokenFilters(c querier) ([]common.TokenID, []string, string, error) { + idsStr := c.Query("ids") + symbolsStr := c.Query("symbols") + nameStr := c.Query("name") + var tokensIDs []common.TokenID + if idsStr != "" { + ids := strings.Split(idsStr, ",") + + for _, id := range ids { + idUint, err := strconv.Atoi(id) + if err != nil { + return nil, nil, "", err + } + tokenID := common.TokenID(idUint) + tokensIDs = append(tokensIDs, tokenID) + } + } + var symbols []string + if symbolsStr != "" { + symbols = strings.Split(symbolsStr, ",") + } + return tokensIDs, symbols, nameStr, nil +} + // Param parsers type paramer interface {
M
api/parsers_test.go
→
api/parsers_test.go
@@ -270,3 +270,28 @@ res, err = parseQueryTxType(c)
assert.NoError(t, err) assert.Equal(t, common.TxTypeTransferToBJJ, *res) } + +func TestParseTokenFilters(t *testing.T) { + ids := "ids" + symbols := "symbols" + name := "name" + nameValue := "1" + symbolsValue := "1,2,3" + idsValue := "2,3,4" + c := &queryParser{} + c.m = make(map[string]string) + // Incorrect values + c.m[name] = nameValue + c.m[ids] = idsValue + c.m[symbols] = symbolsValue + + idsParse, symbolsParse, nameParse, err := parseTokenFilters(c) + assert.NoError(t, err) + + // Correct values + var tokenIds []common.TokenID = []common.TokenID{2, 3, 4} + assert.Equal(t, tokenIds, idsParse) + var symbolsArray []string = []string{"1", "2", "3"} + assert.Equal(t, symbolsArray, symbolsParse) + assert.Equal(t, nameValue, nameParse) +}
M
api/swagger.yml
→
api/swagger.yml
@@ -1917,6 +1917,8 @@ allOf:
- $ref: '#/components/schemas/EthereumAddress' - description: Ethereum address in which the token is deployed. - example: "0xaa942cfcd25ad4d90a62358b0dd84f33b398262a" + itemId: + $ref: '#/components/schemas/ItemId' name: type: string description: full name of the token@@ -1947,6 +1949,7 @@ nullable: true
required: - id - ethereumAddress + - itemId - name - symbol - decimals
M
db/historydb/historydb.go
→
db/historydb/historydb.go
@@ -298,13 +298,75 @@ return token, err
} // GetTokens returns a list of tokens from the DB -func (hdb *HistoryDB) GetTokens() ([]TokenRead, error) { - var tokens []*TokenRead - err := meddler.QueryAll( - hdb.db, &tokens, - "SELECT * FROM token ORDER BY token_id;", - ) - return db.SlicePtrsToSlice(tokens).([]TokenRead), err +func (hdb *HistoryDB) GetTokens(ids []common.TokenID, symbols []string, name string, fromItem, limit *uint, order string) ([]TokenRead, *db.Pagination, error) { + var query string + var args []interface{} + queryStr := `SELECT * , COUNT(*) OVER() AS total_items, MIN(token.item_id) OVER() AS first_item, MAX(token.item_id) OVER() AS last_item FROM token ` + // Apply filters + nextIsAnd := false + if len(ids) > 0 { + queryStr += "WHERE token_id IN (?) " + nextIsAnd = true + args = append(args, ids) + } + if len(symbols) > 0 { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + queryStr += "symbol IN (?) " + args = append(args, symbols) + nextIsAnd = true + } + if name != "" { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + queryStr += "name ~ ? " + args = append(args, name) + nextIsAnd = true + } + if fromItem != nil { + if nextIsAnd { + queryStr += "AND " + } else { + queryStr += "WHERE " + } + if order == OrderAsc { + queryStr += "item_id >= ? " + } else { + queryStr += "item_id <= ? " + } + args = append(args, fromItem) + } + // pagination + queryStr += "ORDER BY item_id " + if order == OrderAsc { + queryStr += "ASC " + } else { + queryStr += "DESC " + } + queryStr += fmt.Sprintf("LIMIT %d;", *limit) + query, argsQ, err := sqlx.In(queryStr, args...) + if err != nil { + return nil, nil, err + } + query = hdb.db.Rebind(query) + tokens := []*TokenRead{} + if err := meddler.QueryAll(hdb.db, &tokens, query, argsQ...); err != nil { + return nil, nil, err + } + if len(tokens) == 0 { + return nil, nil, sql.ErrNoRows + } + return db.SlicePtrsToSlice(tokens).([]TokenRead), &db.Pagination{ + TotalItems: tokens[0].TotalItems, + FirstItem: tokens[0].FirstItem, + LastItem: tokens[0].LastItem, + }, nil } // GetTokenSymbols returns all the token symbols from the DB
M
db/historydb/historydb_test.go
→
db/historydb/historydb_test.go
@@ -145,8 +145,9 @@ const nTokens = 5
tokens := test.GenTokens(nTokens, blocks) err := historyDB.AddTokens(tokens) assert.NoError(t, err) - // Fetch tokens - fetchedTokens, err := historyDB.GetTokens() + limit := uint(10) + // Fetch tokens6 + fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, OrderAsc) assert.NoError(t, err) // Compare fetched tokens vs generated tokens // All the tokens should have USDUpdate setted by the DB trigger
M
db/historydb/views.go
→
db/historydb/views.go
@@ -81,14 +81,18 @@ }
// TokenRead add USD info to common.Token type TokenRead struct { - TokenID common.TokenID `json:"id" meddler:"token_id"` - EthBlockNum int64 `json:"ethereumBlockNum" meddler:"eth_block_num"` // Ethereum block number in which this token was registered - EthAddr ethCommon.Address `json:"ethereumAddress" meddler:"eth_addr"` - Name string `json:"name" meddler:"name"` - Symbol string `json:"symbol" meddler:"symbol"` - Decimals uint64 `json:"decimals" meddler:"decimals"` - USD *float64 `json:"USD" meddler:"usd"` - USDUpdate *time.Time `json:"fiatUpdate" meddler:"usd_update,utctime"` + ItemID int `meddler:"item_id"` + TokenID common.TokenID `meddler:"token_id"` + EthBlockNum int64 `meddler:"eth_block_num"` // Ethereum block number in which this token was registered + EthAddr ethCommon.Address `meddler:"eth_addr"` + Name string `meddler:"name"` + Symbol string `meddler:"symbol"` + Decimals uint64 `meddler:"decimals"` + USD *float64 `meddler:"usd"` + USDUpdate *time.Time `meddler:"usd_update,utctime"` + TotalItems int `meddler:"total_items"` + FirstItem int `meddler:"first_item"` + LastItem int `meddler:"last_item"` } // HistoryExit is a representation of a exit with additional information
M
db/migrations/0001.sql
→
db/migrations/0001.sql
@@ -37,7 +37,8 @@ PRIMARY KEY (slot_num, bid_value)
); CREATE TABLE token ( - token_id INT PRIMARY KEY, + item_id SERIAL PRIMARY KEY, + token_id INT UNIQUE NOT NULL, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, eth_addr BYTEA UNIQUE NOT NULL, name VARCHAR(20) NOT NULL,
M
priceupdater/priceupdater_test.go
→
priceupdater/priceupdater_test.go
@@ -51,7 +51,8 @@ assert.NoError(t, pu.UpdateTokenList())
// Update prices pu.UpdatePrices() // Check that prices have been updated - fetchedTokens, err := historyDB.GetTokens() + limit := uint(10) + fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, historydb.OrderAsc) assert.NoError(t, err) for _, token := range fetchedTokens { assert.NotNil(t, token.USD)
M
test/historydb.go
→
test/historydb.go
@@ -35,7 +35,7 @@ tokens := []common.Token{}
for i := 0; i < nTokens; i++ { token := common.Token{ TokenID: common.TokenID(i), - Name: fmt.Sprint(i), + Name: "NAME" + fmt.Sprint(i), Symbol: fmt.Sprint(i), Decimals: uint64(i), EthBlockNum: blocks[i%len(blocks)].EthBlockNum,