Merge pull request #204 from hermeznetwork/feature/api-gettokens

Add get tokens endpoint

api/api_test.go

@@ -37,7 +37,7 @@ const apiURL = "http://localhost" + apiPort + "/"
 
 type testCommon struct {
 	blocks   []common.Block
-	tokens   []historydb.TokenRead
+	tokens   []tokenAPI
 	batches  []common.Batch
 	usrAddr  string
 	usrBjj   string
@@ -201,9 +201,9 @@ func TestMain(m *testing.M) {
 		panic(err)
 	}
 	// Set token value
-	tokensUSD := []historydb.TokenRead{}
+	tokensUSD := []tokenAPI{}
 	for i, tkn := range tokens {
-		token := historydb.TokenRead{
+		token := tokenAPI{
 			TokenID:     tkn.TokenID,
 			EthBlockNum: tkn.EthBlockNum,
 			EthAddr:     tkn.EthAddr,
@@ -287,7 +287,7 @@ func TestMain(m *testing.M) {
 		}
 		panic("timesamp not found")
 	}
-	getToken := func(id common.TokenID) historydb.TokenRead {
+	getToken := func(id common.TokenID) tokenAPI {
 		for i := 0; i < len(tokensUSD); i++ {
 			if tokensUSD[i].TokenID == id {
 				return tokensUSD[i]
@@ -295,7 +295,7 @@ func TestMain(m *testing.M) {
 		}
 		panic("token not found")
 	}
-	getTokenByIdx := func(idx common.Idx) historydb.TokenRead {
+	getTokenByIdx := func(idx common.Idx) tokenAPI {
 		for _, acc := range accs {
 			if idx == acc.Idx {
 				return getToken(acc.TokenID)
@@ -874,18 +874,113 @@ func assertExitAPIs(t *testing.T, expected, actual []exitAPI) {
 func TestGetToken(t *testing.T) {
 	// Get all txs by their ID
 	endpoint := apiURL + "tokens/"
-	fetchedTokens := []historydb.TokenRead{}
+	fetchedTokens := []tokenAPI{}
 	for _, token := range tc.tokens {
-		fetchedToken := historydb.TokenRead{}
+		fetchedToken := tokenAPI{}
 		assert.NoError(t, doGoodReq("GET", endpoint+strconv.Itoa(int(token.TokenID)), nil, &fetchedToken))
 		fetchedTokens = append(fetchedTokens, fetchedToken)
 	}
 	assertTokensAPIs(t, tc.tokens, fetchedTokens)
 }
 
-func assertTokensAPIs(t *testing.T, expected, actual []historydb.TokenRead) {
+func TestGetTokens(t *testing.T) {
+	endpoint := apiURL + "tokens"
+	fetchedTokens := []tokenAPI{}
+	appendIter := func(intr interface{}) {
+		for i := 0; i < len(intr.(*tokensAPI).Tokens); i++ {
+			tmp, err := copystructure.Copy(intr.(*tokensAPI).Tokens[i])
+			if err != nil {
+				panic(err)
+			}
+			fetchedTokens = append(fetchedTokens, tmp.(tokenAPI))
+		}
+	}
+	// Get all (no filters)
+	limit := 8
+	path := fmt.Sprintf("%s?limit=%d&fromItem=", endpoint, limit)
+	err := doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter)
+	assert.NoError(t, err)
+	assertTokensAPIs(t, tc.tokens, fetchedTokens)
+
+	// Get by tokenIds
+	fetchedTokens = []tokenAPI{}
+	limit = 7
+	stringIds := strconv.Itoa(int(tc.tokens[2].TokenID)) + "," + strconv.Itoa(int(tc.tokens[5].TokenID)) + "," + strconv.Itoa(int(tc.tokens[6].TokenID))
+	path = fmt.Sprintf(
+		"%s?ids=%s&limit=%d&fromItem=",
+		endpoint, stringIds, limit,
+	)
+	err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter)
+	assert.NoError(t, err)
+	var tokensFiltered []tokenAPI
+	tokensFiltered = append(tokensFiltered, tc.tokens[2])
+	tokensFiltered = append(tokensFiltered, tc.tokens[5])
+	tokensFiltered = append(tokensFiltered, tc.tokens[6])
+	assertTokensAPIs(t, tokensFiltered, fetchedTokens)
+
+	// Get by symbols
+	fetchedTokens = []tokenAPI{}
+	limit = 7
+	stringSymbols := tc.tokens[1].Symbol + "," + tc.tokens[3].Symbol
+	path = fmt.Sprintf(
+		"%s?symbols=%s&limit=%d&fromItem=",
+		endpoint, stringSymbols, limit,
+	)
+	err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter)
+	assert.NoError(t, err)
+	tokensFiltered = nil
+	tokensFiltered = append(tokensFiltered, tc.tokens[1])
+	tokensFiltered = append(tokensFiltered, tc.tokens[3])
+	assertTokensAPIs(t, tokensFiltered, fetchedTokens)
+
+	// Get by name
+	fetchedTokens = []tokenAPI{}
+	limit = 5
+	stringName := tc.tokens[8].Name[4:5]
+	path = fmt.Sprintf(
+		"%s?name=%s&limit=%d&fromItem=",
+		endpoint, stringName, limit,
+	)
+	err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter)
+	assert.NoError(t, err)
+	tokensFiltered = nil
+	tokensFiltered = append(tokensFiltered, tc.tokens[8])
+	assertTokensAPIs(t, tokensFiltered, fetchedTokens)
+
+	// Multiple filters
+	fetchedTokens = []tokenAPI{}
+	limit = 5
+	stringSymbols = tc.tokens[2].Symbol + "," + tc.tokens[6].Symbol
+	stringIds = strconv.Itoa(int(tc.tokens[2].TokenID)) + "," + strconv.Itoa(int(tc.tokens[5].TokenID)) + "," + strconv.Itoa(int(tc.tokens[6].TokenID))
+	path = fmt.Sprintf(
+		"%s?symbols=%s&ids=%s&limit=%d&fromItem=",
+		endpoint, stringSymbols, stringIds, limit,
+	)
+	err = doGoodReqPaginated(path, historydb.OrderAsc, &tokensAPI{}, appendIter)
+	assert.NoError(t, err)
+
+	tokensFiltered = nil
+	tokensFiltered = append(tokensFiltered, tc.tokens[2])
+	tokensFiltered = append(tokensFiltered, tc.tokens[6])
+	assertTokensAPIs(t, tokensFiltered, fetchedTokens)
+
+	// All, in reverse order
+	fetchedTokens = []tokenAPI{}
+	limit = 5
+	path = fmt.Sprintf("%s?limit=%d&fromItem=", endpoint, limit)
+	err = doGoodReqPaginated(path, historydb.OrderDesc, &tokensAPI{}, appendIter)
+	assert.NoError(t, err)
+	flipedTokens := []tokenAPI{}
+	for i := 0; i < len(tc.tokens); i++ {
+		flipedTokens = append(flipedTokens, tc.tokens[len(tc.tokens)-1-i])
+	}
+	assertTokensAPIs(t, flipedTokens, fetchedTokens)
+}
+
+func assertTokensAPIs(t *testing.T, expected, actual []tokenAPI) {
 	require.Equal(t, len(expected), len(actual))
 	for i := 0; i < len(actual); i++ { //nolint len(actual) won't change within the loop
+		actual[i].ItemID = 0
 		if expected[i].USDUpdate == nil {
 			assert.Equal(t, expected[i].USDUpdate, actual[i].USDUpdate)
 		} else {

api/dbtoapistructs.go

@@ -71,20 +71,20 @@ type l2Info struct {
 }
 
 type historyTxAPI struct {
-	IsL1        string              `json:"L1orL2"`
-	TxID        string              `json:"id"`
-	ItemID      int                 `json:"itemId"`
-	Type        common.TxType       `json:"type"`
-	Position    int                 `json:"position"`
-	FromIdx     *string             `json:"fromAccountIndex"`
-	ToIdx       string              `json:"toAccountIndex"`
-	Amount      string              `json:"amount"`
-	BatchNum    *common.BatchNum    `json:"batchNum"`
-	HistoricUSD *float64            `json:"historicUSD"`
-	Timestamp   time.Time           `json:"timestamp"`
-	L1Info      *l1Info             `json:"L1Info"`
-	L2Info      *l2Info             `json:"L2Info"`
-	Token       historydb.TokenRead `json:"token"`
+	IsL1        string           `json:"L1orL2"`
+	TxID        string           `json:"id"`
+	ItemID      int              `json:"itemId"`
+	Type        common.TxType    `json:"type"`
+	Position    int              `json:"position"`
+	FromIdx     *string          `json:"fromAccountIndex"`
+	ToIdx       string           `json:"toAccountIndex"`
+	Amount      string           `json:"amount"`
+	BatchNum    *common.BatchNum `json:"batchNum"`
+	HistoricUSD *float64         `json:"historicUSD"`
+	Timestamp   time.Time        `json:"timestamp"`
+	L1Info      *l1Info          `json:"L1Info"`
+	L2Info      *l2Info          `json:"L2Info"`
+	Token       tokenAPI         `json:"token"`
 }
 
 func historyTxsToAPI(dbTxs []historydb.HistoryTx) []historyTxAPI {
@@ -100,7 +100,7 @@ func historyTxsToAPI(dbTxs []historydb.HistoryTx) []historyTxAPI {
 			HistoricUSD: dbTxs[i].HistoricUSD,
 			BatchNum:    dbTxs[i].BatchNum,
 			Timestamp:   dbTxs[i].Timestamp,
-			Token: historydb.TokenRead{
+			Token: tokenAPI{
 				TokenID:     dbTxs[i].TokenID,
 				EthBlockNum: dbTxs[i].TokenEthBlockNum,
 				EthAddr:     dbTxs[i].TokenEthAddr,
@@ -170,7 +170,7 @@ type exitAPI struct {
 	InstantWithdrawn       *int64                          `json:"instantWithdrawn"`
 	DelayedWithdrawRequest *int64                          `json:"delayedWithdrawRequest"`
 	DelayedWithdrawn       *int64                          `json:"delayedWithdrawn"`
-	Token                  historydb.TokenRead             `json:"token"`
+	Token                  tokenAPI                        `json:"token"`
 }
 
 func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI {
@@ -185,7 +185,7 @@ func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI {
 			InstantWithdrawn:       dbExits[i].InstantWithdrawn,
 			DelayedWithdrawRequest: dbExits[i].DelayedWithdrawRequest,
 			DelayedWithdrawn:       dbExits[i].DelayedWithdrawn,
-			Token: historydb.TokenRead{
+			Token: tokenAPI{
 				TokenID:     dbExits[i].TokenID,
 				EthBlockNum: dbExits[i].TokenEthBlockNum,
 				EthAddr:     dbExits[i].TokenEthAddr,
@@ -199,3 +199,52 @@ func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI {
 	}
 	return apiExits
 }
+
+// Tokens
+
+type tokensAPI struct {
+	Tokens     []tokenAPI     `json:"tokens"`
+	Pagination *db.Pagination `json:"pagination"`
+}
+
+func (t *tokensAPI) GetPagination() *db.Pagination {
+	if t.Tokens[0].ItemID < t.Tokens[len(t.Tokens)-1].ItemID {
+		t.Pagination.FirstReturnedItem = t.Tokens[0].ItemID
+		t.Pagination.LastReturnedItem = t.Tokens[len(t.Tokens)-1].ItemID
+	} else {
+		t.Pagination.LastReturnedItem = t.Tokens[0].ItemID
+		t.Pagination.FirstReturnedItem = t.Tokens[len(t.Tokens)-1].ItemID
+	}
+	return t.Pagination
+}
+func (t *tokensAPI) Len() int { return len(t.Tokens) }
+
+type tokenAPI struct {
+	ItemID      int               `json:"itemId"`
+	TokenID     common.TokenID    `json:"id"`
+	EthBlockNum int64             `json:"ethereumBlockNum"` // Ethereum block number in which this token was registered
+	EthAddr     ethCommon.Address `json:"ethereumAddress"`
+	Name        string            `json:"name"`
+	Symbol      string            `json:"symbol"`
+	Decimals    uint64            `json:"decimals"`
+	USD         *float64          `json:"USD"`
+	USDUpdate   *time.Time        `json:"fiatUpdate"`
+}
+
+func tokensToAPI(dbTokens []historydb.TokenRead) []tokenAPI {
+	apiTokens := []tokenAPI{}
+	for i := 0; i < len(dbTokens); i++ {
+		apiTokens = append(apiTokens, tokenAPI{
+			ItemID:      dbTokens[i].ItemID,
+			TokenID:     dbTokens[i].TokenID,
+			EthBlockNum: dbTokens[i].EthBlockNum,
+			EthAddr:     dbTokens[i].EthAddr,
+			Name:        dbTokens[i].Name,
+			Symbol:      dbTokens[i].Symbol,
+			Decimals:    dbTokens[i].Decimals,
+			USD:         dbTokens[i].USD,
+			USDUpdate:   dbTokens[i].USDUpdate,
+		})
+	}
+	return apiTokens
+}

api/handlers.go

@@ -202,7 +202,33 @@ func getConfig(c *gin.Context) {
 }
 
 func getTokens(c *gin.Context) {
+	// Account filters
+	tokenIDs, symbols, name, err := parseTokenFilters(c)
+	if err != nil {
+		retBadReq(err, c)
+		return
+	}
+	// Pagination
+	fromItem, order, limit, err := parsePagination(c)
+	if err != nil {
+		retBadReq(err, c)
+		return
+	}
+	// Fetch exits from historyDB
+	tokens, pagination, err := h.GetTokens(
+		tokenIDs, symbols, name, fromItem, limit, order,
+	)
+	if err != nil {
+		retSQLErr(err, c)
+		return
+	}
 
+	// Build succesfull response
+	apiTokens := tokensToAPI(tokens)
+	c.JSON(http.StatusOK, &tokensAPI{
+		Tokens:     apiTokens,
+		Pagination: pagination,
+	})
 }
 
 func getToken(c *gin.Context) {
@@ -219,7 +245,8 @@ func getToken(c *gin.Context) {
 		retSQLErr(err, c)
 		return
 	}
-	c.JSON(http.StatusOK, token)
+	apiToken := tokensToAPI([]historydb.TokenRead{*token})
+	c.JSON(http.StatusOK, apiToken[0])
 }
 
 func getRecommendedFee(c *gin.Context) {

api/parsers.go

@@ -219,6 +219,30 @@ func parseAccountFilters(c querier) (*common.TokenID, *ethCommon.Address, *babyj
 	return tokenID, addr, bjj, idx, nil
 }
 
+func parseTokenFilters(c querier) ([]common.TokenID, []string, string, error) {
+	idsStr := c.Query("ids")
+	symbolsStr := c.Query("symbols")
+	nameStr := c.Query("name")
+	var tokensIDs []common.TokenID
+	if idsStr != "" {
+		ids := strings.Split(idsStr, ",")
+
+		for _, id := range ids {
+			idUint, err := strconv.Atoi(id)
+			if err != nil {
+				return nil, nil, "", err
+			}
+			tokenID := common.TokenID(idUint)
+			tokensIDs = append(tokensIDs, tokenID)
+		}
+	}
+	var symbols []string
+	if symbolsStr != "" {
+		symbols = strings.Split(symbolsStr, ",")
+	}
+	return tokensIDs, symbols, nameStr, nil
+}
+
 // Param parsers
 
 type paramer interface {

api/parsers_test.go

@@ -270,3 +270,28 @@ func TestParseQueryTxType(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, common.TxTypeTransferToBJJ, *res)
 }
+
+func TestParseTokenFilters(t *testing.T) {
+	ids := "ids"
+	symbols := "symbols"
+	name := "name"
+	nameValue := "1"
+	symbolsValue := "1,2,3"
+	idsValue := "2,3,4"
+	c := &queryParser{}
+	c.m = make(map[string]string)
+	// Incorrect values
+	c.m[name] = nameValue
+	c.m[ids] = idsValue
+	c.m[symbols] = symbolsValue
+
+	idsParse, symbolsParse, nameParse, err := parseTokenFilters(c)
+	assert.NoError(t, err)
+
+	// Correct values
+	var tokenIds []common.TokenID = []common.TokenID{2, 3, 4}
+	assert.Equal(t, tokenIds, idsParse)
+	var symbolsArray []string = []string{"1", "2", "3"}
+	assert.Equal(t, symbolsArray, symbolsParse)
+	assert.Equal(t, nameValue, nameParse)
+}

api/swagger.yml

@@ -1917,6 +1917,8 @@ components:
             - $ref: '#/components/schemas/EthereumAddress'
             - description: Ethereum address in which the token is deployed.
             - example: "0xaa942cfcd25ad4d90a62358b0dd84f33b398262a"
+        itemId:
+          $ref: '#/components/schemas/ItemId'
         name:
           type: string
           description: full name of the token
@@ -1947,6 +1949,7 @@ components:
       required:
         - id
         - ethereumAddress
+        - itemId
         - name
         - symbol
         - decimals

db/historydb/historydb.go

@@ -298,13 +298,75 @@ func (hdb *HistoryDB) GetToken(tokenID common.TokenID) (*TokenRead, error) {
 }
 
 // GetTokens returns a list of tokens from the DB
-func (hdb *HistoryDB) GetTokens() ([]TokenRead, error) {
-	var tokens []*TokenRead
-	err := meddler.QueryAll(
-		hdb.db, &tokens,
-		"SELECT * FROM token ORDER BY token_id;",
-	)
-	return db.SlicePtrsToSlice(tokens).([]TokenRead), err
+func (hdb *HistoryDB) GetTokens(ids []common.TokenID, symbols []string, name string, fromItem, limit *uint, order string) ([]TokenRead, *db.Pagination, error) {
+	var query string
+	var args []interface{}
+	queryStr := `SELECT * , COUNT(*) OVER() AS total_items, MIN(token.item_id) OVER() AS first_item, MAX(token.item_id) OVER() AS last_item FROM token `
+	// Apply filters
+	nextIsAnd := false
+	if len(ids) > 0 {
+		queryStr += "WHERE token_id IN (?) "
+		nextIsAnd = true
+		args = append(args, ids)
+	}
+	if len(symbols) > 0 {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		queryStr += "symbol IN (?) "
+		args = append(args, symbols)
+		nextIsAnd = true
+	}
+	if name != "" {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		queryStr += "name ~ ? "
+		args = append(args, name)
+		nextIsAnd = true
+	}
+	if fromItem != nil {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		if order == OrderAsc {
+			queryStr += "item_id >= ? "
+		} else {
+			queryStr += "item_id <= ? "
+		}
+		args = append(args, fromItem)
+	}
+	// pagination
+	queryStr += "ORDER BY item_id "
+	if order == OrderAsc {
+		queryStr += "ASC "
+	} else {
+		queryStr += "DESC "
+	}
+	queryStr += fmt.Sprintf("LIMIT %d;", *limit)
+	query, argsQ, err := sqlx.In(queryStr, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	query = hdb.db.Rebind(query)
+	tokens := []*TokenRead{}
+	if err := meddler.QueryAll(hdb.db, &tokens, query, argsQ...); err != nil {
+		return nil, nil, err
+	}
+	if len(tokens) == 0 {
+		return nil, nil, sql.ErrNoRows
+	}
+	return db.SlicePtrsToSlice(tokens).([]TokenRead), &db.Pagination{
+		TotalItems: tokens[0].TotalItems,
+		FirstItem:  tokens[0].FirstItem,
+		LastItem:   tokens[0].LastItem,
+	}, nil
 }
 
 // GetTokenSymbols returns all the token symbols from the DB

db/historydb/historydb_test.go

@@ -145,8 +145,9 @@ func TestTokens(t *testing.T) {
 	tokens := test.GenTokens(nTokens, blocks)
 	err := historyDB.AddTokens(tokens)
 	assert.NoError(t, err)
-	// Fetch tokens
-	fetchedTokens, err := historyDB.GetTokens()
+	limit := uint(10)
+	// Fetch tokens6
+	fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, OrderAsc)
 	assert.NoError(t, err)
 	// Compare fetched tokens vs generated tokens
 	// All the tokens should have USDUpdate setted by the DB trigger

db/historydb/views.go

@@ -81,14 +81,18 @@ type txWrite struct {
 
 // TokenRead add USD info to common.Token
 type TokenRead struct {
-	TokenID     common.TokenID    `json:"id" meddler:"token_id"`
-	EthBlockNum int64             `json:"ethereumBlockNum" meddler:"eth_block_num"` // Ethereum block number in which this token was registered
-	EthAddr     ethCommon.Address `json:"ethereumAddress" meddler:"eth_addr"`
-	Name        string            `json:"name" meddler:"name"`
-	Symbol      string            `json:"symbol" meddler:"symbol"`
-	Decimals    uint64            `json:"decimals" meddler:"decimals"`
-	USD         *float64          `json:"USD" meddler:"usd"`
-	USDUpdate   *time.Time        `json:"fiatUpdate" meddler:"usd_update,utctime"`
+	ItemID      int               `meddler:"item_id"`
+	TokenID     common.TokenID    `meddler:"token_id"`
+	EthBlockNum int64             `meddler:"eth_block_num"` // Ethereum block number in which this token was registered
+	EthAddr     ethCommon.Address `meddler:"eth_addr"`
+	Name        string            `meddler:"name"`
+	Symbol      string            `meddler:"symbol"`
+	Decimals    uint64            `meddler:"decimals"`
+	USD         *float64          `meddler:"usd"`
+	USDUpdate   *time.Time        `meddler:"usd_update,utctime"`
+	TotalItems  int               `meddler:"total_items"`
+	FirstItem   int               `meddler:"first_item"`
+	LastItem    int               `meddler:"last_item"`
 }
 
 // HistoryExit is a representation of a exit with additional information

db/migrations/0001.sql

@@ -37,7 +37,8 @@ CREATE TABLE bid (
 );
 
 CREATE TABLE token (
-    token_id INT PRIMARY KEY,
+    item_id SERIAL PRIMARY KEY,
+    token_id INT UNIQUE NOT NULL,
     eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
     eth_addr BYTEA UNIQUE NOT NULL,
     name VARCHAR(20) NOT NULL,

priceupdater/priceupdater_test.go

@@ -51,7 +51,8 @@ func TestPriceUpdater(t *testing.T) {
 	// Update prices
 	pu.UpdatePrices()
 	// Check that prices have been updated
-	fetchedTokens, err := historyDB.GetTokens()
+	limit := uint(10)
+	fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, historydb.OrderAsc)
 	assert.NoError(t, err)
 	for _, token := range fetchedTokens {
 		assert.NotNil(t, token.USD)

test/historydb.go

@@ -35,7 +35,7 @@ func GenTokens(nTokens int, blocks []common.Block) []common.Token {
 	for i := 0; i < nTokens; i++ {
 		token := common.Token{
 			TokenID:     common.TokenID(i),
-			Name:        fmt.Sprint(i),
+			Name:        "NAME" + fmt.Sprint(i),
 			Symbol:      fmt.Sprint(i),
 			Decimals:    uint64(i),
 			EthBlockNum: blocks[i%len(blocks)].EthBlockNum,