Merge pull request #620 from hermeznetwork/feature/priceupdater-by-SC-addr

Add coingecko client to price updater

cli/node/cfg.buidler.toml

@@ -10,6 +10,8 @@ SQLConnectionTimeout = "2s"
 Interval = "10s"
 URL = "https://api-pub.bitfinex.com/v2/"
 Type = "bitfinexV2"
+# URL = "https://api.coingecko.com/api/v3/"
+# Type = "coingeckoV3"
 
 [Debug]
 APIAddress = "localhost:12345"

db/historydb/historydb.go

@@ -486,23 +486,14 @@ func (hdb *HistoryDB) GetAllTokens() ([]TokenWithUSD, error) {
 	return db.SlicePtrsToSlice(tokens).([]TokenWithUSD), tracerr.Wrap(err)
 }
 
-// GetTokenSymbols returns all the token symbols from the DB
+// GetTokenSymbolsAndAddrs returns all the token symbols and addresses from the DB
-func (hdb *HistoryDB) GetTokenSymbols() ([]string, error) {
+func (hdb *HistoryDB) GetTokenSymbolsAndAddrs() ([]TokenSymbolAndAddr, error) {
-	var tokenSymbols []string
+	var tokens []*TokenSymbolAndAddr
-	rows, err := hdb.dbRead.Query("SELECT symbol FROM token;")
+	err := meddler.QueryAll(
-	if err != nil {
+		hdb.dbRead, &tokens,
-		return nil, tracerr.Wrap(err)
+		"SELECT symbol, eth_addr FROM token;",
-	}
+	)
-	defer db.RowsClose(rows)
+	return db.SlicePtrsToSlice(tokens).([]TokenSymbolAndAddr), tracerr.Wrap(err)
-	sym := new(string)
-	for rows.Next() {
-		err = rows.Scan(sym)
-		if err != nil {
-			return nil, tracerr.Wrap(err)
-		}
-		tokenSymbols = append(tokenSymbols, *sym)
-	}
-	return tokenSymbols, nil
 }
 
 // AddAccounts insert accounts into the DB

db/historydb/views.go

@@ -147,6 +147,12 @@ type txWrite struct {
 	Nonce *common.Nonce       `meddler:"nonce"`
 }
 
+// TokenSymbolAndAddr token representation with only Eth addr and symbol
+type TokenSymbolAndAddr struct {
+	Symbol string            `meddler:"symbol"`
+	Addr   ethCommon.Address `meddler:"eth_addr"`
+}
+
 // TokenWithUSD add USD info to common.Token
 type TokenWithUSD struct {
 	ItemID      uint64            `json:"itemId" meddler:"item_id"`

go.sum

@@ -24,8 +24,6 @@ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/uf
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
-github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
@@ -87,8 +85,6 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
-github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
-github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM=
@@ -174,8 +170,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c=
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
-github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
-github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
@@ -608,8 +602,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
-github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -628,8 +620,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg=
 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
-github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
-github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM=

priceupdater/priceupdater.go

@@ -4,9 +4,12 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/dghubble/sling"
+	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db/historydb"
 	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/tracerr"
@@ -23,12 +26,16 @@ type APIType string
 const (
 	// APITypeBitFinexV2 is the http API used by bitfinex V2
 	APITypeBitFinexV2 APIType = "bitfinexV2"
+	// APITypeCoingeckoV3 is the http API used by copingecko V3
+	APITypeCoingeckoV3 APIType = "coingeckoV3"
 )
 
 func (t *APIType) valid() bool {
 	switch *t {
 	case APITypeBitFinexV2:
 		return true
+	case APITypeCoingeckoV3:
+		return true
 	default:
 		return false
 	}
@@ -36,24 +43,23 @@ func (t *APIType) valid() bool {
 
 // PriceUpdater definition
 type PriceUpdater struct {
-	db           *historydb.HistoryDB
+	db      *historydb.HistoryDB
-	apiURL       string
+	apiURL  string
-	apiType      APIType
+	apiType APIType
-	tokenSymbols []string
+	tokens  []historydb.TokenSymbolAndAddr
 }
 
 // NewPriceUpdater is the constructor for the updater
 func NewPriceUpdater(apiURL string, apiType APIType, db *historydb.HistoryDB) (*PriceUpdater,
 	error) {
-	tokenSymbols := []string{}
 	if !apiType.valid() {
 		return nil, tracerr.Wrap(fmt.Errorf("Invalid apiType: %v", apiType))
 	}
 	return &PriceUpdater{
-		db:           db,
+		db:      db,
-		apiURL:       apiURL,
+		apiURL:  apiURL,
-		apiType:      apiType,
+		apiType: apiType,
-		tokenSymbols: tokenSymbols,
+		tokens:  []historydb.TokenSymbolAndAddr{},
 	}, nil
 }
 
@@ -74,6 +80,37 @@ func getTokenPriceBitfinex(ctx context.Context, client *sling.Sling,
 	return state[6], nil
 }
 
+func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling,
+	tokenAddr ethCommon.Address) (float64, error) {
+	responseObject := make(map[string]map[string]float64)
+	var url string
+	var id string
+	if tokenAddr == common.EmptyAddr { // Special case for Ether
+		url = "simple/price?ids=ethereum&vs_currencies=usd"
+		id = "ethereum"
+	} else { // Common case (ERC20)
+		id = strings.ToLower(tokenAddr.String())
+		url = "simple/token_price/ethereum?contract_addresses=" +
+			id + "&vs_currencies=usd"
+	}
+	req, err := client.New().Get(url).Request()
+	if err != nil {
+		return 0, tracerr.Wrap(err)
+	}
+	res, err := client.Do(req.WithContext(ctx), &responseObject, nil)
+	if err != nil {
+		return 0, tracerr.Wrap(err)
+	}
+	if res.StatusCode != http.StatusOK {
+		return 0, tracerr.Wrap(fmt.Errorf("http response is not is %v", res.StatusCode))
+	}
+	price := responseObject[id]["usd"]
+	if price <= 0 {
+		return 0, tracerr.Wrap(fmt.Errorf("price not found for %v", id))
+	}
+	return price, nil
+}
+
 // UpdatePrices is triggered by the Coordinator, and internally will update the
 // token prices in the db
 func (p *PriceUpdater) UpdatePrices(ctx context.Context) {
@@ -85,33 +122,35 @@ func (p *PriceUpdater) UpdatePrices(ctx context.Context) {
 	httpClient := &http.Client{Transport: tr}
 	client := sling.New().Base(p.apiURL).Client(httpClient)
 
-	for _, tokenSymbol := range p.tokenSymbols {
+	for _, token := range p.tokens {
 		var tokenPrice float64
 		var err error
 		switch p.apiType {
 		case APITypeBitFinexV2:
-			tokenPrice, err = getTokenPriceBitfinex(ctx, client, tokenSymbol)
+			tokenPrice, err = getTokenPriceBitfinex(ctx, client, token.Symbol)
+		case APITypeCoingeckoV3:
+			tokenPrice, err = getTokenPriceCoingecko(ctx, client, token.Addr)
 		}
 		if ctx.Err() != nil {
 			return
 		}
 		if err != nil {
 			log.Warnw("token price not updated (get error)",
-				"err", err, "token", tokenSymbol, "apiType", p.apiType)
+				"err", err, "token", token.Symbol, "apiType", p.apiType)
 		}
-		if err = p.db.UpdateTokenValue(tokenSymbol, tokenPrice); err != nil {
+		if err = p.db.UpdateTokenValue(token.Symbol, tokenPrice); err != nil {
 			log.Errorw("token price not updated (db error)",
-				"err", err, "token", tokenSymbol, "apiType", p.apiType)
+				"err", err, "token", token.Symbol, "apiType", p.apiType)
 		}
 	}
 }
 
 // UpdateTokenList get the registered token symbols from HistoryDB
 func (p *PriceUpdater) UpdateTokenList() error {
-	tokenSymbols, err := p.db.GetTokenSymbols()
+	tokens, err := p.db.GetTokenSymbolsAndAddrs()
 	if err != nil {
 		return tracerr.Wrap(err)
 	}
-	p.tokenSymbols = tokenSymbols
+	p.tokens = tokens
 	return nil
 }

priceupdater/priceupdater_test.go

@@ -2,7 +2,6 @@ package priceupdater
 
 import (
 	"context"
-	"math/big"
 	"os"
 	"testing"
 
@@ -15,29 +14,45 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestPriceUpdater(t *testing.T) {
+var historyDB *historydb.HistoryDB
+
+func TestMain(m *testing.M) {
 	// Init DB
 	pass := os.Getenv("POSTGRES_PASS")
 	db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
-	assert.NoError(t, err)
+	if err != nil {
-	historyDB := historydb.NewHistoryDB(db, db, nil)
+		panic(err)
+	}
+	historyDB = historydb.NewHistoryDB(db, db, nil)
 	// Clean DB
 	test.WipeDB(historyDB.DB())
 	// Populate DB
 	// Gen blocks and add them to DB
 	blocks := test.GenBlocks(1, 2)
-	assert.NoError(t, historyDB.AddBlocks(blocks))
+	err = historyDB.AddBlocks(blocks)
+	if err != nil {
+		panic(err)
+	}
 	// Gen tokens and add them to DB
 	tokens := []common.Token{}
 	tokens = append(tokens, common.Token{
 		TokenID:     1,
 		EthBlockNum: blocks[0].Num,
-		EthAddr:     ethCommon.BigToAddress(big.NewInt(2)),
+		EthAddr:     ethCommon.HexToAddress("0x6b175474e89094c44da98b954eedeac495271d0f"),
 		Name:        "DAI",
 		Symbol:      "DAI",
 		Decimals:    18,
 	})
-	assert.NoError(t, historyDB.AddTokens(tokens))
+	err = historyDB.AddTokens(tokens)
+	if err != nil {
+		panic(err)
+	}
+
+	result := m.Run()
+	os.Exit(result)
+}
+
+func TestPriceUpdaterBitfinex(t *testing.T) {
 	// Init price updater
 	pu, err := NewPriceUpdater("https://api-pub.bitfinex.com/v2/", APITypeBitFinexV2, historyDB)
 	require.NoError(t, err)
@@ -45,13 +60,29 @@ func TestPriceUpdater(t *testing.T) {
 	assert.NoError(t, pu.UpdateTokenList())
 	// Update prices
 	pu.UpdatePrices(context.Background())
+	assertTokenHasPriceAndClean(t)
+}
+
+func TestPriceUpdaterCoingecko(t *testing.T) {
+	// Init price updater
+	pu, err := NewPriceUpdater("https://api.coingecko.com/api/v3/", APITypeCoingeckoV3, historyDB)
+	require.NoError(t, err)
+	// Update token list
+	assert.NoError(t, pu.UpdateTokenList())
+	// Update prices
+	pu.UpdatePrices(context.Background())
+	assertTokenHasPriceAndClean(t)
+}
+
+func assertTokenHasPriceAndClean(t *testing.T) {
 	// Check that prices have been updated
 	fetchedTokens, err := historyDB.GetTokensTest()
 	require.NoError(t, err)
 	// TokenID 0 (ETH) is always on the DB
 	assert.Equal(t, 2, len(fetchedTokens))
 	for _, token := range fetchedTokens {
-		assert.NotNil(t, token.USD)
+		require.NotNil(t, token.USD)
-		assert.NotNil(t, token.USDUpdate)
+		require.NotNil(t, token.USDUpdate)
+		assert.Greater(t, *token.USD, 0.0)
 	}
 }