From ac66ede91734a5937915503a283f2145e4383e36 Mon Sep 17 00:00:00 2001 From: arnaubennassar Date: Mon, 8 Mar 2021 15:06:42 +0100 Subject: [PATCH] Add coingecko client to price updater --- cli/node/cfg.buidler.toml | 2 + db/historydb/historydb.go | 25 ++++------- db/historydb/views.go | 6 +++ go.sum | 10 ----- priceupdater/priceupdater.go | 71 ++++++++++++++++++++++++------- priceupdater/priceupdater_test.go | 49 +++++++++++++++++---- 6 files changed, 111 insertions(+), 52 deletions(-) diff --git a/cli/node/cfg.buidler.toml b/cli/node/cfg.buidler.toml index 5f524f3..f36dffb 100644 --- a/cli/node/cfg.buidler.toml +++ b/cli/node/cfg.buidler.toml @@ -10,6 +10,8 @@ SQLConnectionTimeout = "2s" Interval = "10s" URL = "https://api-pub.bitfinex.com/v2/" Type = "bitfinexV2" +# URL = "https://api.coingecko.com/api/v3/" +# Type = "coingeckoV3" [Debug] APIAddress = "localhost:12345" diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index cfb1b18..ea3338e 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -486,23 +486,14 @@ func (hdb *HistoryDB) GetAllTokens() ([]TokenWithUSD, error) { return db.SlicePtrsToSlice(tokens).([]TokenWithUSD), tracerr.Wrap(err) } -// GetTokenSymbols returns all the token symbols from the DB -func (hdb *HistoryDB) GetTokenSymbols() ([]string, error) { - var tokenSymbols []string - rows, err := hdb.dbRead.Query("SELECT symbol FROM token;") - if err != nil { - return nil, tracerr.Wrap(err) - } - defer db.RowsClose(rows) - sym := new(string) - for rows.Next() { - err = rows.Scan(sym) - if err != nil { - return nil, tracerr.Wrap(err) - } - tokenSymbols = append(tokenSymbols, *sym) - } - return tokenSymbols, nil +// GetTokenSymbolsAndAddrs returns all the token symbols and addresses from the DB +func (hdb *HistoryDB) GetTokenSymbolsAndAddrs() ([]TokenSymbolAndAddr, error) { + var tokens []*TokenSymbolAndAddr + err := meddler.QueryAll( + hdb.dbRead, &tokens, + "SELECT symbol, eth_addr FROM token;", + ) + return db.SlicePtrsToSlice(tokens).([]TokenSymbolAndAddr), tracerr.Wrap(err) } // AddAccounts insert accounts into the DB diff --git a/db/historydb/views.go b/db/historydb/views.go index 32d153e..0e396ff 100644 --- a/db/historydb/views.go +++ b/db/historydb/views.go @@ -147,6 +147,12 @@ type txWrite struct { Nonce *common.Nonce `meddler:"nonce"` } +// TokenSymbolAndAddr token representation with only Eth addr and symbol +type TokenSymbolAndAddr struct { + Symbol string `meddler:"symbol"` + Addr ethCommon.Address `meddler:"eth_addr"` +} + // TokenWithUSD add USD info to common.Token type TokenWithUSD struct { ItemID uint64 `json:"itemId" meddler:"item_id"` diff --git a/go.sum b/go.sum index 3def156..516b033 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,6 @@ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/uf github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= -github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -87,8 +85,6 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= -github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= @@ -174,8 +170,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= @@ -608,8 +602,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= -github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -628,8 +620,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw= -github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= diff --git a/priceupdater/priceupdater.go b/priceupdater/priceupdater.go index f7072a7..4b1894a 100644 --- a/priceupdater/priceupdater.go +++ b/priceupdater/priceupdater.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "net/http" + "strings" "time" "github.com/dghubble/sling" + ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/common" "github.com/hermeznetwork/hermez-node/db/historydb" "github.com/hermeznetwork/hermez-node/log" "github.com/hermeznetwork/tracerr" @@ -23,12 +26,16 @@ type APIType string const ( // APITypeBitFinexV2 is the http API used by bitfinex V2 APITypeBitFinexV2 APIType = "bitfinexV2" + // APITypeCoingeckoV3 is the http API used by copingecko V3 + APITypeCoingeckoV3 APIType = "coingeckoV3" ) func (t *APIType) valid() bool { switch *t { case APITypeBitFinexV2: return true + case APITypeCoingeckoV3: + return true default: return false } @@ -36,24 +43,23 @@ func (t *APIType) valid() bool { // PriceUpdater definition type PriceUpdater struct { - db *historydb.HistoryDB - apiURL string - apiType APIType - tokenSymbols []string + db *historydb.HistoryDB + apiURL string + apiType APIType + tokens []historydb.TokenSymbolAndAddr } // NewPriceUpdater is the constructor for the updater func NewPriceUpdater(apiURL string, apiType APIType, db *historydb.HistoryDB) (*PriceUpdater, error) { - tokenSymbols := []string{} if !apiType.valid() { return nil, tracerr.Wrap(fmt.Errorf("Invalid apiType: %v", apiType)) } return &PriceUpdater{ - db: db, - apiURL: apiURL, - apiType: apiType, - tokenSymbols: tokenSymbols, + db: db, + apiURL: apiURL, + apiType: apiType, + tokens: []historydb.TokenSymbolAndAddr{}, }, nil } @@ -74,6 +80,37 @@ func getTokenPriceBitfinex(ctx context.Context, client *sling.Sling, return state[6], nil } +func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling, + tokenAddr ethCommon.Address) (float64, error) { + responseObject := make(map[string]map[string]float64) + var url string + var id string + if tokenAddr == common.EmptyAddr { // Special case for Ether + url = "simple/price?ids=ethereum&vs_currencies=usd" + id = "ethereum" + } else { // Common case (ERC20) + id = strings.ToLower(tokenAddr.String()) + url = "simple/token_price/ethereum?contract_addresses=" + + id + "&vs_currencies=usd" + } + req, err := client.New().Get(url).Request() + if err != nil { + return 0, tracerr.Wrap(err) + } + res, err := client.Do(req.WithContext(ctx), &responseObject, nil) + if err != nil { + return 0, tracerr.Wrap(err) + } + if res.StatusCode != http.StatusOK { + return 0, tracerr.Wrap(fmt.Errorf("http response is not is %v", res.StatusCode)) + } + price := responseObject[id]["usd"] + if price <= 0 { + return 0, tracerr.Wrap(fmt.Errorf("price not found for %v", id)) + } + return price, nil +} + // UpdatePrices is triggered by the Coordinator, and internally will update the // token prices in the db func (p *PriceUpdater) UpdatePrices(ctx context.Context) { @@ -85,33 +122,35 @@ func (p *PriceUpdater) UpdatePrices(ctx context.Context) { httpClient := &http.Client{Transport: tr} client := sling.New().Base(p.apiURL).Client(httpClient) - for _, tokenSymbol := range p.tokenSymbols { + for _, token := range p.tokens { var tokenPrice float64 var err error switch p.apiType { case APITypeBitFinexV2: - tokenPrice, err = getTokenPriceBitfinex(ctx, client, tokenSymbol) + tokenPrice, err = getTokenPriceBitfinex(ctx, client, token.Symbol) + case APITypeCoingeckoV3: + tokenPrice, err = getTokenPriceCoingecko(ctx, client, token.Addr) } if ctx.Err() != nil { return } if err != nil { log.Warnw("token price not updated (get error)", - "err", err, "token", tokenSymbol, "apiType", p.apiType) + "err", err, "token", token.Symbol, "apiType", p.apiType) } - if err = p.db.UpdateTokenValue(tokenSymbol, tokenPrice); err != nil { + if err = p.db.UpdateTokenValue(token.Symbol, tokenPrice); err != nil { log.Errorw("token price not updated (db error)", - "err", err, "token", tokenSymbol, "apiType", p.apiType) + "err", err, "token", token.Symbol, "apiType", p.apiType) } } } // UpdateTokenList get the registered token symbols from HistoryDB func (p *PriceUpdater) UpdateTokenList() error { - tokenSymbols, err := p.db.GetTokenSymbols() + tokens, err := p.db.GetTokenSymbolsAndAddrs() if err != nil { return tracerr.Wrap(err) } - p.tokenSymbols = tokenSymbols + p.tokens = tokens return nil } diff --git a/priceupdater/priceupdater_test.go b/priceupdater/priceupdater_test.go index 4945fde..c93db24 100644 --- a/priceupdater/priceupdater_test.go +++ b/priceupdater/priceupdater_test.go @@ -2,7 +2,6 @@ package priceupdater import ( "context" - "math/big" "os" "testing" @@ -15,29 +14,45 @@ import ( "github.com/stretchr/testify/require" ) -func TestPriceUpdater(t *testing.T) { +var historyDB *historydb.HistoryDB + +func TestMain(m *testing.M) { // Init DB pass := os.Getenv("POSTGRES_PASS") db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez") - assert.NoError(t, err) - historyDB := historydb.NewHistoryDB(db, db, nil) + if err != nil { + panic(err) + } + historyDB = historydb.NewHistoryDB(db, db, nil) // Clean DB test.WipeDB(historyDB.DB()) // Populate DB // Gen blocks and add them to DB blocks := test.GenBlocks(1, 2) - assert.NoError(t, historyDB.AddBlocks(blocks)) + err = historyDB.AddBlocks(blocks) + if err != nil { + panic(err) + } // Gen tokens and add them to DB tokens := []common.Token{} tokens = append(tokens, common.Token{ TokenID: 1, EthBlockNum: blocks[0].Num, - EthAddr: ethCommon.BigToAddress(big.NewInt(2)), + EthAddr: ethCommon.HexToAddress("0x6b175474e89094c44da98b954eedeac495271d0f"), Name: "DAI", Symbol: "DAI", Decimals: 18, }) - assert.NoError(t, historyDB.AddTokens(tokens)) + err = historyDB.AddTokens(tokens) + if err != nil { + panic(err) + } + + result := m.Run() + os.Exit(result) +} + +func TestPriceUpdaterBitfinex(t *testing.T) { // Init price updater pu, err := NewPriceUpdater("https://api-pub.bitfinex.com/v2/", APITypeBitFinexV2, historyDB) require.NoError(t, err) @@ -45,13 +60,29 @@ func TestPriceUpdater(t *testing.T) { assert.NoError(t, pu.UpdateTokenList()) // Update prices pu.UpdatePrices(context.Background()) + assertTokenHasPriceAndClean(t) +} + +func TestPriceUpdaterCoingecko(t *testing.T) { + // Init price updater + pu, err := NewPriceUpdater("https://api.coingecko.com/api/v3/", APITypeCoingeckoV3, historyDB) + require.NoError(t, err) + // Update token list + assert.NoError(t, pu.UpdateTokenList()) + // Update prices + pu.UpdatePrices(context.Background()) + assertTokenHasPriceAndClean(t) +} + +func assertTokenHasPriceAndClean(t *testing.T) { // Check that prices have been updated fetchedTokens, err := historyDB.GetTokensTest() require.NoError(t, err) // TokenID 0 (ETH) is always on the DB assert.Equal(t, 2, len(fetchedTokens)) for _, token := range fetchedTokens { - assert.NotNil(t, token.USD) - assert.NotNil(t, token.USDUpdate) + require.NotNil(t, token.USD) + require.NotNil(t, token.USDUpdate) + assert.Greater(t, *token.USD, 0.0) } }