diff --git a/api/api_test.go b/api/api_test.go index afe8d94..87197be 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -306,7 +306,7 @@ func TestMain(m *testing.M) { USD: ðUSD, USDUpdate: ðNow, }) - err = api.h.UpdateTokenValue(test.EthToken.Symbol, ethUSD) + err = api.h.UpdateTokenValue(common.EmptyAddr, ethUSD) if err != nil { panic(err) } @@ -333,7 +333,7 @@ func TestMain(m *testing.M) { token.USD = &value token.USDUpdate = &now // Set value in DB - err = api.h.UpdateTokenValue(token.Symbol, value) + err = api.h.UpdateTokenValue(token.EthAddr, value) if err != nil { panic(err) } diff --git a/cli/node/cfg.buidler.toml b/cli/node/cfg.buidler.toml index f36dffb..452a22d 100644 --- a/cli/node/cfg.buidler.toml +++ b/cli/node/cfg.buidler.toml @@ -8,10 +8,31 @@ SQLConnectionTimeout = "2s" [PriceUpdater] Interval = "10s" -URL = "https://api-pub.bitfinex.com/v2/" -Type = "bitfinexV2" -# URL = "https://api.coingecko.com/api/v3/" -# Type = "coingeckoV3" +URLBitfinexV2 = "https://api-pub.bitfinex.com/v2/" +URLCoinGeckoV3 = "https://api.coingecko.com/api/v3/" +# Available update methods: +# - coingeckoV3 (recommended): get price by SC addr using coingecko API +# - bitfinexV2: get price by token symbol using bitfinex API +# - static (recommended for blacklisting tokens): use the given StaticValue to set the price (if not provided 0 will be used) +# - ignore: don't update the price leave it as it is on the DB +DefaultUpdateMethod = "coingeckoV3" # Update method used for all the tokens registered on the network, and not listed in [[PriceUpdater.TokensConfig]] +[[PriceUpdater.TokensConfig]] +UpdateMethod = "bitfinexV2" +Symbol = "USDT" +Addr = "0xdac17f958d2ee523a2206206994597c13d831ec7" +[[PriceUpdater.TokensConfig]] +UpdateMethod = "coingeckoV3" +Symbol = "ETH" +Addr = "0x0000000000000000000000000000000000000000" +[[PriceUpdater.TokensConfig]] +UpdateMethod = "static" +Symbol = "UNI" +Addr = "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984" +StaticValue = 30.12 +[[PriceUpdater.TokensConfig]] +UpdateMethod = "ignore" +Symbol = "SUSHI" +Addr = "0x6b3595068778dd592e39a122f4f5a5cf09c90fe2" [Debug] APIAddress = "localhost:12345" diff --git a/config/config.go b/config/config.go index f4f108e..0961c9c 100644 --- a/config/config.go +++ b/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/BurntSushi/toml" ethCommon "github.com/ethereum/go-ethereum/common" "github.com/hermeznetwork/hermez-node/common" + "github.com/hermeznetwork/hermez-node/priceupdater" "github.com/hermeznetwork/tracerr" "github.com/iden3/go-iden3-crypto/babyjub" "gopkg.in/go-playground/validator.v9" @@ -219,11 +220,15 @@ type Coordinator struct { type Node struct { PriceUpdater struct { // Interval between price updater calls - Interval Duration `valudate:"required"` - // URL of the token prices provider - URL string `valudate:"required"` - // Type of the API of the token prices provider - Type string `valudate:"required"` + Interval Duration `validate:"required"` + // URLBitfinexV2 is the URL of bitfinex V2 API + URLBitfinexV2 string `validate:"required"` + // URLCoinGeckoV3 is the URL of coingecko V3 API + URLCoinGeckoV3 string `validate:"required"` + // DefaultUpdateMethod to get token prices + DefaultUpdateMethod priceupdater.UpdateMethodType `validate:"required"` + // TokensConfig to specify how each token get it's price updated + TokensConfig []priceupdater.TokenConfig } `validate:"required"` StateDB struct { // Path where the synchronizer StateDB is stored diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index ea3338e..6d99b52 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -456,13 +456,10 @@ func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error { // UpdateTokenValue updates the USD value of a token. Value is the price in // USD of a normalized token (1 token = 10^decimals units) -func (hdb *HistoryDB) UpdateTokenValue(tokenSymbol string, value float64) error { - // Sanitize symbol - tokenSymbol = strings.ToValidUTF8(tokenSymbol, " ") - +func (hdb *HistoryDB) UpdateTokenValue(tokenAddr ethCommon.Address, value float64) error { _, err := hdb.dbWrite.Exec( - "UPDATE token SET usd = $1 WHERE symbol = $2;", - value, tokenSymbol, + "UPDATE token SET usd = $1 WHERE eth_addr = $2;", + value, tokenAddr, ) return tracerr.Wrap(err) } @@ -1161,7 +1158,7 @@ func (hdb *HistoryDB) GetTokensTest() ([]TokenWithUSD, error) { tokens := []*TokenWithUSD{} if err := meddler.QueryAll( hdb.dbRead, &tokens, - "SELECT * FROM TOKEN", + "SELECT * FROM token ORDER BY token_id ASC", ); err != nil { return nil, tracerr.Wrap(err) } diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index f860016..d391f0b 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -166,7 +166,7 @@ func TestBatches(t *testing.T) { if i%2 != 0 { // Set value to the token value := (float64(i) + 5) * 5.389329 - assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value)) + assert.NoError(t, historyDB.UpdateTokenValue(token.EthAddr, value)) tokensValue[token.TokenID] = value / math.Pow(10, float64(token.Decimals)) } } @@ -276,7 +276,7 @@ func TestTokens(t *testing.T) { // Update token value for i, token := range tokens { value := 1.01 * float64(i) - assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value)) + assert.NoError(t, historyDB.UpdateTokenValue(token.EthAddr, value)) } // Fetch tokens fetchedTokens, err = historyDB.GetTokensTest() @@ -302,7 +302,7 @@ func TestTokensUTF8(t *testing.T) { // Generate fake tokens const nTokens = 5 tokens, ethToken := test.GenTokens(nTokens, blocks) - nonUTFTokens := make([]common.Token, len(tokens)+1) + nonUTFTokens := make([]common.Token, len(tokens)) // Force token.name and token.symbol to be non UTF-8 Strings for i, token := range tokens { token.Name = fmt.Sprint("NON-UTF8-NAME-\xc5-", i) @@ -332,7 +332,7 @@ func TestTokensUTF8(t *testing.T) { // Update token value for i, token := range nonUTFTokens { value := 1.01 * float64(i) - assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value)) + assert.NoError(t, historyDB.UpdateTokenValue(token.EthAddr, value)) } // Fetch tokens fetchedTokens, err = historyDB.GetTokensTest() diff --git a/db/l2db/l2db_test.go b/db/l2db/l2db_test.go index 1d814fa..52f206c 100644 --- a/db/l2db/l2db_test.go +++ b/db/l2db/l2db_test.go @@ -121,7 +121,7 @@ func prepareHistoryDB(historyDB *historydb.HistoryDB) error { } tokens[token.TokenID] = readToken // Set value to the tokens - err := historyDB.UpdateTokenValue(readToken.Symbol, *readToken.USD) + err := historyDB.UpdateTokenValue(readToken.EthAddr, *readToken.USD) if err != nil { return tracerr.Wrap(err) } diff --git a/node/node.go b/node/node.go index a6db6d2..4b696e0 100644 --- a/node/node.go +++ b/node/node.go @@ -425,8 +425,13 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) { if cfg.Debug.APIAddress != "" { debugAPI = debugapi.NewDebugAPI(cfg.Debug.APIAddress, stateDB, sync) } - priceUpdater, err := priceupdater.NewPriceUpdater(cfg.PriceUpdater.URL, - priceupdater.APIType(cfg.PriceUpdater.Type), historyDB) + priceUpdater, err := priceupdater.NewPriceUpdater( + cfg.PriceUpdater.DefaultUpdateMethod, + cfg.PriceUpdater.TokensConfig, + historyDB, + cfg.PriceUpdater.URLBitfinexV2, + cfg.PriceUpdater.URLCoinGeckoV3, + ) if err != nil { return nil, tracerr.Wrap(err) } diff --git a/priceupdater/priceupdater.go b/priceupdater/priceupdater.go index 4b1894a..7c56bf3 100644 --- a/priceupdater/priceupdater.go +++ b/priceupdater/priceupdater.go @@ -20,57 +20,107 @@ const ( defaultIdleConnTimeout = 2 * time.Second ) -// APIType defines the token exchange API -type APIType string +// UpdateMethodType defines the token price update mechanism +type UpdateMethodType string const ( - // APITypeBitFinexV2 is the http API used by bitfinex V2 - APITypeBitFinexV2 APIType = "bitfinexV2" - // APITypeCoingeckoV3 is the http API used by copingecko V3 - APITypeCoingeckoV3 APIType = "coingeckoV3" + // UpdateMethodTypeBitFinexV2 is the http API used by bitfinex V2 + UpdateMethodTypeBitFinexV2 UpdateMethodType = "bitfinexV2" + // UpdateMethodTypeCoingeckoV3 is the http API used by copingecko V3 + UpdateMethodTypeCoingeckoV3 UpdateMethodType = "coingeckoV3" + // UpdateMethodTypeStatic is the value given by the configuration + UpdateMethodTypeStatic UpdateMethodType = "static" + // UpdateMethodTypeIgnore indicates to not update the value, to set value 0 + // it's better to use UpdateMethodTypeStatic + UpdateMethodTypeIgnore UpdateMethodType = "ignore" ) -func (t *APIType) valid() bool { +func (t *UpdateMethodType) valid() bool { switch *t { - case APITypeBitFinexV2: + case UpdateMethodTypeBitFinexV2: return true - case APITypeCoingeckoV3: + case UpdateMethodTypeCoingeckoV3: + return true + case UpdateMethodTypeStatic: + return true + case UpdateMethodTypeIgnore: return true default: return false } } +// TokenConfig specifies how a single token get its price updated +type TokenConfig struct { + UpdateMethod UpdateMethodType + StaticValue float64 // required by UpdateMethodTypeStatic + Symbol string + Addr ethCommon.Address +} + +func (t *TokenConfig) valid() bool { + if (t.Addr == common.EmptyAddr && t.Symbol != "ETH") || + (t.Symbol == "" && t.UpdateMethod == UpdateMethodTypeBitFinexV2) { + return false + } + return t.UpdateMethod.valid() +} + // PriceUpdater definition type PriceUpdater struct { - db *historydb.HistoryDB - apiURL string - apiType APIType - tokens []historydb.TokenSymbolAndAddr + db *historydb.HistoryDB + defaultUpdateMethod UpdateMethodType + tokensList []historydb.TokenSymbolAndAddr + tokensConfig map[ethCommon.Address]TokenConfig + clientCoingeckoV3 *sling.Sling + clientBitfinexV2 *sling.Sling } // NewPriceUpdater is the constructor for the updater -func NewPriceUpdater(apiURL string, apiType APIType, db *historydb.HistoryDB) (*PriceUpdater, - error) { - if !apiType.valid() { - return nil, tracerr.Wrap(fmt.Errorf("Invalid apiType: %v", apiType)) +func NewPriceUpdater( + defaultUpdateMethodType UpdateMethodType, + tokensConfig []TokenConfig, + db *historydb.HistoryDB, + bitfinexV2URL, coingeckoV3URL string, +) (*PriceUpdater, error) { + // Validate params + if !defaultUpdateMethodType.valid() || defaultUpdateMethodType == UpdateMethodTypeStatic { + return nil, tracerr.Wrap( + fmt.Errorf("Invalid defaultUpdateMethodType: %v", defaultUpdateMethodType), + ) + } + tokensConfigMap := make(map[ethCommon.Address]TokenConfig) + for _, t := range tokensConfig { + if !t.valid() { + return nil, tracerr.Wrap(fmt.Errorf("Invalid tokensConfig, wrong entry: %+v", t)) + } + tokensConfigMap[t.Addr] = t } + // Init + tr := &http.Transport{ + MaxIdleConns: defaultMaxIdleConns, + IdleConnTimeout: defaultIdleConnTimeout, + DisableCompression: true, + } + httpClient := &http.Client{Transport: tr} return &PriceUpdater{ - db: db, - apiURL: apiURL, - apiType: apiType, - tokens: []historydb.TokenSymbolAndAddr{}, + db: db, + defaultUpdateMethod: defaultUpdateMethodType, + tokensList: []historydb.TokenSymbolAndAddr{}, + tokensConfig: tokensConfigMap, + clientCoingeckoV3: sling.New().Base(coingeckoV3URL).Client(httpClient), + clientBitfinexV2: sling.New().Base(bitfinexV2URL).Client(httpClient), }, nil } -func getTokenPriceBitfinex(ctx context.Context, client *sling.Sling, - tokenSymbol string) (float64, error) { +func (p *PriceUpdater) getTokenPriceBitfinex(ctx context.Context, tokenSymbol string) (float64, error) { state := [10]float64{} - req, err := client.New().Get("ticker/t" + tokenSymbol + "USD").Request() + url := "ticker/t" + tokenSymbol + "USD" + req, err := p.clientBitfinexV2.New().Get(url).Request() if err != nil { return 0, tracerr.Wrap(err) } - res, err := client.Do(req.WithContext(ctx), &state, nil) + res, err := p.clientBitfinexV2.Do(req.WithContext(ctx), &state, nil) if err != nil { return 0, tracerr.Wrap(err) } @@ -80,8 +130,7 @@ func getTokenPriceBitfinex(ctx context.Context, client *sling.Sling, return state[6], nil } -func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling, - tokenAddr ethCommon.Address) (float64, error) { +func (p *PriceUpdater) getTokenPriceCoingecko(ctx context.Context, tokenAddr ethCommon.Address) (float64, error) { responseObject := make(map[string]map[string]float64) var url string var id string @@ -93,11 +142,11 @@ func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling, url = "simple/token_price/ethereum?contract_addresses=" + id + "&vs_currencies=usd" } - req, err := client.New().Get(url).Request() + req, err := p.clientCoingeckoV3.New().Get(url).Request() if err != nil { return 0, tracerr.Wrap(err) } - res, err := client.Do(req.WithContext(ctx), &responseObject, nil) + res, err := p.clientCoingeckoV3.Do(req.WithContext(ctx), &responseObject, nil) if err != nil { return 0, tracerr.Wrap(err) } @@ -114,43 +163,50 @@ func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling, // UpdatePrices is triggered by the Coordinator, and internally will update the // token prices in the db func (p *PriceUpdater) UpdatePrices(ctx context.Context) { - tr := &http.Transport{ - MaxIdleConns: defaultMaxIdleConns, - IdleConnTimeout: defaultIdleConnTimeout, - DisableCompression: true, - } - httpClient := &http.Client{Transport: tr} - client := sling.New().Base(p.apiURL).Client(httpClient) - - for _, token := range p.tokens { + for _, token := range p.tokensConfig { var tokenPrice float64 var err error - switch p.apiType { - case APITypeBitFinexV2: - tokenPrice, err = getTokenPriceBitfinex(ctx, client, token.Symbol) - case APITypeCoingeckoV3: - tokenPrice, err = getTokenPriceCoingecko(ctx, client, token.Addr) + switch token.UpdateMethod { + case UpdateMethodTypeBitFinexV2: + tokenPrice, err = p.getTokenPriceBitfinex(ctx, token.Symbol) + case UpdateMethodTypeCoingeckoV3: + tokenPrice, err = p.getTokenPriceCoingecko(ctx, token.Addr) + case UpdateMethodTypeStatic: + tokenPrice = token.StaticValue + case UpdateMethodTypeIgnore: + continue } if ctx.Err() != nil { return } if err != nil { log.Warnw("token price not updated (get error)", - "err", err, "token", token.Symbol, "apiType", p.apiType) + "err", err, "token", token.Symbol, "updateMethod", token.UpdateMethod) } - if err = p.db.UpdateTokenValue(token.Symbol, tokenPrice); err != nil { + if err = p.db.UpdateTokenValue(token.Addr, tokenPrice); err != nil { log.Errorw("token price not updated (db error)", - "err", err, "token", token.Symbol, "apiType", p.apiType) + "err", err, "token", token.Symbol, "updateMethod", token.UpdateMethod) } } } // UpdateTokenList get the registered token symbols from HistoryDB func (p *PriceUpdater) UpdateTokenList() error { - tokens, err := p.db.GetTokenSymbolsAndAddrs() + dbTokens, err := p.db.GetTokenSymbolsAndAddrs() if err != nil { return tracerr.Wrap(err) } - p.tokens = tokens + // For each token from the DB + for _, dbToken := range dbTokens { + // If the token doesn't exists in the config list, + // add it with default update emthod + if _, ok := p.tokensConfig[dbToken.Addr]; !ok { + p.tokensConfig[dbToken.Addr] = TokenConfig{ + UpdateMethod: p.defaultUpdateMethod, + Symbol: dbToken.Symbol, + Addr: dbToken.Addr, + } + } + } return nil } diff --git a/priceupdater/priceupdater_test.go b/priceupdater/priceupdater_test.go index c93db24..76f8d0e 100644 --- a/priceupdater/priceupdater_test.go +++ b/priceupdater/priceupdater_test.go @@ -16,7 +16,9 @@ import ( var historyDB *historydb.HistoryDB -func TestMain(m *testing.M) { +const usdtAddr = "0xdac17f958d2ee523a2206206994597c13d831ec7" + +func TestPriceUpdaterBitfinex(t *testing.T) { // Init DB pass := os.Getenv("POSTGRES_PASS") db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez") @@ -29,60 +31,113 @@ func TestMain(m *testing.M) { // Populate DB // Gen blocks and add them to DB blocks := test.GenBlocks(1, 2) - err = historyDB.AddBlocks(blocks) - if err != nil { - panic(err) - } + require.NoError(t, historyDB.AddBlocks(blocks)) // Gen tokens and add them to DB - tokens := []common.Token{} - tokens = append(tokens, common.Token{ - TokenID: 1, - EthBlockNum: blocks[0].Num, - EthAddr: ethCommon.HexToAddress("0x6b175474e89094c44da98b954eedeac495271d0f"), - Name: "DAI", - Symbol: "DAI", - Decimals: 18, - }) - err = historyDB.AddTokens(tokens) - if err != nil { - panic(err) + tokens := []common.Token{ + { + TokenID: 1, + EthBlockNum: blocks[0].Num, + EthAddr: ethCommon.HexToAddress("0x1"), + Name: "DAI", + Symbol: "DAI", + Decimals: 18, + }, // Used to test get by SC addr + { + TokenID: 2, + EthBlockNum: blocks[0].Num, + EthAddr: ethCommon.HexToAddress(usdtAddr), + Name: "Tether", + Symbol: "USDT", + Decimals: 18, + }, // Used to test get by token symbol + { + TokenID: 3, + EthBlockNum: blocks[0].Num, + EthAddr: ethCommon.HexToAddress("0x2"), + Name: "FOO", + Symbol: "FOO", + Decimals: 18, + }, // Used to test ignore + { + TokenID: 4, + EthBlockNum: blocks[0].Num, + EthAddr: ethCommon.HexToAddress("0x3"), + Name: "BAR", + Symbol: "BAR", + Decimals: 18, + }, // Used to test static + { + TokenID: 5, + EthBlockNum: blocks[0].Num, + EthAddr: ethCommon.HexToAddress("0x1f9840a85d5af5bf1d1762f925bdaddc4201f984"), + Name: "Uniswap", + Symbol: "UNI", + Decimals: 18, + }, // Used to test default } + require.NoError(t, historyDB.AddTokens(tokens)) // ETH token exist in DB by default + // Update token price used to test ignore + ignoreValue := 44.44 + require.NoError(t, historyDB.UpdateTokenValue(tokens[2].EthAddr, ignoreValue)) - result := m.Run() - os.Exit(result) -} - -func TestPriceUpdaterBitfinex(t *testing.T) { - // Init price updater - pu, err := NewPriceUpdater("https://api-pub.bitfinex.com/v2/", APITypeBitFinexV2, historyDB) - require.NoError(t, err) - // Update token list - assert.NoError(t, pu.UpdateTokenList()) - // Update prices - pu.UpdatePrices(context.Background()) - assertTokenHasPriceAndClean(t) -} + // Prepare token config + staticValue := 0.12345 + tc := []TokenConfig{ + // ETH and UNI tokens use default method + { // DAI uses SC addr + UpdateMethod: UpdateMethodTypeBitFinexV2, + Addr: ethCommon.HexToAddress("0x1"), + Symbol: "DAI", + }, + { // USDT uses symbol + UpdateMethod: UpdateMethodTypeCoingeckoV3, + Addr: ethCommon.HexToAddress(usdtAddr), + }, + { // FOO uses ignore + UpdateMethod: UpdateMethodTypeIgnore, + Addr: ethCommon.HexToAddress("0x2"), + }, + { // BAR uses static + UpdateMethod: UpdateMethodTypeStatic, + Addr: ethCommon.HexToAddress("0x3"), + StaticValue: staticValue, + }, + } -func TestPriceUpdaterCoingecko(t *testing.T) { + bitfinexV2URL := "https://api-pub.bitfinex.com/v2/" + coingeckoV3URL := "https://api.coingecko.com/api/v3/" // Init price updater - pu, err := NewPriceUpdater("https://api.coingecko.com/api/v3/", APITypeCoingeckoV3, historyDB) + pu, err := NewPriceUpdater( + UpdateMethodTypeCoingeckoV3, + tc, + historyDB, + bitfinexV2URL, + coingeckoV3URL, + ) require.NoError(t, err) // Update token list - assert.NoError(t, pu.UpdateTokenList()) + require.NoError(t, pu.UpdateTokenList()) // Update prices pu.UpdatePrices(context.Background()) - assertTokenHasPriceAndClean(t) -} -func assertTokenHasPriceAndClean(t *testing.T) { - // Check that prices have been updated + // Check results: get tokens from DB fetchedTokens, err := historyDB.GetTokensTest() require.NoError(t, err) - // TokenID 0 (ETH) is always on the DB - assert.Equal(t, 2, len(fetchedTokens)) - for _, token := range fetchedTokens { - require.NotNil(t, token.USD) - require.NotNil(t, token.USDUpdate) - assert.Greater(t, *token.USD, 0.0) - } + // Check that tokens that are updated via API have value: + // ETH + require.NotNil(t, fetchedTokens[0].USDUpdate) + assert.Greater(t, *fetchedTokens[0].USD, 0.0) + // DAI + require.NotNil(t, fetchedTokens[1].USDUpdate) + assert.Greater(t, *fetchedTokens[1].USD, 0.0) + // USDT + require.NotNil(t, fetchedTokens[2].USDUpdate) + assert.Greater(t, *fetchedTokens[2].USD, 0.0) + // UNI + require.NotNil(t, fetchedTokens[5].USDUpdate) + assert.Greater(t, *fetchedTokens[5].USD, 0.0) + // Check ignored token + assert.Equal(t, ignoreValue, *fetchedTokens[3].USD) + // Check static value + assert.Equal(t, staticValue, *fetchedTokens[4].USD) }