Browse Source

Allow price update configuration to be specified per token

feature/update-smart-contracts
arnaubennassar 3 years ago
parent
commit
97062afc90
9 changed files with 257 additions and 118 deletions
  1. +2
    -2
      api/api_test.go
  2. +25
    -4
      cli/node/cfg.buidler.toml
  3. +10
    -5
      config/config.go
  4. +4
    -7
      db/historydb/historydb.go
  5. +4
    -4
      db/historydb/historydb_test.go
  6. +1
    -1
      db/l2db/l2db_test.go
  7. +7
    -2
      node/node.go
  8. +104
    -48
      priceupdater/priceupdater.go
  9. +100
    -45
      priceupdater/priceupdater_test.go

+ 2
- 2
api/api_test.go

@ -306,7 +306,7 @@ func TestMain(m *testing.M) {
USD: &ethUSD,
USDUpdate: &ethNow,
})
err = api.h.UpdateTokenValue(test.EthToken.Symbol, ethUSD)
err = api.h.UpdateTokenValue(common.EmptyAddr, ethUSD)
if err != nil {
panic(err)
}
@ -333,7 +333,7 @@ func TestMain(m *testing.M) {
token.USD = &value
token.USDUpdate = &now
// Set value in DB
err = api.h.UpdateTokenValue(token.Symbol, value)
err = api.h.UpdateTokenValue(token.EthAddr, value)
if err != nil {
panic(err)
}

+ 25
- 4
cli/node/cfg.buidler.toml

@ -8,10 +8,31 @@ SQLConnectionTimeout = "2s"
[PriceUpdater]
Interval = "10s"
URL = "https://api-pub.bitfinex.com/v2/"
Type = "bitfinexV2"
# URL = "https://api.coingecko.com/api/v3/"
# Type = "coingeckoV3"
URLBitfinexV2 = "https://api-pub.bitfinex.com/v2/"
URLCoinGeckoV3 = "https://api.coingecko.com/api/v3/"
# Available update methods:
# - coingeckoV3 (recommended): get price by SC addr using coingecko API
# - bitfinexV2: get price by token symbol using bitfinex API
# - static (recommended for blacklisting tokens): use the given StaticValue to set the price (if not provided 0 will be used)
# - ignore: don't update the price leave it as it is on the DB
DefaultUpdateMethod = "coingeckoV3" # Update method used for all the tokens registered on the network, and not listed in [[PriceUpdater.TokensConfig]]
[[PriceUpdater.TokensConfig]]
UpdateMethod = "bitfinexV2"
Symbol = "USDT"
Addr = "0xdac17f958d2ee523a2206206994597c13d831ec7"
[[PriceUpdater.TokensConfig]]
UpdateMethod = "coingeckoV3"
Symbol = "ETH"
Addr = "0x0000000000000000000000000000000000000000"
[[PriceUpdater.TokensConfig]]
UpdateMethod = "static"
Symbol = "UNI"
Addr = "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984"
StaticValue = 30.12
[[PriceUpdater.TokensConfig]]
UpdateMethod = "ignore"
Symbol = "SUSHI"
Addr = "0x6b3595068778dd592e39a122f4f5a5cf09c90fe2"
[Debug]
APIAddress = "localhost:12345"

+ 10
- 5
config/config.go

@ -9,6 +9,7 @@ import (
"github.com/BurntSushi/toml"
ethCommon "github.com/ethereum/go-ethereum/common"
"github.com/hermeznetwork/hermez-node/common"
"github.com/hermeznetwork/hermez-node/priceupdater"
"github.com/hermeznetwork/tracerr"
"github.com/iden3/go-iden3-crypto/babyjub"
"gopkg.in/go-playground/validator.v9"
@ -219,11 +220,15 @@ type Coordinator struct {
type Node struct {
PriceUpdater struct {
// Interval between price updater calls
Interval Duration `valudate:"required"`
// URL of the token prices provider
URL string `valudate:"required"`
// Type of the API of the token prices provider
Type string `valudate:"required"`
Interval Duration `validate:"required"`
// URLBitfinexV2 is the URL of bitfinex V2 API
URLBitfinexV2 string `validate:"required"`
// URLCoinGeckoV3 is the URL of coingecko V3 API
URLCoinGeckoV3 string `validate:"required"`
// DefaultUpdateMethod to get token prices
DefaultUpdateMethod priceupdater.UpdateMethodType `validate:"required"`
// TokensConfig to specify how each token get it's price updated
TokensConfig []priceupdater.TokenConfig
} `validate:"required"`
StateDB struct {
// Path where the synchronizer StateDB is stored

+ 4
- 7
db/historydb/historydb.go

@ -456,13 +456,10 @@ func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
// UpdateTokenValue updates the USD value of a token. Value is the price in
// USD of a normalized token (1 token = 10^decimals units)
func (hdb *HistoryDB) UpdateTokenValue(tokenSymbol string, value float64) error {
// Sanitize symbol
tokenSymbol = strings.ToValidUTF8(tokenSymbol, " ")
func (hdb *HistoryDB) UpdateTokenValue(tokenAddr ethCommon.Address, value float64) error {
_, err := hdb.dbWrite.Exec(
"UPDATE token SET usd = $1 WHERE symbol = $2;",
value, tokenSymbol,
"UPDATE token SET usd = $1 WHERE eth_addr = $2;",
value, tokenAddr,
)
return tracerr.Wrap(err)
}
@ -1161,7 +1158,7 @@ func (hdb *HistoryDB) GetTokensTest() ([]TokenWithUSD, error) {
tokens := []*TokenWithUSD{}
if err := meddler.QueryAll(
hdb.dbRead, &tokens,
"SELECT * FROM TOKEN",
"SELECT * FROM token ORDER BY token_id ASC",
); err != nil {
return nil, tracerr.Wrap(err)
}

+ 4
- 4
db/historydb/historydb_test.go

@ -166,7 +166,7 @@ func TestBatches(t *testing.T) {
if i%2 != 0 {
// Set value to the token
value := (float64(i) + 5) * 5.389329
assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
assert.NoError(t, historyDB.UpdateTokenValue(token.EthAddr, value))
tokensValue[token.TokenID] = value / math.Pow(10, float64(token.Decimals))
}
}
@ -276,7 +276,7 @@ func TestTokens(t *testing.T) {
// Update token value
for i, token := range tokens {
value := 1.01 * float64(i)
assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
assert.NoError(t, historyDB.UpdateTokenValue(token.EthAddr, value))
}
// Fetch tokens
fetchedTokens, err = historyDB.GetTokensTest()
@ -302,7 +302,7 @@ func TestTokensUTF8(t *testing.T) {
// Generate fake tokens
const nTokens = 5
tokens, ethToken := test.GenTokens(nTokens, blocks)
nonUTFTokens := make([]common.Token, len(tokens)+1)
nonUTFTokens := make([]common.Token, len(tokens))
// Force token.name and token.symbol to be non UTF-8 Strings
for i, token := range tokens {
token.Name = fmt.Sprint("NON-UTF8-NAME-\xc5-", i)
@ -332,7 +332,7 @@ func TestTokensUTF8(t *testing.T) {
// Update token value
for i, token := range nonUTFTokens {
value := 1.01 * float64(i)
assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
assert.NoError(t, historyDB.UpdateTokenValue(token.EthAddr, value))
}
// Fetch tokens
fetchedTokens, err = historyDB.GetTokensTest()

+ 1
- 1
db/l2db/l2db_test.go

@ -121,7 +121,7 @@ func prepareHistoryDB(historyDB *historydb.HistoryDB) error {
}
tokens[token.TokenID] = readToken
// Set value to the tokens
err := historyDB.UpdateTokenValue(readToken.Symbol, *readToken.USD)
err := historyDB.UpdateTokenValue(readToken.EthAddr, *readToken.USD)
if err != nil {
return tracerr.Wrap(err)
}

+ 7
- 2
node/node.go

@ -423,8 +423,13 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) {
if cfg.Debug.APIAddress != "" {
debugAPI = debugapi.NewDebugAPI(cfg.Debug.APIAddress, stateDB, sync)
}
priceUpdater, err := priceupdater.NewPriceUpdater(cfg.PriceUpdater.URL,
priceupdater.APIType(cfg.PriceUpdater.Type), historyDB)
priceUpdater, err := priceupdater.NewPriceUpdater(
cfg.PriceUpdater.DefaultUpdateMethod,
cfg.PriceUpdater.TokensConfig,
historyDB,
cfg.PriceUpdater.URLBitfinexV2,
cfg.PriceUpdater.URLCoinGeckoV3,
)
if err != nil {
return nil, tracerr.Wrap(err)
}

+ 104
- 48
priceupdater/priceupdater.go

@ -20,57 +20,107 @@ const (
defaultIdleConnTimeout = 2 * time.Second
)
// APIType defines the token exchange API
type APIType string
// UpdateMethodType defines the token price update mechanism
type UpdateMethodType string
const (
// APITypeBitFinexV2 is the http API used by bitfinex V2
APITypeBitFinexV2 APIType = "bitfinexV2"
// APITypeCoingeckoV3 is the http API used by copingecko V3
APITypeCoingeckoV3 APIType = "coingeckoV3"
// UpdateMethodTypeBitFinexV2 is the http API used by bitfinex V2
UpdateMethodTypeBitFinexV2 UpdateMethodType = "bitfinexV2"
// UpdateMethodTypeCoingeckoV3 is the http API used by copingecko V3
UpdateMethodTypeCoingeckoV3 UpdateMethodType = "coingeckoV3"
// UpdateMethodTypeStatic is the value given by the configuration
UpdateMethodTypeStatic UpdateMethodType = "static"
// UpdateMethodTypeIgnore indicates to not update the value, to set value 0
// it's better to use UpdateMethodTypeStatic
UpdateMethodTypeIgnore UpdateMethodType = "ignore"
)
func (t *APIType) valid() bool {
func (t *UpdateMethodType) valid() bool {
switch *t {
case APITypeBitFinexV2:
case UpdateMethodTypeBitFinexV2:
return true
case APITypeCoingeckoV3:
case UpdateMethodTypeCoingeckoV3:
return true
case UpdateMethodTypeStatic:
return true
case UpdateMethodTypeIgnore:
return true
default:
return false
}
}
// TokenConfig specifies how a single token get its price updated
type TokenConfig struct {
UpdateMethod UpdateMethodType
StaticValue float64 // required by UpdateMethodTypeStatic
Symbol string
Addr ethCommon.Address
}
func (t *TokenConfig) valid() bool {
if (t.Addr == common.EmptyAddr && t.Symbol != "ETH") ||
(t.Symbol == "" && t.UpdateMethod == UpdateMethodTypeBitFinexV2) {
return false
}
return t.UpdateMethod.valid()
}
// PriceUpdater definition
type PriceUpdater struct {
db *historydb.HistoryDB
apiURL string
apiType APIType
tokens []historydb.TokenSymbolAndAddr
db *historydb.HistoryDB
defaultUpdateMethod UpdateMethodType
tokensList []historydb.TokenSymbolAndAddr
tokensConfig map[ethCommon.Address]TokenConfig
clientCoingeckoV3 *sling.Sling
clientBitfinexV2 *sling.Sling
}
// NewPriceUpdater is the constructor for the updater
func NewPriceUpdater(apiURL string, apiType APIType, db *historydb.HistoryDB) (*PriceUpdater,
error) {
if !apiType.valid() {
return nil, tracerr.Wrap(fmt.Errorf("Invalid apiType: %v", apiType))
func NewPriceUpdater(
defaultUpdateMethodType UpdateMethodType,
tokensConfig []TokenConfig,
db *historydb.HistoryDB,
bitfinexV2URL, coingeckoV3URL string,
) (*PriceUpdater, error) {
// Validate params
if !defaultUpdateMethodType.valid() || defaultUpdateMethodType == UpdateMethodTypeStatic {
return nil, tracerr.Wrap(
fmt.Errorf("Invalid defaultUpdateMethodType: %v", defaultUpdateMethodType),
)
}
tokensConfigMap := make(map[ethCommon.Address]TokenConfig)
for _, t := range tokensConfig {
if !t.valid() {
return nil, tracerr.Wrap(fmt.Errorf("Invalid tokensConfig, wrong entry: %+v", t))
}
tokensConfigMap[t.Addr] = t
}
// Init
tr := &http.Transport{
MaxIdleConns: defaultMaxIdleConns,
IdleConnTimeout: defaultIdleConnTimeout,
DisableCompression: true,
}
httpClient := &http.Client{Transport: tr}
return &PriceUpdater{
db: db,
apiURL: apiURL,
apiType: apiType,
tokens: []historydb.TokenSymbolAndAddr{},
db: db,
defaultUpdateMethod: defaultUpdateMethodType,
tokensList: []historydb.TokenSymbolAndAddr{},
tokensConfig: tokensConfigMap,
clientCoingeckoV3: sling.New().Base(coingeckoV3URL).Client(httpClient),
clientBitfinexV2: sling.New().Base(bitfinexV2URL).Client(httpClient),
}, nil
}
func getTokenPriceBitfinex(ctx context.Context, client *sling.Sling,
tokenSymbol string) (float64, error) {
func (p *PriceUpdater) getTokenPriceBitfinex(ctx context.Context, tokenSymbol string) (float64, error) {
state := [10]float64{}
req, err := client.New().Get("ticker/t" + tokenSymbol + "USD").Request()
url := "ticker/t" + tokenSymbol + "USD"
req, err := p.clientBitfinexV2.New().Get(url).Request()
if err != nil {
return 0, tracerr.Wrap(err)
}
res, err := client.Do(req.WithContext(ctx), &state, nil)
res, err := p.clientBitfinexV2.Do(req.WithContext(ctx), &state, nil)
if err != nil {
return 0, tracerr.Wrap(err)
}
@ -80,8 +130,7 @@ func getTokenPriceBitfinex(ctx context.Context, client *sling.Sling,
return state[6], nil
}
func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling,
tokenAddr ethCommon.Address) (float64, error) {
func (p *PriceUpdater) getTokenPriceCoingecko(ctx context.Context, tokenAddr ethCommon.Address) (float64, error) {
responseObject := make(map[string]map[string]float64)
var url string
var id string
@ -93,11 +142,11 @@ func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling,
url = "simple/token_price/ethereum?contract_addresses=" +
id + "&vs_currencies=usd"
}
req, err := client.New().Get(url).Request()
req, err := p.clientCoingeckoV3.New().Get(url).Request()
if err != nil {
return 0, tracerr.Wrap(err)
}
res, err := client.Do(req.WithContext(ctx), &responseObject, nil)
res, err := p.clientCoingeckoV3.Do(req.WithContext(ctx), &responseObject, nil)
if err != nil {
return 0, tracerr.Wrap(err)
}
@ -114,43 +163,50 @@ func getTokenPriceCoingecko(ctx context.Context, client *sling.Sling,
// UpdatePrices is triggered by the Coordinator, and internally will update the
// token prices in the db
func (p *PriceUpdater) UpdatePrices(ctx context.Context) {
tr := &http.Transport{
MaxIdleConns: defaultMaxIdleConns,
IdleConnTimeout: defaultIdleConnTimeout,
DisableCompression: true,
}
httpClient := &http.Client{Transport: tr}
client := sling.New().Base(p.apiURL).Client(httpClient)
for _, token := range p.tokens {
for _, token := range p.tokensConfig {
var tokenPrice float64
var err error
switch p.apiType {
case APITypeBitFinexV2:
tokenPrice, err = getTokenPriceBitfinex(ctx, client, token.Symbol)
case APITypeCoingeckoV3:
tokenPrice, err = getTokenPriceCoingecko(ctx, client, token.Addr)
switch token.UpdateMethod {
case UpdateMethodTypeBitFinexV2:
tokenPrice, err = p.getTokenPriceBitfinex(ctx, token.Symbol)
case UpdateMethodTypeCoingeckoV3:
tokenPrice, err = p.getTokenPriceCoingecko(ctx, token.Addr)
case UpdateMethodTypeStatic:
tokenPrice = token.StaticValue
case UpdateMethodTypeIgnore:
continue
}
if ctx.Err() != nil {
return
}
if err != nil {
log.Warnw("token price not updated (get error)",
"err", err, "token", token.Symbol, "apiType", p.apiType)
"err", err, "token", token.Symbol, "updateMethod", token.UpdateMethod)
}
if err = p.db.UpdateTokenValue(token.Symbol, tokenPrice); err != nil {
if err = p.db.UpdateTokenValue(token.Addr, tokenPrice); err != nil {
log.Errorw("token price not updated (db error)",
"err", err, "token", token.Symbol, "apiType", p.apiType)
"err", err, "token", token.Symbol, "updateMethod", token.UpdateMethod)
}
}
}
// UpdateTokenList get the registered token symbols from HistoryDB
func (p *PriceUpdater) UpdateTokenList() error {
tokens, err := p.db.GetTokenSymbolsAndAddrs()
dbTokens, err := p.db.GetTokenSymbolsAndAddrs()
if err != nil {
return tracerr.Wrap(err)
}
p.tokens = tokens
// For each token from the DB
for _, dbToken := range dbTokens {
// If the token doesn't exists in the config list,
// add it with default update emthod
if _, ok := p.tokensConfig[dbToken.Addr]; !ok {
p.tokensConfig[dbToken.Addr] = TokenConfig{
UpdateMethod: p.defaultUpdateMethod,
Symbol: dbToken.Symbol,
Addr: dbToken.Addr,
}
}
}
return nil
}

+ 100
- 45
priceupdater/priceupdater_test.go

@ -16,7 +16,9 @@ import (
var historyDB *historydb.HistoryDB
func TestMain(m *testing.M) {
const usdtAddr = "0xdac17f958d2ee523a2206206994597c13d831ec7"
func TestPriceUpdaterBitfinex(t *testing.T) {
// Init DB
pass := os.Getenv("POSTGRES_PASS")
db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
@ -29,60 +31,113 @@ func TestMain(m *testing.M) {
// Populate DB
// Gen blocks and add them to DB
blocks := test.GenBlocks(1, 2)
err = historyDB.AddBlocks(blocks)
if err != nil {
panic(err)
}
require.NoError(t, historyDB.AddBlocks(blocks))
// Gen tokens and add them to DB
tokens := []common.Token{}
tokens = append(tokens, common.Token{
TokenID: 1,
EthBlockNum: blocks[0].Num,
EthAddr: ethCommon.HexToAddress("0x6b175474e89094c44da98b954eedeac495271d0f"),
Name: "DAI",
Symbol: "DAI",
Decimals: 18,
})
err = historyDB.AddTokens(tokens)
if err != nil {
panic(err)
tokens := []common.Token{
{
TokenID: 1,
EthBlockNum: blocks[0].Num,
EthAddr: ethCommon.HexToAddress("0x1"),
Name: "DAI",
Symbol: "DAI",
Decimals: 18,
}, // Used to test get by SC addr
{
TokenID: 2,
EthBlockNum: blocks[0].Num,
EthAddr: ethCommon.HexToAddress(usdtAddr),
Name: "Tether",
Symbol: "USDT",
Decimals: 18,
}, // Used to test get by token symbol
{
TokenID: 3,
EthBlockNum: blocks[0].Num,
EthAddr: ethCommon.HexToAddress("0x2"),
Name: "FOO",
Symbol: "FOO",
Decimals: 18,
}, // Used to test ignore
{
TokenID: 4,
EthBlockNum: blocks[0].Num,
EthAddr: ethCommon.HexToAddress("0x3"),
Name: "BAR",
Symbol: "BAR",
Decimals: 18,
}, // Used to test static
{
TokenID: 5,
EthBlockNum: blocks[0].Num,
EthAddr: ethCommon.HexToAddress("0x1f9840a85d5af5bf1d1762f925bdaddc4201f984"),
Name: "Uniswap",
Symbol: "UNI",
Decimals: 18,
}, // Used to test default
}
require.NoError(t, historyDB.AddTokens(tokens)) // ETH token exist in DB by default
// Update token price used to test ignore
ignoreValue := 44.44
require.NoError(t, historyDB.UpdateTokenValue(tokens[2].EthAddr, ignoreValue))
result := m.Run()
os.Exit(result)
}
func TestPriceUpdaterBitfinex(t *testing.T) {
// Init price updater
pu, err := NewPriceUpdater("https://api-pub.bitfinex.com/v2/", APITypeBitFinexV2, historyDB)
require.NoError(t, err)
// Update token list
assert.NoError(t, pu.UpdateTokenList())
// Update prices
pu.UpdatePrices(context.Background())
assertTokenHasPriceAndClean(t)
}
// Prepare token config
staticValue := 0.12345
tc := []TokenConfig{
// ETH and UNI tokens use default method
{ // DAI uses SC addr
UpdateMethod: UpdateMethodTypeBitFinexV2,
Addr: ethCommon.HexToAddress("0x1"),
Symbol: "DAI",
},
{ // USDT uses symbol
UpdateMethod: UpdateMethodTypeCoingeckoV3,
Addr: ethCommon.HexToAddress(usdtAddr),
},
{ // FOO uses ignore
UpdateMethod: UpdateMethodTypeIgnore,
Addr: ethCommon.HexToAddress("0x2"),
},
{ // BAR uses static
UpdateMethod: UpdateMethodTypeStatic,
Addr: ethCommon.HexToAddress("0x3"),
StaticValue: staticValue,
},
}
func TestPriceUpdaterCoingecko(t *testing.T) {
bitfinexV2URL := "https://api-pub.bitfinex.com/v2/"
coingeckoV3URL := "https://api.coingecko.com/api/v3/"
// Init price updater
pu, err := NewPriceUpdater("https://api.coingecko.com/api/v3/", APITypeCoingeckoV3, historyDB)
pu, err := NewPriceUpdater(
UpdateMethodTypeCoingeckoV3,
tc,
historyDB,
bitfinexV2URL,
coingeckoV3URL,
)
require.NoError(t, err)
// Update token list
assert.NoError(t, pu.UpdateTokenList())
require.NoError(t, pu.UpdateTokenList())
// Update prices
pu.UpdatePrices(context.Background())
assertTokenHasPriceAndClean(t)
}
func assertTokenHasPriceAndClean(t *testing.T) {
// Check that prices have been updated
// Check results: get tokens from DB
fetchedTokens, err := historyDB.GetTokensTest()
require.NoError(t, err)
// TokenID 0 (ETH) is always on the DB
assert.Equal(t, 2, len(fetchedTokens))
for _, token := range fetchedTokens {
require.NotNil(t, token.USD)
require.NotNil(t, token.USDUpdate)
assert.Greater(t, *token.USD, 0.0)
}
// Check that tokens that are updated via API have value:
// ETH
require.NotNil(t, fetchedTokens[0].USDUpdate)
assert.Greater(t, *fetchedTokens[0].USD, 0.0)
// DAI
require.NotNil(t, fetchedTokens[1].USDUpdate)
assert.Greater(t, *fetchedTokens[1].USD, 0.0)
// USDT
require.NotNil(t, fetchedTokens[2].USDUpdate)
assert.Greater(t, *fetchedTokens[2].USD, 0.0)
// UNI
require.NotNil(t, fetchedTokens[5].USDUpdate)
assert.Greater(t, *fetchedTokens[5].USD, 0.0)
// Check ignored token
assert.Equal(t, ignoreValue, *fetchedTokens[3].USD)
// Check static value
assert.Equal(t, staticValue, *fetchedTokens[4].USD)
}

Loading…
Cancel
Save