Browse Source

Merge pull request #451 from hermeznetwork/feature/varchar-utf8

Sanitize varchars that may not be UTF-8
feature/sql-semaphore1
a_bennassar 3 years ago
committed by GitHub
parent
commit
44b0b8f778
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 68 additions and 0 deletions
  1. +10
    -0
      db/historydb/historydb.go
  2. +58
    -0
      db/historydb/historydb_test.go

+ 10
- 0
db/historydb/historydb.go

@ -5,6 +5,7 @@ import (
"fmt"
"math"
"math/big"
"strings"
ethCommon "github.com/ethereum/go-ethereum/common"
"github.com/hermeznetwork/hermez-node/common"
@ -638,6 +639,12 @@ func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
if len(tokens) == 0 {
return nil
}
// Sanitize name and symbol
for i, token := range tokens {
token.Name = strings.ToValidUTF8(token.Name, " ")
token.Symbol = strings.ToValidUTF8(token.Symbol, " ")
tokens[i] = token
}
return tracerr.Wrap(db.BulkInsert(
d,
`INSERT INTO token (
@ -654,6 +661,9 @@ func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
// UpdateTokenValue updates the USD value of a token
func (hdb *HistoryDB) UpdateTokenValue(tokenSymbol string, value float64) error {
// Sanitize symbol
tokenSymbol = strings.ToValidUTF8(tokenSymbol, " ")
_, err := hdb.db.Exec(
"UPDATE token SET usd = $1 WHERE symbol = $2;",
value, tokenSymbol,

+ 58
- 0
db/historydb/historydb_test.go

@ -6,6 +6,7 @@ import (
"math"
"math/big"
"os"
"strings"
"testing"
"time"
@ -279,6 +280,63 @@ func TestTokens(t *testing.T) {
}
}
func TestTokensUTF8(t *testing.T) {
// Reset DB
test.WipeDB(historyDB.DB())
const fromBlock int64 = 1
const toBlock int64 = 5
// Prepare blocks in the DB
blocks := setTestBlocks(fromBlock, toBlock)
// Generate fake tokens
const nTokens = 5
tokens, ethToken := test.GenTokens(nTokens, blocks)
nonUTFTokens := make([]common.Token, len(tokens)+1)
// Force token.name and token.symbol to be non UTF-8 Strings
for i, token := range tokens {
token.Name = fmt.Sprint("NON-UTF8-NAME-\xc5-", i)
token.Symbol = fmt.Sprint("S-\xc5-", i)
tokens[i] = token
nonUTFTokens[i] = token
}
err := historyDB.AddTokens(tokens)
assert.NoError(t, err)
// Work with nonUTFTokens as tokens one gets updated and non UTF-8 characters are lost
nonUTFTokens = append([]common.Token{ethToken}, nonUTFTokens...)
limit := uint(10)
// Fetch tokens
fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, OrderAsc)
assert.NoError(t, err)
// Compare fetched tokens vs generated tokens
// All the tokens should have USDUpdate setted by the DB trigger
for i, token := range fetchedTokens {
assert.Equal(t, nonUTFTokens[i].TokenID, token.TokenID)
assert.Equal(t, nonUTFTokens[i].EthBlockNum, token.EthBlockNum)
assert.Equal(t, nonUTFTokens[i].EthAddr, token.EthAddr)
assert.Equal(t, strings.ToValidUTF8(nonUTFTokens[i].Name, " "), token.Name)
assert.Equal(t, strings.ToValidUTF8(nonUTFTokens[i].Symbol, " "), token.Symbol)
assert.Nil(t, token.USD)
assert.Nil(t, token.USDUpdate)
}
// Update token value
for i, token := range nonUTFTokens {
value := 1.01 * float64(i)
assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
}
// Fetch tokens
fetchedTokens, _, err = historyDB.GetTokens(nil, nil, "", nil, &limit, OrderAsc)
assert.NoError(t, err)
// Compare fetched tokens vs generated tokens
// All the tokens should have USDUpdate setted by the DB trigger
for i, token := range fetchedTokens {
value := 1.01 * float64(i)
assert.Equal(t, value, *token.USD)
nameZone, offset := token.USDUpdate.Zone()
assert.Equal(t, "UTC", nameZone)
assert.Equal(t, 0, offset)
}
}
func TestAccounts(t *testing.T) {
const fromBlock int64 = 1
const toBlock int64 = 5

Loading…
Cancel
Save