Merge pull request #451 from hermeznetwork/feature/varchar-utf8

Sanitize varchars that may not be UTF-8

db/historydb/historydb.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"math/big"
+	"strings"
 
 	ethCommon "github.com/ethereum/go-ethereum/common"
 	"github.com/hermeznetwork/hermez-node/common"
@@ -638,6 +639,12 @@ func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
 	if len(tokens) == 0 {
 		return nil
 	}
+	// Sanitize name and symbol
+	for i, token := range tokens {
+		token.Name = strings.ToValidUTF8(token.Name, " ")
+		token.Symbol = strings.ToValidUTF8(token.Symbol, " ")
+		tokens[i] = token
+	}
 	return tracerr.Wrap(db.BulkInsert(
 		d,
 		`INSERT INTO token (
@@ -654,6 +661,9 @@ func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
 
 // UpdateTokenValue updates the USD value of a token
 func (hdb *HistoryDB) UpdateTokenValue(tokenSymbol string, value float64) error {
+	// Sanitize symbol
+	tokenSymbol = strings.ToValidUTF8(tokenSymbol, " ")
+
 	_, err := hdb.db.Exec(
 		"UPDATE token SET usd = $1 WHERE symbol = $2;",
 		value, tokenSymbol,

db/historydb/historydb_test.go

@@ -6,6 +6,7 @@ import (
 	"math"
 	"math/big"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -279,6 +280,63 @@ func TestTokens(t *testing.T) {
 	}
 }
 
+func TestTokensUTF8(t *testing.T) {
+	// Reset DB
+	test.WipeDB(historyDB.DB())
+	const fromBlock int64 = 1
+	const toBlock int64 = 5
+	// Prepare blocks in the DB
+	blocks := setTestBlocks(fromBlock, toBlock)
+	// Generate fake tokens
+	const nTokens = 5
+	tokens, ethToken := test.GenTokens(nTokens, blocks)
+	nonUTFTokens := make([]common.Token, len(tokens)+1)
+	// Force token.name and token.symbol to be non UTF-8 Strings
+	for i, token := range tokens {
+		token.Name = fmt.Sprint("NON-UTF8-NAME-\xc5-", i)
+		token.Symbol = fmt.Sprint("S-\xc5-", i)
+		tokens[i] = token
+		nonUTFTokens[i] = token
+	}
+	err := historyDB.AddTokens(tokens)
+	assert.NoError(t, err)
+	// Work with nonUTFTokens as tokens one gets updated and non UTF-8 characters are lost
+	nonUTFTokens = append([]common.Token{ethToken}, nonUTFTokens...)
+	limit := uint(10)
+	// Fetch tokens
+	fetchedTokens, _, err := historyDB.GetTokens(nil, nil, "", nil, &limit, OrderAsc)
+	assert.NoError(t, err)
+	// Compare fetched tokens vs generated tokens
+	// All the tokens should have USDUpdate setted by the DB trigger
+	for i, token := range fetchedTokens {
+		assert.Equal(t, nonUTFTokens[i].TokenID, token.TokenID)
+		assert.Equal(t, nonUTFTokens[i].EthBlockNum, token.EthBlockNum)
+		assert.Equal(t, nonUTFTokens[i].EthAddr, token.EthAddr)
+		assert.Equal(t, strings.ToValidUTF8(nonUTFTokens[i].Name, " "), token.Name)
+		assert.Equal(t, strings.ToValidUTF8(nonUTFTokens[i].Symbol, " "), token.Symbol)
+		assert.Nil(t, token.USD)
+		assert.Nil(t, token.USDUpdate)
+	}
+
+	// Update token value
+	for i, token := range nonUTFTokens {
+		value := 1.01 * float64(i)
+		assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
+	}
+	// Fetch tokens
+	fetchedTokens, _, err = historyDB.GetTokens(nil, nil, "", nil, &limit, OrderAsc)
+	assert.NoError(t, err)
+	// Compare fetched tokens vs generated tokens
+	// All the tokens should have USDUpdate setted by the DB trigger
+	for i, token := range fetchedTokens {
+		value := 1.01 * float64(i)
+		assert.Equal(t, value, *token.USD)
+		nameZone, offset := token.USDUpdate.Zone()
+		assert.Equal(t, "UTC", nameZone)
+		assert.Equal(t, 0, offset)
+	}
+}
+
 func TestAccounts(t *testing.T) {
 	const fromBlock int64 = 1
 	const toBlock int64 = 5