Merge pull request #649 from hermeznetwork/feature/dbbigintsdecimal

Store *big.Int as DECIMAL in sql

apitypes/apitypes.go

@@ -18,7 +18,10 @@ import (
 
 // BigIntStr is used to scan/value *big.Int directly into strings from/to sql DBs.
 // It assumes that *big.Int are inserted/fetched to/from the DB using the BigIntMeddler meddler
-// defined at github.com/hermeznetwork/hermez-node/db
+// defined at github.com/hermeznetwork/hermez-node/db.  Since *big.Int is
+// stored as DECIMAL in SQL, there's no need to implement Scan()/Value()
+// because DECIMALS are encoded/decoded as strings by the sql driver, and
+// BigIntStr is already a string.
 type BigIntStr string
 
 // NewBigIntStr creates a *BigIntStr from a *big.Int.
@@ -31,34 +34,6 @@ func NewBigIntStr(bigInt *big.Int) *BigIntStr {
 	return &bigIntStr
 }
 
-// Scan implements Scanner for database/sql
-func (b *BigIntStr) Scan(src interface{}) error {
-	srcBytes, ok := src.([]byte)
-	if !ok {
-		return tracerr.Wrap(fmt.Errorf("can't scan %T into apitypes.BigIntStr", src))
-	}
-	// bytes to *big.Int
-	bigInt := new(big.Int).SetBytes(srcBytes)
-	// *big.Int to BigIntStr
-	bigIntStr := NewBigIntStr(bigInt)
-	if bigIntStr == nil {
-		return nil
-	}
-	*b = *bigIntStr
-	return nil
-}
-
-// Value implements valuer for database/sql
-func (b BigIntStr) Value() (driver.Value, error) {
-	// string to *big.Int
-	bigInt, ok := new(big.Int).SetString(string(b), 10)
-	if !ok || bigInt == nil {
-		return nil, tracerr.Wrap(errors.New("invalid representation of a *big.Int"))
-	}
-	// *big.Int to bytes
-	return bigInt.Bytes(), nil
-}
-
 // StrBigInt is used to unmarshal BigIntStr directly into an alias of big.Int
 type StrBigInt big.Int
 

db/historydb/historydb.go

@@ -693,11 +693,11 @@ func (hdb *HistoryDB) GetAllExits() ([]common.ExitInfo, error) {
 func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) {
 	var txs []*common.L1Tx
 	err := meddler.QueryAll(
-		hdb.dbRead, &txs, // Note that '\x' gets parsed as a big.Int with value = 0
+		hdb.dbRead, &txs,
 		`SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
 		tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
-		tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE '\x' END) AS effective_amount,
-		tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE '\x' END) AS effective_deposit_amount,
+		tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE 0 END) AS effective_amount,
+		tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE 0 END) AS effective_deposit_amount,
 		tx.eth_block_num, tx.type, tx.batch_num
 		FROM tx WHERE is_l1 = TRUE AND user_origin = TRUE ORDER BY item_id;`,
 	)

db/migrations/0001.sql

@@ -1,5 +1,11 @@
 -- +migrate Up
 
+-- NOTE: We use "DECIMAL(78,0)" to encode go *big.Int types.  All the *big.Int
+-- that we deal with represent a value in the SNARK field, which is an integer
+-- of 256 bits.  `log(2**256, 10) = 77.06`: that is, a 256 bit number can have
+-- at most 78 digits, so we use this value to specify the precision in the
+-- PostgreSQL DECIMAL guaranteeing that we will never lose precision.
+
 -- History
 CREATE TABLE block (
     eth_block_num BIGINT PRIMARY KEY,
@@ -22,10 +28,10 @@ CREATE TABLE batch (
     forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator
     fees_collected BYTEA NOT NULL,
     fee_idxs_coordinator BYTEA NOT NULL,
-    state_root BYTEA NOT NULL,
+    state_root DECIMAL(78,0) NOT NULL,
     num_accounts BIGINT NOT NULL,
     last_idx BIGINT NOT NULL,
-    exit_root BYTEA NOT NULL,
+    exit_root DECIMAL(78,0) NOT NULL,
     forge_l1_txs_num BIGINT,
     slot_num BIGINT NOT NULL,
     total_fees_usd NUMERIC
@@ -34,7 +40,7 @@ CREATE TABLE batch (
 CREATE TABLE bid (
     item_id SERIAL PRIMARY KEY,
     slot_num BIGINT NOT NULL,
-    bid_value BYTEA NOT NULL,
+    bid_value DECIMAL(78,0) NOT NULL,
     eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
     bidder_addr BYTEA NOT NULL -- fake foreign key for coordinator
 );
@@ -106,7 +112,7 @@ CREATE TABLE account_update (
     batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
     idx BIGINT NOT NULL REFERENCES account (idx) ON DELETE CASCADE,
     nonce BIGINT NOT NULL,
-    balance BYTEA NOT NULL
+    balance DECIMAL(78,0) NOT NULL
 );
 
 CREATE TABLE exit_tree (
@@ -114,7 +120,7 @@ CREATE TABLE exit_tree (
     batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE,
     account_idx BIGINT REFERENCES account (idx) ON DELETE CASCADE,
     merkle_proof BYTEA NOT NULL,
-    balance BYTEA NOT NULL,
+    balance DECIMAL(78,0) NOT NULL,
     instant_withdrawn BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL,
     delayed_withdraw_request BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL,
     owner BYTEA,
@@ -164,7 +170,7 @@ CREATE TABLE tx (
     to_idx BIGINT NOT NULL,
     to_eth_addr BYTEA,
     to_bjj BYTEA,
-    amount BYTEA NOT NULL,
+    amount DECIMAL(78,0) NOT NULL,
     amount_success BOOLEAN NOT NULL DEFAULT true,
     amount_f NUMERIC NOT NULL,
     token_id INT NOT NULL REFERENCES token (token_id),
@@ -174,7 +180,7 @@ CREATE TABLE tx (
     -- L1
     to_forge_l1_txs_num BIGINT,
     user_origin BOOLEAN,
-    deposit_amount BYTEA,
+    deposit_amount DECIMAL(78,0),
     deposit_amount_success BOOLEAN NOT NULL DEFAULT true,
     deposit_amount_f NUMERIC,
     deposit_amount_usd NUMERIC,
@@ -544,7 +550,7 @@ FOR EACH ROW EXECUTE PROCEDURE forge_l1_user_txs();
 
 CREATE TABLE rollup_vars (
     eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE,
-    fee_add_token BYTEA NOT NULL,
+    fee_add_token DECIMAL(78,0) NOT NULL,
     forge_l1_timeout BIGINT NOT NULL,
     withdrawal_delay BIGINT NOT NULL,
     buckets BYTEA NOT NULL,
@@ -556,7 +562,7 @@ CREATE TABLE bucket_update (
     eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
     num_bucket BIGINT NOT NULL,
     block_stamp BIGINT NOT NULL,
-    withdrawals BYTEA NOT NULL
+    withdrawals DECIMAL(78,0) NOT NULL
 );
 
 CREATE TABLE token_exchange (
@@ -572,7 +578,7 @@ CREATE TABLE escape_hatch_withdrawal (
     who_addr BYTEA NOT NULL,
     to_addr BYTEA NOT NULL,
     token_addr BYTEA NOT NULL,
-    amount BYTEA NOT NULL
+    amount DECIMAL(78,0) NOT NULL
 );
 
 CREATE TABLE auction_vars (
@@ -610,7 +616,7 @@ CREATE TABLE tx_pool (
     effective_to_eth_addr BYTEA,
     effective_to_bjj BYTEA,
     token_id INT NOT NULL REFERENCES token (token_id) ON DELETE CASCADE,
-    amount BYTEA NOT NULL,
+    amount DECIMAL(78,0) NOT NULL,
     amount_f NUMERIC NOT NULL,
     fee SMALLINT NOT NULL,
     nonce BIGINT NOT NULL,
@@ -624,7 +630,7 @@ CREATE TABLE tx_pool (
     rq_to_eth_addr BYTEA,
     rq_to_bjj BYTEA,
     rq_token_id INT,
-    rq_amount BYTEA,
+    rq_amount DECIMAL(78,0),
     rq_fee SMALLINT,
     rq_nonce BIGINT,
     tx_type VARCHAR(40) NOT NULL,

db/utils.go

@@ -13,6 +13,9 @@ import (
 	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/tracerr"
 	"github.com/jmoiron/sqlx"
+
+	//nolint:errcheck // driver for postgres DB
+	_ "github.com/lib/pq"
 	migrate "github.com/rubenv/sql-migrate"
 	"github.com/russross/meddler"
 	"golang.org/x/sync/semaphore"
@@ -165,7 +168,11 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 		return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer"))
 	}
 	field := fieldPtr.(**big.Int)
-	*field = new(big.Int).SetBytes([]byte(*ptr))
+	var ok bool
+	*field, ok = new(big.Int).SetString(*ptr, 10)
+	if !ok {
+		return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", *ptr))
+	}
 	return nil
 }
 
@@ -173,7 +180,7 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 func (b BigIntMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) {
 	field := fieldPtr.(*big.Int)
 
-	return field.Bytes(), nil
+	return field.String(), nil
 }
 
 // BigIntNullMeddler encodes or decodes the field value to or from JSON
@@ -198,7 +205,12 @@ func (b BigIntNullMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 	if ptr == nil {
 		return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer"))
 	}
-	*field = new(big.Int).SetBytes(ptr)
+	var ok bool
+	*field, ok = new(big.Int).SetString(string(ptr), 10)
+	if !ok {
+		return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", string(ptr)))
+	}
+
 	return nil
 }
 
@@ -208,7 +220,7 @@ func (b BigIntNullMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}
 	if field == nil {
 		return nil, nil
 	}
-	return field.Bytes(), nil
+	return field.String(), nil
 }
 
 // SliceToSlicePtrs converts any []Foo to []*Foo

db/utils_test.go

@@ -1,9 +1,13 @@
 package db
 
 import (
+	"math/big"
+	"os"
 	"testing"
 
+	"github.com/russross/meddler"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type foo struct {
@@ -33,3 +37,42 @@ func TestSlicePtrsToSlice(t *testing.T) {
 		assert.Equal(t, *a[i], b[i])
 	}
 }
+
+func TestBigInt(t *testing.T) {
+	pass := os.Getenv("POSTGRES_PASS")
+	db, err := InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
+	require.NoError(t, err)
+	defer func() {
+		_, err := db.Exec("DROP TABLE IF EXISTS test_big_int;")
+		require.NoError(t, err)
+		err = db.Close()
+		require.NoError(t, err)
+	}()
+
+	_, err = db.Exec("DROP TABLE IF EXISTS test_big_int;")
+	require.NoError(t, err)
+
+	_, err = db.Exec(`CREATE TABLE test_big_int (
+		item_id SERIAL PRIMARY KEY,
+		value1 DECIMAL(78, 0) NOT NULL,
+		value2 DECIMAL(78, 0),
+		value3 DECIMAL(78, 0)
+	);`)
+	require.NoError(t, err)
+
+	type Entry struct {
+		ItemID int      `meddler:"item_id"`
+		Value1 *big.Int `meddler:"value1,bigint"`
+		Value2 *big.Int `meddler:"value2,bigintnull"`
+		Value3 *big.Int `meddler:"value3,bigintnull"`
+	}
+
+	entry := Entry{ItemID: 1, Value1: big.NewInt(1234567890), Value2: big.NewInt(9876543210), Value3: nil}
+	err = meddler.Insert(db, "test_big_int", &entry)
+	require.NoError(t, err)
+
+	var dbEntry Entry
+	err = meddler.QueryRow(db, &dbEntry, "SELECT * FROM test_big_int WHERE item_id = 1;")
+	require.NoError(t, err)
+	assert.Equal(t, entry, dbEntry)
+}