From 91ffdc250b104375a9c53f6d55ef80a110e4df0f Mon Sep 17 00:00:00 2001 From: Eduard S Date: Thu, 18 Mar 2021 12:46:59 +0100 Subject: [PATCH] Store *big.Int as DECIMAL in sql --- apitypes/apitypes.go | 33 ++++-------------------------- db/historydb/historydb.go | 6 +++--- db/migrations/0001.sql | 30 ++++++++++++++++----------- db/utils.go | 20 ++++++++++++++---- db/utils_test.go | 43 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 48 deletions(-) diff --git a/apitypes/apitypes.go b/apitypes/apitypes.go index 274f363..b9e25e9 100644 --- a/apitypes/apitypes.go +++ b/apitypes/apitypes.go @@ -18,7 +18,10 @@ import ( // BigIntStr is used to scan/value *big.Int directly into strings from/to sql DBs. // It assumes that *big.Int are inserted/fetched to/from the DB using the BigIntMeddler meddler -// defined at github.com/hermeznetwork/hermez-node/db +// defined at github.com/hermeznetwork/hermez-node/db. Since *big.Int is +// stored as DECIMAL in SQL, there's no need to implement Scan()/Value() +// because DECIMALS are encoded/decoded as strings by the sql driver, and +// BigIntStr is already a string. type BigIntStr string // NewBigIntStr creates a *BigIntStr from a *big.Int. @@ -31,34 +34,6 @@ func NewBigIntStr(bigInt *big.Int) *BigIntStr { return &bigIntStr } -// Scan implements Scanner for database/sql -func (b *BigIntStr) Scan(src interface{}) error { - srcBytes, ok := src.([]byte) - if !ok { - return tracerr.Wrap(fmt.Errorf("can't scan %T into apitypes.BigIntStr", src)) - } - // bytes to *big.Int - bigInt := new(big.Int).SetBytes(srcBytes) - // *big.Int to BigIntStr - bigIntStr := NewBigIntStr(bigInt) - if bigIntStr == nil { - return nil - } - *b = *bigIntStr - return nil -} - -// Value implements valuer for database/sql -func (b BigIntStr) Value() (driver.Value, error) { - // string to *big.Int - bigInt, ok := new(big.Int).SetString(string(b), 10) - if !ok || bigInt == nil { - return nil, tracerr.Wrap(errors.New("invalid representation of a *big.Int")) - } - // *big.Int to bytes - return bigInt.Bytes(), nil -} - // StrBigInt is used to unmarshal BigIntStr directly into an alias of big.Int type StrBigInt big.Int diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index d16a0c5..8261a6e 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -693,11 +693,11 @@ func (hdb *HistoryDB) GetAllExits() ([]common.ExitInfo, error) { func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) { var txs []*common.L1Tx err := meddler.QueryAll( - hdb.dbRead, &txs, // Note that '\x' gets parsed as a big.Int with value = 0 + hdb.dbRead, &txs, `SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin, tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id, - tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE '\x' END) AS effective_amount, - tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE '\x' END) AS effective_deposit_amount, + tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE 0 END) AS effective_amount, + tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE 0 END) AS effective_deposit_amount, tx.eth_block_num, tx.type, tx.batch_num FROM tx WHERE is_l1 = TRUE AND user_origin = TRUE ORDER BY item_id;`, ) diff --git a/db/migrations/0001.sql b/db/migrations/0001.sql index 268c0e6..ec481e3 100644 --- a/db/migrations/0001.sql +++ b/db/migrations/0001.sql @@ -1,5 +1,11 @@ -- +migrate Up +-- NOTE: We use "DECIMAL(78,0)" to encode go *big.Int types. All the *big.Int +-- that we deal with represent a value in the SNARK field, which is an integer +-- of 256 bits. `log(2**256, 10) = 77.06`: that is, a 256 bit number can have +-- at most 78 digits, so we use this value to specify the precision in the +-- PostgreSQL DECIMAL guaranteeing that we will never lose precision. + -- History CREATE TABLE block ( eth_block_num BIGINT PRIMARY KEY, @@ -22,10 +28,10 @@ CREATE TABLE batch ( forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator fees_collected BYTEA NOT NULL, fee_idxs_coordinator BYTEA NOT NULL, - state_root BYTEA NOT NULL, + state_root DECIMAL(78,0) NOT NULL, num_accounts BIGINT NOT NULL, last_idx BIGINT NOT NULL, - exit_root BYTEA NOT NULL, + exit_root DECIMAL(78,0) NOT NULL, forge_l1_txs_num BIGINT, slot_num BIGINT NOT NULL, total_fees_usd NUMERIC @@ -34,7 +40,7 @@ CREATE TABLE batch ( CREATE TABLE bid ( item_id SERIAL PRIMARY KEY, slot_num BIGINT NOT NULL, - bid_value BYTEA NOT NULL, + bid_value DECIMAL(78,0) NOT NULL, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, bidder_addr BYTEA NOT NULL -- fake foreign key for coordinator ); @@ -106,7 +112,7 @@ CREATE TABLE account_update ( batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE, idx BIGINT NOT NULL REFERENCES account (idx) ON DELETE CASCADE, nonce BIGINT NOT NULL, - balance BYTEA NOT NULL + balance DECIMAL(78,0) NOT NULL ); CREATE TABLE exit_tree ( @@ -114,7 +120,7 @@ CREATE TABLE exit_tree ( batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE, account_idx BIGINT REFERENCES account (idx) ON DELETE CASCADE, merkle_proof BYTEA NOT NULL, - balance BYTEA NOT NULL, + balance DECIMAL(78,0) NOT NULL, instant_withdrawn BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL, delayed_withdraw_request BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL, owner BYTEA, @@ -164,7 +170,7 @@ CREATE TABLE tx ( to_idx BIGINT NOT NULL, to_eth_addr BYTEA, to_bjj BYTEA, - amount BYTEA NOT NULL, + amount DECIMAL(78,0) NOT NULL, amount_success BOOLEAN NOT NULL DEFAULT true, amount_f NUMERIC NOT NULL, token_id INT NOT NULL REFERENCES token (token_id), @@ -174,7 +180,7 @@ CREATE TABLE tx ( -- L1 to_forge_l1_txs_num BIGINT, user_origin BOOLEAN, - deposit_amount BYTEA, + deposit_amount DECIMAL(78,0), deposit_amount_success BOOLEAN NOT NULL DEFAULT true, deposit_amount_f NUMERIC, deposit_amount_usd NUMERIC, @@ -544,7 +550,7 @@ FOR EACH ROW EXECUTE PROCEDURE forge_l1_user_txs(); CREATE TABLE rollup_vars ( eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE, - fee_add_token BYTEA NOT NULL, + fee_add_token DECIMAL(78,0) NOT NULL, forge_l1_timeout BIGINT NOT NULL, withdrawal_delay BIGINT NOT NULL, buckets BYTEA NOT NULL, @@ -556,7 +562,7 @@ CREATE TABLE bucket_update ( eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, num_bucket BIGINT NOT NULL, block_stamp BIGINT NOT NULL, - withdrawals BYTEA NOT NULL + withdrawals DECIMAL(78,0) NOT NULL ); CREATE TABLE token_exchange ( @@ -572,7 +578,7 @@ CREATE TABLE escape_hatch_withdrawal ( who_addr BYTEA NOT NULL, to_addr BYTEA NOT NULL, token_addr BYTEA NOT NULL, - amount BYTEA NOT NULL + amount DECIMAL(78,0) NOT NULL ); CREATE TABLE auction_vars ( @@ -610,7 +616,7 @@ CREATE TABLE tx_pool ( effective_to_eth_addr BYTEA, effective_to_bjj BYTEA, token_id INT NOT NULL REFERENCES token (token_id) ON DELETE CASCADE, - amount BYTEA NOT NULL, + amount DECIMAL(78,0) NOT NULL, amount_f NUMERIC NOT NULL, fee SMALLINT NOT NULL, nonce BIGINT NOT NULL, @@ -624,7 +630,7 @@ CREATE TABLE tx_pool ( rq_to_eth_addr BYTEA, rq_to_bjj BYTEA, rq_token_id INT, - rq_amount BYTEA, + rq_amount DECIMAL(78,0), rq_fee SMALLINT, rq_nonce BIGINT, tx_type VARCHAR(40) NOT NULL, diff --git a/db/utils.go b/db/utils.go index bf7d304..f7bdcc5 100644 --- a/db/utils.go +++ b/db/utils.go @@ -13,6 +13,9 @@ import ( "github.com/hermeznetwork/hermez-node/log" "github.com/hermeznetwork/tracerr" "github.com/jmoiron/sqlx" + + //nolint:errcheck // driver for postgres DB + _ "github.com/lib/pq" migrate "github.com/rubenv/sql-migrate" "github.com/russross/meddler" "golang.org/x/sync/semaphore" @@ -165,7 +168,11 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error { return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer")) } field := fieldPtr.(**big.Int) - *field = new(big.Int).SetBytes([]byte(*ptr)) + var ok bool + *field, ok = new(big.Int).SetString(*ptr, 10) + if !ok { + return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", *ptr)) + } return nil } @@ -173,7 +180,7 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error { func (b BigIntMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) { field := fieldPtr.(*big.Int) - return field.Bytes(), nil + return field.String(), nil } // BigIntNullMeddler encodes or decodes the field value to or from JSON @@ -198,7 +205,12 @@ func (b BigIntNullMeddler) PostRead(fieldPtr, scanTarget interface{}) error { if ptr == nil { return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer")) } - *field = new(big.Int).SetBytes(ptr) + var ok bool + *field, ok = new(big.Int).SetString(string(ptr), 10) + if !ok { + return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", string(ptr))) + } + return nil } @@ -208,7 +220,7 @@ func (b BigIntNullMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{} if field == nil { return nil, nil } - return field.Bytes(), nil + return field.String(), nil } // SliceToSlicePtrs converts any []Foo to []*Foo diff --git a/db/utils_test.go b/db/utils_test.go index a5c83b2..b007abf 100644 --- a/db/utils_test.go +++ b/db/utils_test.go @@ -1,9 +1,13 @@ package db import ( + "math/big" + "os" "testing" + "github.com/russross/meddler" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type foo struct { @@ -33,3 +37,42 @@ func TestSlicePtrsToSlice(t *testing.T) { assert.Equal(t, *a[i], b[i]) } } + +func TestBigInt(t *testing.T) { + pass := os.Getenv("POSTGRES_PASS") + db, err := InitSQLDB(5432, "localhost", "hermez", pass, "hermez") + require.NoError(t, err) + defer func() { + _, err := db.Exec("DROP TABLE IF EXISTS test_big_int;") + require.NoError(t, err) + err = db.Close() + require.NoError(t, err) + }() + + _, err = db.Exec("DROP TABLE IF EXISTS test_big_int;") + require.NoError(t, err) + + _, err = db.Exec(`CREATE TABLE test_big_int ( + item_id SERIAL PRIMARY KEY, + value1 DECIMAL(78, 0) NOT NULL, + value2 DECIMAL(78, 0), + value3 DECIMAL(78, 0) + );`) + require.NoError(t, err) + + type Entry struct { + ItemID int `meddler:"item_id"` + Value1 *big.Int `meddler:"value1,bigint"` + Value2 *big.Int `meddler:"value2,bigintnull"` + Value3 *big.Int `meddler:"value3,bigintnull"` + } + + entry := Entry{ItemID: 1, Value1: big.NewInt(1234567890), Value2: big.NewInt(9876543210), Value3: nil} + err = meddler.Insert(db, "test_big_int", &entry) + require.NoError(t, err) + + var dbEntry Entry + err = meddler.QueryRow(db, &dbEntry, "SELECT * FROM test_big_int WHERE item_id = 1;") + require.NoError(t, err) + assert.Equal(t, entry, dbEntry) +}