Browse Source

Merge pull request #649 from hermeznetwork/feature/dbbigintsdecimal

Store *big.Int as DECIMAL in sql
feature/update-smart-contracts
a_bennassar 3 years ago
committed by GitHub
parent
commit
8018f348a7
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 84 additions and 48 deletions
  1. +4
    -29
      apitypes/apitypes.go
  2. +3
    -3
      db/historydb/historydb.go
  3. +18
    -12
      db/migrations/0001.sql
  4. +16
    -4
      db/utils.go
  5. +43
    -0
      db/utils_test.go

+ 4
- 29
apitypes/apitypes.go

@ -18,7 +18,10 @@ import (
// BigIntStr is used to scan/value *big.Int directly into strings from/to sql DBs. // BigIntStr is used to scan/value *big.Int directly into strings from/to sql DBs.
// It assumes that *big.Int are inserted/fetched to/from the DB using the BigIntMeddler meddler // It assumes that *big.Int are inserted/fetched to/from the DB using the BigIntMeddler meddler
// defined at github.com/hermeznetwork/hermez-node/db
// defined at github.com/hermeznetwork/hermez-node/db. Since *big.Int is
// stored as DECIMAL in SQL, there's no need to implement Scan()/Value()
// because DECIMALS are encoded/decoded as strings by the sql driver, and
// BigIntStr is already a string.
type BigIntStr string type BigIntStr string
// NewBigIntStr creates a *BigIntStr from a *big.Int. // NewBigIntStr creates a *BigIntStr from a *big.Int.
@ -31,34 +34,6 @@ func NewBigIntStr(bigInt *big.Int) *BigIntStr {
return &bigIntStr return &bigIntStr
} }
// Scan implements Scanner for database/sql
func (b *BigIntStr) Scan(src interface{}) error {
srcBytes, ok := src.([]byte)
if !ok {
return tracerr.Wrap(fmt.Errorf("can't scan %T into apitypes.BigIntStr", src))
}
// bytes to *big.Int
bigInt := new(big.Int).SetBytes(srcBytes)
// *big.Int to BigIntStr
bigIntStr := NewBigIntStr(bigInt)
if bigIntStr == nil {
return nil
}
*b = *bigIntStr
return nil
}
// Value implements valuer for database/sql
func (b BigIntStr) Value() (driver.Value, error) {
// string to *big.Int
bigInt, ok := new(big.Int).SetString(string(b), 10)
if !ok || bigInt == nil {
return nil, tracerr.Wrap(errors.New("invalid representation of a *big.Int"))
}
// *big.Int to bytes
return bigInt.Bytes(), nil
}
// StrBigInt is used to unmarshal BigIntStr directly into an alias of big.Int // StrBigInt is used to unmarshal BigIntStr directly into an alias of big.Int
type StrBigInt big.Int type StrBigInt big.Int

+ 3
- 3
db/historydb/historydb.go

@ -693,11 +693,11 @@ func (hdb *HistoryDB) GetAllExits() ([]common.ExitInfo, error) {
func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) { func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) {
var txs []*common.L1Tx var txs []*common.L1Tx
err := meddler.QueryAll( err := meddler.QueryAll(
hdb.dbRead, &txs, // Note that '\x' gets parsed as a big.Int with value = 0
hdb.dbRead, &txs,
`SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin, `SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id, tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE '\x' END) AS effective_amount,
tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE '\x' END) AS effective_deposit_amount,
tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE 0 END) AS effective_amount,
tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE 0 END) AS effective_deposit_amount,
tx.eth_block_num, tx.type, tx.batch_num tx.eth_block_num, tx.type, tx.batch_num
FROM tx WHERE is_l1 = TRUE AND user_origin = TRUE ORDER BY item_id;`, FROM tx WHERE is_l1 = TRUE AND user_origin = TRUE ORDER BY item_id;`,
) )

+ 18
- 12
db/migrations/0001.sql

@ -1,5 +1,11 @@
-- +migrate Up -- +migrate Up
-- NOTE: We use "DECIMAL(78,0)" to encode go *big.Int types. All the *big.Int
-- that we deal with represent a value in the SNARK field, which is an integer
-- of 256 bits. `log(2**256, 10) = 77.06`: that is, a 256 bit number can have
-- at most 78 digits, so we use this value to specify the precision in the
-- PostgreSQL DECIMAL guaranteeing that we will never lose precision.
-- History -- History
CREATE TABLE block ( CREATE TABLE block (
eth_block_num BIGINT PRIMARY KEY, eth_block_num BIGINT PRIMARY KEY,
@ -22,10 +28,10 @@ CREATE TABLE batch (
forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator
fees_collected BYTEA NOT NULL, fees_collected BYTEA NOT NULL,
fee_idxs_coordinator BYTEA NOT NULL, fee_idxs_coordinator BYTEA NOT NULL,
state_root BYTEA NOT NULL,
state_root DECIMAL(78,0) NOT NULL,
num_accounts BIGINT NOT NULL, num_accounts BIGINT NOT NULL,
last_idx BIGINT NOT NULL, last_idx BIGINT NOT NULL,
exit_root BYTEA NOT NULL,
exit_root DECIMAL(78,0) NOT NULL,
forge_l1_txs_num BIGINT, forge_l1_txs_num BIGINT,
slot_num BIGINT NOT NULL, slot_num BIGINT NOT NULL,
total_fees_usd NUMERIC total_fees_usd NUMERIC
@ -34,7 +40,7 @@ CREATE TABLE batch (
CREATE TABLE bid ( CREATE TABLE bid (
item_id SERIAL PRIMARY KEY, item_id SERIAL PRIMARY KEY,
slot_num BIGINT NOT NULL, slot_num BIGINT NOT NULL,
bid_value BYTEA NOT NULL,
bid_value DECIMAL(78,0) NOT NULL,
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
bidder_addr BYTEA NOT NULL -- fake foreign key for coordinator bidder_addr BYTEA NOT NULL -- fake foreign key for coordinator
); );
@ -106,7 +112,7 @@ CREATE TABLE account_update (
batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE, batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
idx BIGINT NOT NULL REFERENCES account (idx) ON DELETE CASCADE, idx BIGINT NOT NULL REFERENCES account (idx) ON DELETE CASCADE,
nonce BIGINT NOT NULL, nonce BIGINT NOT NULL,
balance BYTEA NOT NULL
balance DECIMAL(78,0) NOT NULL
); );
CREATE TABLE exit_tree ( CREATE TABLE exit_tree (
@ -114,7 +120,7 @@ CREATE TABLE exit_tree (
batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE, batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE,
account_idx BIGINT REFERENCES account (idx) ON DELETE CASCADE, account_idx BIGINT REFERENCES account (idx) ON DELETE CASCADE,
merkle_proof BYTEA NOT NULL, merkle_proof BYTEA NOT NULL,
balance BYTEA NOT NULL,
balance DECIMAL(78,0) NOT NULL,
instant_withdrawn BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL, instant_withdrawn BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL,
delayed_withdraw_request BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL, delayed_withdraw_request BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL,
owner BYTEA, owner BYTEA,
@ -164,7 +170,7 @@ CREATE TABLE tx (
to_idx BIGINT NOT NULL, to_idx BIGINT NOT NULL,
to_eth_addr BYTEA, to_eth_addr BYTEA,
to_bjj BYTEA, to_bjj BYTEA,
amount BYTEA NOT NULL,
amount DECIMAL(78,0) NOT NULL,
amount_success BOOLEAN NOT NULL DEFAULT true, amount_success BOOLEAN NOT NULL DEFAULT true,
amount_f NUMERIC NOT NULL, amount_f NUMERIC NOT NULL,
token_id INT NOT NULL REFERENCES token (token_id), token_id INT NOT NULL REFERENCES token (token_id),
@ -174,7 +180,7 @@ CREATE TABLE tx (
-- L1 -- L1
to_forge_l1_txs_num BIGINT, to_forge_l1_txs_num BIGINT,
user_origin BOOLEAN, user_origin BOOLEAN,
deposit_amount BYTEA,
deposit_amount DECIMAL(78,0),
deposit_amount_success BOOLEAN NOT NULL DEFAULT true, deposit_amount_success BOOLEAN NOT NULL DEFAULT true,
deposit_amount_f NUMERIC, deposit_amount_f NUMERIC,
deposit_amount_usd NUMERIC, deposit_amount_usd NUMERIC,
@ -544,7 +550,7 @@ FOR EACH ROW EXECUTE PROCEDURE forge_l1_user_txs();
CREATE TABLE rollup_vars ( CREATE TABLE rollup_vars (
eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE, eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE,
fee_add_token BYTEA NOT NULL,
fee_add_token DECIMAL(78,0) NOT NULL,
forge_l1_timeout BIGINT NOT NULL, forge_l1_timeout BIGINT NOT NULL,
withdrawal_delay BIGINT NOT NULL, withdrawal_delay BIGINT NOT NULL,
buckets BYTEA NOT NULL, buckets BYTEA NOT NULL,
@ -556,7 +562,7 @@ CREATE TABLE bucket_update (
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE, eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
num_bucket BIGINT NOT NULL, num_bucket BIGINT NOT NULL,
block_stamp BIGINT NOT NULL, block_stamp BIGINT NOT NULL,
withdrawals BYTEA NOT NULL
withdrawals DECIMAL(78,0) NOT NULL
); );
CREATE TABLE token_exchange ( CREATE TABLE token_exchange (
@ -572,7 +578,7 @@ CREATE TABLE escape_hatch_withdrawal (
who_addr BYTEA NOT NULL, who_addr BYTEA NOT NULL,
to_addr BYTEA NOT NULL, to_addr BYTEA NOT NULL,
token_addr BYTEA NOT NULL, token_addr BYTEA NOT NULL,
amount BYTEA NOT NULL
amount DECIMAL(78,0) NOT NULL
); );
CREATE TABLE auction_vars ( CREATE TABLE auction_vars (
@ -610,7 +616,7 @@ CREATE TABLE tx_pool (
effective_to_eth_addr BYTEA, effective_to_eth_addr BYTEA,
effective_to_bjj BYTEA, effective_to_bjj BYTEA,
token_id INT NOT NULL REFERENCES token (token_id) ON DELETE CASCADE, token_id INT NOT NULL REFERENCES token (token_id) ON DELETE CASCADE,
amount BYTEA NOT NULL,
amount DECIMAL(78,0) NOT NULL,
amount_f NUMERIC NOT NULL, amount_f NUMERIC NOT NULL,
fee SMALLINT NOT NULL, fee SMALLINT NOT NULL,
nonce BIGINT NOT NULL, nonce BIGINT NOT NULL,
@ -624,7 +630,7 @@ CREATE TABLE tx_pool (
rq_to_eth_addr BYTEA, rq_to_eth_addr BYTEA,
rq_to_bjj BYTEA, rq_to_bjj BYTEA,
rq_token_id INT, rq_token_id INT,
rq_amount BYTEA,
rq_amount DECIMAL(78,0),
rq_fee SMALLINT, rq_fee SMALLINT,
rq_nonce BIGINT, rq_nonce BIGINT,
tx_type VARCHAR(40) NOT NULL, tx_type VARCHAR(40) NOT NULL,

+ 16
- 4
db/utils.go

@ -13,6 +13,9 @@ import (
"github.com/hermeznetwork/hermez-node/log" "github.com/hermeznetwork/hermez-node/log"
"github.com/hermeznetwork/tracerr" "github.com/hermeznetwork/tracerr"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
//nolint:errcheck // driver for postgres DB
_ "github.com/lib/pq"
migrate "github.com/rubenv/sql-migrate" migrate "github.com/rubenv/sql-migrate"
"github.com/russross/meddler" "github.com/russross/meddler"
"golang.org/x/sync/semaphore" "golang.org/x/sync/semaphore"
@ -165,7 +168,11 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer")) return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer"))
} }
field := fieldPtr.(**big.Int) field := fieldPtr.(**big.Int)
*field = new(big.Int).SetBytes([]byte(*ptr))
var ok bool
*field, ok = new(big.Int).SetString(*ptr, 10)
if !ok {
return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", *ptr))
}
return nil return nil
} }
@ -173,7 +180,7 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
func (b BigIntMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) { func (b BigIntMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) {
field := fieldPtr.(*big.Int) field := fieldPtr.(*big.Int)
return field.Bytes(), nil
return field.String(), nil
} }
// BigIntNullMeddler encodes or decodes the field value to or from JSON // BigIntNullMeddler encodes or decodes the field value to or from JSON
@ -198,7 +205,12 @@ func (b BigIntNullMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
if ptr == nil { if ptr == nil {
return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer")) return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer"))
} }
*field = new(big.Int).SetBytes(ptr)
var ok bool
*field, ok = new(big.Int).SetString(string(ptr), 10)
if !ok {
return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", string(ptr)))
}
return nil return nil
} }
@ -208,7 +220,7 @@ func (b BigIntNullMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}
if field == nil { if field == nil {
return nil, nil return nil, nil
} }
return field.Bytes(), nil
return field.String(), nil
} }
// SliceToSlicePtrs converts any []Foo to []*Foo // SliceToSlicePtrs converts any []Foo to []*Foo

+ 43
- 0
db/utils_test.go

@ -1,9 +1,13 @@
package db package db
import ( import (
"math/big"
"os"
"testing" "testing"
"github.com/russross/meddler"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
type foo struct { type foo struct {
@ -33,3 +37,42 @@ func TestSlicePtrsToSlice(t *testing.T) {
assert.Equal(t, *a[i], b[i]) assert.Equal(t, *a[i], b[i])
} }
} }
func TestBigInt(t *testing.T) {
pass := os.Getenv("POSTGRES_PASS")
db, err := InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
require.NoError(t, err)
defer func() {
_, err := db.Exec("DROP TABLE IF EXISTS test_big_int;")
require.NoError(t, err)
err = db.Close()
require.NoError(t, err)
}()
_, err = db.Exec("DROP TABLE IF EXISTS test_big_int;")
require.NoError(t, err)
_, err = db.Exec(`CREATE TABLE test_big_int (
item_id SERIAL PRIMARY KEY,
value1 DECIMAL(78, 0) NOT NULL,
value2 DECIMAL(78, 0),
value3 DECIMAL(78, 0)
);`)
require.NoError(t, err)
type Entry struct {
ItemID int `meddler:"item_id"`
Value1 *big.Int `meddler:"value1,bigint"`
Value2 *big.Int `meddler:"value2,bigintnull"`
Value3 *big.Int `meddler:"value3,bigintnull"`
}
entry := Entry{ItemID: 1, Value1: big.NewInt(1234567890), Value2: big.NewInt(9876543210), Value3: nil}
err = meddler.Insert(db, "test_big_int", &entry)
require.NoError(t, err)
var dbEntry Entry
err = meddler.QueryRow(db, &dbEntry, "SELECT * FROM test_big_int WHERE item_id = 1;")
require.NoError(t, err)
assert.Equal(t, entry, dbEntry)
}

Loading…
Cancel
Save