Browse Source

Store *big.Int as DECIMAL in sql

feature/update-smart-contracts
Eduard S 3 years ago
parent
commit
91ffdc250b
5 changed files with 84 additions and 48 deletions
  1. +4
    -29
      apitypes/apitypes.go
  2. +3
    -3
      db/historydb/historydb.go
  3. +18
    -12
      db/migrations/0001.sql
  4. +16
    -4
      db/utils.go
  5. +43
    -0
      db/utils_test.go

+ 4
- 29
apitypes/apitypes.go

@ -18,7 +18,10 @@ import (
// BigIntStr is used to scan/value *big.Int directly into strings from/to sql DBs.
// It assumes that *big.Int are inserted/fetched to/from the DB using the BigIntMeddler meddler
// defined at github.com/hermeznetwork/hermez-node/db
// defined at github.com/hermeznetwork/hermez-node/db. Since *big.Int is
// stored as DECIMAL in SQL, there's no need to implement Scan()/Value()
// because DECIMALS are encoded/decoded as strings by the sql driver, and
// BigIntStr is already a string.
type BigIntStr string
// NewBigIntStr creates a *BigIntStr from a *big.Int.
@ -31,34 +34,6 @@ func NewBigIntStr(bigInt *big.Int) *BigIntStr {
return &bigIntStr
}
// Scan implements Scanner for database/sql
func (b *BigIntStr) Scan(src interface{}) error {
srcBytes, ok := src.([]byte)
if !ok {
return tracerr.Wrap(fmt.Errorf("can't scan %T into apitypes.BigIntStr", src))
}
// bytes to *big.Int
bigInt := new(big.Int).SetBytes(srcBytes)
// *big.Int to BigIntStr
bigIntStr := NewBigIntStr(bigInt)
if bigIntStr == nil {
return nil
}
*b = *bigIntStr
return nil
}
// Value implements valuer for database/sql
func (b BigIntStr) Value() (driver.Value, error) {
// string to *big.Int
bigInt, ok := new(big.Int).SetString(string(b), 10)
if !ok || bigInt == nil {
return nil, tracerr.Wrap(errors.New("invalid representation of a *big.Int"))
}
// *big.Int to bytes
return bigInt.Bytes(), nil
}
// StrBigInt is used to unmarshal BigIntStr directly into an alias of big.Int
type StrBigInt big.Int

+ 3
- 3
db/historydb/historydb.go

@ -693,11 +693,11 @@ func (hdb *HistoryDB) GetAllExits() ([]common.ExitInfo, error) {
func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) {
var txs []*common.L1Tx
err := meddler.QueryAll(
hdb.dbRead, &txs, // Note that '\x' gets parsed as a big.Int with value = 0
hdb.dbRead, &txs,
`SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE '\x' END) AS effective_amount,
tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE '\x' END) AS effective_deposit_amount,
tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE 0 END) AS effective_amount,
tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE 0 END) AS effective_deposit_amount,
tx.eth_block_num, tx.type, tx.batch_num
FROM tx WHERE is_l1 = TRUE AND user_origin = TRUE ORDER BY item_id;`,
)

+ 18
- 12
db/migrations/0001.sql

@ -1,5 +1,11 @@
-- +migrate Up
-- NOTE: We use "DECIMAL(78,0)" to encode go *big.Int types. All the *big.Int
-- that we deal with represent a value in the SNARK field, which is an integer
-- of 256 bits. `log(2**256, 10) = 77.06`: that is, a 256 bit number can have
-- at most 78 digits, so we use this value to specify the precision in the
-- PostgreSQL DECIMAL guaranteeing that we will never lose precision.
-- History
CREATE TABLE block (
eth_block_num BIGINT PRIMARY KEY,
@ -22,10 +28,10 @@ CREATE TABLE batch (
forger_addr BYTEA NOT NULL, -- fake foreign key for coordinator
fees_collected BYTEA NOT NULL,
fee_idxs_coordinator BYTEA NOT NULL,
state_root BYTEA NOT NULL,
state_root DECIMAL(78,0) NOT NULL,
num_accounts BIGINT NOT NULL,
last_idx BIGINT NOT NULL,
exit_root BYTEA NOT NULL,
exit_root DECIMAL(78,0) NOT NULL,
forge_l1_txs_num BIGINT,
slot_num BIGINT NOT NULL,
total_fees_usd NUMERIC
@ -34,7 +40,7 @@ CREATE TABLE batch (
CREATE TABLE bid (
item_id SERIAL PRIMARY KEY,
slot_num BIGINT NOT NULL,
bid_value BYTEA NOT NULL,
bid_value DECIMAL(78,0) NOT NULL,
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
bidder_addr BYTEA NOT NULL -- fake foreign key for coordinator
);
@ -106,7 +112,7 @@ CREATE TABLE account_update (
batch_num BIGINT NOT NULL REFERENCES batch (batch_num) ON DELETE CASCADE,
idx BIGINT NOT NULL REFERENCES account (idx) ON DELETE CASCADE,
nonce BIGINT NOT NULL,
balance BYTEA NOT NULL
balance DECIMAL(78,0) NOT NULL
);
CREATE TABLE exit_tree (
@ -114,7 +120,7 @@ CREATE TABLE exit_tree (
batch_num BIGINT REFERENCES batch (batch_num) ON DELETE CASCADE,
account_idx BIGINT REFERENCES account (idx) ON DELETE CASCADE,
merkle_proof BYTEA NOT NULL,
balance BYTEA NOT NULL,
balance DECIMAL(78,0) NOT NULL,
instant_withdrawn BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL,
delayed_withdraw_request BIGINT REFERENCES block (eth_block_num) ON DELETE SET NULL,
owner BYTEA,
@ -164,7 +170,7 @@ CREATE TABLE tx (
to_idx BIGINT NOT NULL,
to_eth_addr BYTEA,
to_bjj BYTEA,
amount BYTEA NOT NULL,
amount DECIMAL(78,0) NOT NULL,
amount_success BOOLEAN NOT NULL DEFAULT true,
amount_f NUMERIC NOT NULL,
token_id INT NOT NULL REFERENCES token (token_id),
@ -174,7 +180,7 @@ CREATE TABLE tx (
-- L1
to_forge_l1_txs_num BIGINT,
user_origin BOOLEAN,
deposit_amount BYTEA,
deposit_amount DECIMAL(78,0),
deposit_amount_success BOOLEAN NOT NULL DEFAULT true,
deposit_amount_f NUMERIC,
deposit_amount_usd NUMERIC,
@ -544,7 +550,7 @@ FOR EACH ROW EXECUTE PROCEDURE forge_l1_user_txs();
CREATE TABLE rollup_vars (
eth_block_num BIGINT PRIMARY KEY REFERENCES block (eth_block_num) ON DELETE CASCADE,
fee_add_token BYTEA NOT NULL,
fee_add_token DECIMAL(78,0) NOT NULL,
forge_l1_timeout BIGINT NOT NULL,
withdrawal_delay BIGINT NOT NULL,
buckets BYTEA NOT NULL,
@ -556,7 +562,7 @@ CREATE TABLE bucket_update (
eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
num_bucket BIGINT NOT NULL,
block_stamp BIGINT NOT NULL,
withdrawals BYTEA NOT NULL
withdrawals DECIMAL(78,0) NOT NULL
);
CREATE TABLE token_exchange (
@ -572,7 +578,7 @@ CREATE TABLE escape_hatch_withdrawal (
who_addr BYTEA NOT NULL,
to_addr BYTEA NOT NULL,
token_addr BYTEA NOT NULL,
amount BYTEA NOT NULL
amount DECIMAL(78,0) NOT NULL
);
CREATE TABLE auction_vars (
@ -610,7 +616,7 @@ CREATE TABLE tx_pool (
effective_to_eth_addr BYTEA,
effective_to_bjj BYTEA,
token_id INT NOT NULL REFERENCES token (token_id) ON DELETE CASCADE,
amount BYTEA NOT NULL,
amount DECIMAL(78,0) NOT NULL,
amount_f NUMERIC NOT NULL,
fee SMALLINT NOT NULL,
nonce BIGINT NOT NULL,
@ -624,7 +630,7 @@ CREATE TABLE tx_pool (
rq_to_eth_addr BYTEA,
rq_to_bjj BYTEA,
rq_token_id INT,
rq_amount BYTEA,
rq_amount DECIMAL(78,0),
rq_fee SMALLINT,
rq_nonce BIGINT,
tx_type VARCHAR(40) NOT NULL,

+ 16
- 4
db/utils.go

@ -13,6 +13,9 @@ import (
"github.com/hermeznetwork/hermez-node/log"
"github.com/hermeznetwork/tracerr"
"github.com/jmoiron/sqlx"
//nolint:errcheck // driver for postgres DB
_ "github.com/lib/pq"
migrate "github.com/rubenv/sql-migrate"
"github.com/russross/meddler"
"golang.org/x/sync/semaphore"
@ -165,7 +168,11 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer"))
}
field := fieldPtr.(**big.Int)
*field = new(big.Int).SetBytes([]byte(*ptr))
var ok bool
*field, ok = new(big.Int).SetString(*ptr, 10)
if !ok {
return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", *ptr))
}
return nil
}
@ -173,7 +180,7 @@ func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
func (b BigIntMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) {
field := fieldPtr.(*big.Int)
return field.Bytes(), nil
return field.String(), nil
}
// BigIntNullMeddler encodes or decodes the field value to or from JSON
@ -198,7 +205,12 @@ func (b BigIntNullMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
if ptr == nil {
return tracerr.Wrap(fmt.Errorf("BigIntMeddler.PostRead: nil pointer"))
}
*field = new(big.Int).SetBytes(ptr)
var ok bool
*field, ok = new(big.Int).SetString(string(ptr), 10)
if !ok {
return tracerr.Wrap(fmt.Errorf("big.Int.SetString failed on \"%v\"", string(ptr)))
}
return nil
}
@ -208,7 +220,7 @@ func (b BigIntNullMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}
if field == nil {
return nil, nil
}
return field.Bytes(), nil
return field.String(), nil
}
// SliceToSlicePtrs converts any []Foo to []*Foo

+ 43
- 0
db/utils_test.go

@ -1,9 +1,13 @@
package db
import (
"math/big"
"os"
"testing"
"github.com/russross/meddler"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type foo struct {
@ -33,3 +37,42 @@ func TestSlicePtrsToSlice(t *testing.T) {
assert.Equal(t, *a[i], b[i])
}
}
func TestBigInt(t *testing.T) {
pass := os.Getenv("POSTGRES_PASS")
db, err := InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
require.NoError(t, err)
defer func() {
_, err := db.Exec("DROP TABLE IF EXISTS test_big_int;")
require.NoError(t, err)
err = db.Close()
require.NoError(t, err)
}()
_, err = db.Exec("DROP TABLE IF EXISTS test_big_int;")
require.NoError(t, err)
_, err = db.Exec(`CREATE TABLE test_big_int (
item_id SERIAL PRIMARY KEY,
value1 DECIMAL(78, 0) NOT NULL,
value2 DECIMAL(78, 0),
value3 DECIMAL(78, 0)
);`)
require.NoError(t, err)
type Entry struct {
ItemID int `meddler:"item_id"`
Value1 *big.Int `meddler:"value1,bigint"`
Value2 *big.Int `meddler:"value2,bigintnull"`
Value3 *big.Int `meddler:"value3,bigintnull"`
}
entry := Entry{ItemID: 1, Value1: big.NewInt(1234567890), Value2: big.NewInt(9876543210), Value3: nil}
err = meddler.Insert(db, "test_big_int", &entry)
require.NoError(t, err)
var dbEntry Entry
err = meddler.QueryRow(db, &dbEntry, "SELECT * FROM test_big_int WHERE item_id = 1;")
require.NoError(t, err)
assert.Equal(t, entry, dbEntry)
}

Loading…
Cancel
Save