From 94e1f11a98c43da539810d786b605a569d0c2080 Mon Sep 17 00:00:00 2001 From: Arnau B Date: Wed, 16 Sep 2020 15:19:33 +0200 Subject: [PATCH] Implement L2DB --- common/accountcreationauths.go | 8 +- common/pooll2tx.go | 30 +-- common/token.go | 2 +- config/config.go | 9 +- db/historydb/historydb.go | 2 +- db/historydb/historydb_test.go | 2 +- db/l2db/l2db.go | 205 ++++++++++++++----- db/l2db/l2db_test.go | 347 +++++++++++++++++++++++++------- db/l2db/migrations/001_init.sql | 12 +- go.sum | 1 + test/l2db.go | 85 +++++++- 11 files changed, 553 insertions(+), 150 deletions(-) diff --git a/common/accountcreationauths.go b/common/accountcreationauths.go index 1ba996b..6d2c8e6 100644 --- a/common/accountcreationauths.go +++ b/common/accountcreationauths.go @@ -9,8 +9,8 @@ import ( // AccountCreationAuth authorizations sent by users to the L2DB, to be used for account creations when necessary type AccountCreationAuth struct { - Timestamp time.Time - EthAddr ethCommon.Address - BJJ *babyjub.PublicKey - Signature []byte + EthAddr ethCommon.Address `meddler:"eth_addr"` + BJJ *babyjub.PublicKey `meddler:"bjj"` + Signature []byte `meddler:"signature"` + Timestamp time.Time `meddler:"timestamp,utctime"` } diff --git a/common/pooll2tx.go b/common/pooll2tx.go index 04844b2..4ec1ca8 100644 --- a/common/pooll2tx.go +++ b/common/pooll2tx.go @@ -43,18 +43,20 @@ func NonceFromBytes(b [5]byte) Nonce { // PoolL2Tx is a struct that represents a L2Tx sent by an account to the coordinator hat is waiting to be forged type PoolL2Tx struct { // Stored in DB: mandatory fileds - TxID TxID `meddler:"tx_id"` - FromIdx Idx `meddler:"from_idx"` // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit) - ToIdx Idx `meddler:"to_idx"` // ToIdx is ignored in L1Tx/Deposit, but used in the L1Tx/DepositAndTransfer - ToEthAddr ethCommon.Address `meddler:"to_eth_addr"` - ToBJJ *babyjub.PublicKey `meddler:"to_bjj"` // TODO: stop using json, use scanner/valuer - TokenID TokenID `meddler:"token_id"` - Amount *big.Int `meddler:"amount,bigint"` // TODO: change to float16 - Fee FeeSelector `meddler:"fee"` - Nonce Nonce `meddler:"nonce"` // effective 40 bits used - State PoolL2TxState `meddler:"state"` - Signature *babyjub.Signature `meddler:"signature"` // tx signature - Timestamp time.Time `meddler:"timestamp,utctime"` // time when added to the tx pool + TxID TxID `meddler:"tx_id"` + FromIdx Idx `meddler:"from_idx"` // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit) + ToIdx Idx `meddler:"to_idx"` // ToIdx is ignored in L1Tx/Deposit, but used in the L1Tx/DepositAndTransfer + ToEthAddr ethCommon.Address `meddler:"to_eth_addr"` + ToBJJ *babyjub.PublicKey `meddler:"to_bjj"` // TODO: stop using json, use scanner/valuer + TokenID TokenID `meddler:"token_id"` + Amount *big.Int `meddler:"amount,bigint"` // TODO: change to float16 + AmountFloat float64 `meddler:"amount_f"` // TODO: change to float16 + USD float64 `meddler:"value_usd"` // TODO: change to float16 + Fee FeeSelector `meddler:"fee"` + Nonce Nonce `meddler:"nonce"` // effective 40 bits used + State PoolL2TxState `meddler:"state"` + Signature *babyjub.Signature `meddler:"signature"` // tx signature + Timestamp time.Time `meddler:"timestamp,utctime"` // time when added to the tx pool // Stored in DB: optional fileds, may be uninitialized BatchNum BatchNum `meddler:"batch_num,zeroisnull"` // batchNum in which this tx was forged. Presence indicates "forged" state. RqFromIdx Idx `meddler:"rq_from_idx,zeroisnull"` // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit) @@ -65,8 +67,8 @@ type PoolL2Tx struct { RqAmount *big.Int `meddler:"rq_amount,bigintnull"` // TODO: change to float16 RqFee FeeSelector `meddler:"rq_fee,zeroisnull"` RqNonce uint64 `meddler:"rq_nonce,zeroisnull"` // effective 48 bits used - AbsoluteFee float64 `meddler:"absolute_fee,zeroisnull"` - AbsoluteFeeUpdate time.Time `meddler:"absolute_fee_update,utctimez"` + AbsoluteFee float64 `meddler:"fee_usd,zeroisnull"` + AbsoluteFeeUpdate time.Time `meddler:"usd_update,utctimez"` Type TxType `meddler:"tx_type"` // Extra metadata, may be uninitialized RqTxCompressedData []byte `meddler:"-"` // 253 bits, optional for atomic txs diff --git a/common/token.go b/common/token.go index 4a473ca..58e5279 100644 --- a/common/token.go +++ b/common/token.go @@ -16,7 +16,7 @@ type Token struct { Name string `meddler:"name"` Symbol string `meddler:"symbol"` Decimals uint64 `meddler:"decimals"` - USD float32 `meddler:"usd,zeroisnull"` + USD float64 `meddler:"usd,zeroisnull"` USDUpdate time.Time `meddler:"usd_update,utctimez"` } diff --git a/config/config.go b/config/config.go index 1dd3711..d9812dd 100644 --- a/config/config.go +++ b/config/config.go @@ -7,6 +7,7 @@ import ( "github.com/BurntSushi/toml" ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/common" "gopkg.in/go-playground/validator.v9" ) @@ -35,10 +36,10 @@ type Coordinator struct { ForgerAddress ethCommon.Address `validate:"required"` ForgeLoopInterval Duration `validate:"required"` L2DB struct { - Name string `validate:"required"` - SafetyPeriod uint16 `validate:"required"` - MaxTxs uint32 `validate:"required"` - TTL Duration `validate:"required"` + Name string `validate:"required"` + SafetyPeriod common.BatchNum `validate:"required"` + MaxTxs uint32 `validate:"required"` + TTL Duration `validate:"required"` } `validate:"required"` TxSelector struct { Path string `validate:"required"` diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index 43ea6df..73eea17 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -213,7 +213,7 @@ func (hdb *HistoryDB) AddTokens(tokens []common.Token) error { } // UpdateTokenValue updates the USD value of a token -func (hdb *HistoryDB) UpdateTokenValue(tokenID common.TokenID, value float32) error { +func (hdb *HistoryDB) UpdateTokenValue(tokenID common.TokenID, value float64) error { _, err := hdb.db.Exec( "UPDATE token SET usd = $1 WHERE token_id = $2;", value, tokenID, diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index 1c4a3ae..3a4c3f7 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -143,7 +143,7 @@ func TestTokens(t *testing.T) { // Update price of generated tokens without price for i := 0; i < len(tokens); i++ { if tokens[i].USD == 0 { - value := 3.33 + float32(i) + value := 3.33 + float64(i) tokens[i].USD = value err := historyDB.UpdateTokenValue(tokens[i].TokenID, value) assert.NoError(t, err) diff --git a/db/l2db/l2db.go b/db/l2db/l2db.go index e0b961d..6af711c 100644 --- a/db/l2db/l2db.go +++ b/db/l2db/l2db.go @@ -2,10 +2,9 @@ package l2db import ( "fmt" - "strconv" "time" - eth "github.com/ethereum/go-ethereum/common" + ethCommon "github.com/ethereum/go-ethereum/common" "github.com/gobuffalo/packr/v2" "github.com/hermeznetwork/hermez-node/common" "github.com/hermeznetwork/hermez-node/db" @@ -21,21 +20,17 @@ import ( // due to them being forged or invalid after a safety period type L2DB struct { db *sqlx.DB - safetyPeriod uint16 + safetyPeriod common.BatchNum ttl time.Duration maxTxs uint32 } // NewL2DB creates a L2DB. -// More info on how to set dbDialect and dbArgs here: http://gorm.io/docs/connecting_to_the_database.html -// safetyPeriod is the amount of blockchain blocks that must be waited before deleting anything (to avoid reorg problems). -// maxTxs indicates the desired maximum amount of txs stored on the L2DB. -// TTL indicates the maximum amount of time that a tx can be in the L2DB -// (to prevent tx that won't ever be forged to stay there, will be used if maxTxs is exceeded). -// autoPurgePeriod will be used as delay between calls to Purge. If the value is 0, it will be disabled. +// To create it, it's needed postgres configuration, safety period expressed in batches, +// maxTxs that the DB should have and TTL (time to live) for pending txs. func NewL2DB( port int, host, user, password, dbname string, - safetyPeriod uint16, + safetyPeriod common.BatchNum, maxTxs uint32, TTL time.Duration, ) (*L2DB, error) { @@ -71,17 +66,26 @@ func (l2db *L2DB) DB() *sqlx.DB { return l2db.db } +// AddAccountCreationAuth inserts an account creation authorization into the DB +func (l2db *L2DB) AddAccountCreationAuth(auth *common.AccountCreationAuth) error { + return meddler.Insert(l2db.db, "account_creation_auth", auth) +} + +// GetAccountCreationAuth returns an account creation authorization into the DB +func (l2db *L2DB) GetAccountCreationAuth(addr ethCommon.Address) (*common.AccountCreationAuth, error) { + auth := new(common.AccountCreationAuth) + return auth, meddler.QueryRow( + l2db.db, auth, + "SELECT * FROM account_creation_auth WHERE eth_addr = $1;", + addr, + ) +} + // AddTx inserts a tx into the L2DB func (l2db *L2DB) AddTx(tx *common.PoolL2Tx) error { return meddler.Insert(l2db.db, "tx_pool", tx) } -// AddAccountCreationAuth inserts an account creation authorization into the DB -func (l2db *L2DB) AddAccountCreationAuth(auth *common.AccountCreationAuth) error { - // TODO: impl - return nil -} - // GetTx return the specified Tx func (l2db *L2DB) GetTx(txID common.TxID) (*common.PoolL2Tx, error) { tx := new(common.PoolL2Tx) @@ -103,12 +107,6 @@ func (l2db *L2DB) GetPendingTxs() ([]*common.PoolL2Tx, error) { return txs, err } -// GetAccountCreationAuth return the authorization to make registers of an Ethereum address -func (l2db *L2DB) GetAccountCreationAuth(ethAddr eth.Address) (*common.AccountCreationAuth, error) { - // TODO: impl - return nil, nil -} - // StartForging updates the state of the transactions that will begin the forging process. // The state of the txs referenced by txIDs will be changed from Pending -> Forging func (l2db *L2DB) StartForging(txIDs []common.TxID, batchNum common.BatchNum) error { @@ -116,9 +114,9 @@ func (l2db *L2DB) StartForging(txIDs []common.TxID, batchNum common.BatchNum) er `UPDATE tx_pool SET state = ?, batch_num = ? WHERE state = ? AND tx_id IN (?);`, - string(common.PoolL2TxStateForging), - strconv.Itoa(int(batchNum)), - string(common.PoolL2TxStatePending), + common.PoolL2TxStateForging, + batchNum, + common.PoolL2TxStatePending, txIDs, ) if err != nil { @@ -131,48 +129,157 @@ func (l2db *L2DB) StartForging(txIDs []common.TxID, batchNum common.BatchNum) er // DoneForging updates the state of the transactions that have been forged // so the state of the txs referenced by txIDs will be changed from Forging -> Forged -func (l2db *L2DB) DoneForging(txIDs []common.TxID) error { - // TODO: impl - return nil +func (l2db *L2DB) DoneForging(txIDs []common.TxID, batchNum common.BatchNum) error { + query, args, err := sqlx.In( + `UPDATE tx_pool + SET state = ?, batch_num = ? + WHERE state = ? AND tx_id IN (?);`, + common.PoolL2TxStateForged, + batchNum, + common.PoolL2TxStateForging, + txIDs, + ) + if err != nil { + return err + } + query = l2db.db.Rebind(query) + _, err = l2db.db.Exec(query, args...) + return err } // InvalidateTxs updates the state of the transactions that are invalid. // The state of the txs referenced by txIDs will be changed from * -> Invalid -func (l2db *L2DB) InvalidateTxs(txIDs []common.TxID) error { - return nil +func (l2db *L2DB) InvalidateTxs(txIDs []common.TxID, batchNum common.BatchNum) error { + query, args, err := sqlx.In( + `UPDATE tx_pool + SET state = ?, batch_num = ? + WHERE tx_id IN (?);`, + common.PoolL2TxStateInvalid, + batchNum, + txIDs, + ) + if err != nil { + return err + } + query = l2db.db.Rebind(query) + _, err = l2db.db.Exec(query, args...) + return err } -// CheckNonces invalidate txs with nonces that are smaller than their respective accounts nonces. +// CheckNonces invalidate txs with nonces that are smaller or equal than their respective accounts nonces. // The state of the affected txs will be changed from Pending -> Invalid -func (l2db *L2DB) CheckNonces(updatedAccounts []common.Account) error { - // TODO: impl - return nil -} - -// GetTxsByAbsoluteFeeUpdate return the txs that have an AbsoluteFee updated before olderThan -func (l2db *L2DB) GetTxsByAbsoluteFeeUpdate(olderThan time.Time) ([]*common.PoolL2Tx, error) { - // TODO: impl - return nil, nil +func (l2db *L2DB) CheckNonces(updatedAccounts []common.Account, batchNum common.BatchNum) error { + txn, err := l2db.db.Begin() + if err != nil { + return err + } + defer func() { + // Rollback the transaction if there was an error. + if err != nil { + err = txn.Rollback() + } + }() + for i := 0; i < len(updatedAccounts); i++ { + _, err = txn.Exec( + `UPDATE tx_pool + SET state = $1, batch_num = $2 + WHERE state = $3 AND from_idx = $4 AND nonce <= $5;`, + common.PoolL2TxStateInvalid, + batchNum, + common.PoolL2TxStatePending, + updatedAccounts[i].Idx, + updatedAccounts[i].Nonce, + ) + if err != nil { + return err + } + } + return txn.Commit() } -// UpdateTxs update existing txs from the pool (TxID must exist) -func (l2db *L2DB) UpdateTxs(txs []*common.PoolL2Tx) error { - // TODO: impl - return nil +// UpdateTxValue updates the absolute fee and value of txs given a token list that include their price in USD +func (l2db *L2DB) UpdateTxValue(tokens []common.Token) error { + // WARNING: this is very slow and should be optimized + txn, err := l2db.db.Begin() + if err != nil { + return err + } + defer func() { + // Rollback the transaction if there was an error. + if err != nil { + err = txn.Rollback() + } + }() + now := time.Now() + for i := 0; i < len(tokens); i++ { + _, err = txn.Exec( + `UPDATE tx_pool + SET usd_update = $1, value_usd = amount_f * $2, fee_usd = $2 * amount_f * CASE + WHEN fee = 0 THEN 0 + WHEN fee >= 1 AND fee <= 32 THEN POWER(10,-24+(fee::float/2)) + WHEN fee >= 33 AND fee <= 223 THEN POWER(10,-8+(0.041666666666667*(fee::float-32))) + WHEN fee >= 224 AND fee <= 255 THEN POWER(10,fee-224) END + WHERE token_id = $3;`, + now, + tokens[i].USD, + tokens[i].TokenID, + ) + if err != nil { + return err + } + } + return txn.Commit() } // Reorg updates the state of txs that were updated in a batch that has been discarted due to a blockchain reorg. // The state of the affected txs can change form Forged -> Pending or from Invalid -> Pending func (l2db *L2DB) Reorg(lastValidBatch common.BatchNum) error { - // TODO: impl - return nil + _, err := l2db.db.Exec( + `UPDATE tx_pool SET batch_num = NULL, state = $1 + WHERE (state = $2 OR state = $3) AND batch_num > $4`, + common.PoolL2TxStatePending, + common.PoolL2TxStateForged, + common.PoolL2TxStateInvalid, + lastValidBatch, + ) + return err } // Purge deletes transactions that have been forged or marked as invalid for longer than the safety period // it also deletes txs that has been in the L2DB for longer than the ttl if maxTxs has been exceeded -func (l2db *L2DB) Purge() error { - // TODO: impl - return nil +func (l2db *L2DB) Purge(currentBatchNum common.BatchNum) error { + txn, err := l2db.db.Begin() + if err != nil { + return err + } + defer func() { + // Rollback the transaction if there was an error. + if err != nil { + err = txn.Rollback() + } + }() + // Delete pending txs that have been in the pool after the TTL if maxTxs is reached + now := time.Now().UTC().Unix() + _, err = txn.Exec( + `DELETE FROM tx_pool WHERE (SELECT count(*) FROM tx_pool) > $1 AND timestamp < $2`, + l2db.maxTxs, + time.Unix(now-int64(l2db.ttl.Seconds()), 0), + ) + if err != nil { + return err + } + // Delete txs that have been marked as forged / invalid after the safety period + _, err = txn.Exec( + `DELETE FROM tx_pool + WHERE batch_num < $1 AND (state = $2 OR state = $3)`, + currentBatchNum-l2db.safetyPeriod, + common.PoolL2TxStateForged, + common.PoolL2TxStateInvalid, + ) + if err != nil { + return err + } + return txn.Commit() } // Close frees the resources used by the L2DB diff --git a/db/l2db/l2db_test.go b/db/l2db/l2db_test.go index 6ae58e9..9df0b41 100644 --- a/db/l2db/l2db_test.go +++ b/db/l2db/l2db_test.go @@ -2,32 +2,24 @@ package l2db import ( "fmt" - "math/big" + "math" "os" - "strconv" "testing" "time" - eth "github.com/ethereum/go-ethereum/common" "github.com/hermeznetwork/hermez-node/common" - "github.com/iden3/go-iden3-crypto/babyjub" + "github.com/hermeznetwork/hermez-node/log" + "github.com/hermeznetwork/hermez-node/test" "github.com/stretchr/testify/assert" ) var l2DB *L2DB -// In order to run the test you need to run a Posgres DB with -// a database named "l2" that is accessible by -// user: "hermez" -// pass: set it using the env var POSTGRES_PASS -// This can be achieved by running: POSTGRES_PASS=your_strong_pass && sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=history -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD=$POSTGRES_PASS -d postgres && sleep 2s && sudo docker exec -it hermez-db-test psql -a history -U hermez -c "CREATE DATABASE l2;" -// After running the test you can stop the container by running: sudo docker kill hermez-ydb-test -// If you already did that for the HistoryDB you don't have to do it again func TestMain(m *testing.M) { // init DB var err error pass := os.Getenv("POSTGRES_PASS") - l2DB, err = NewL2DB(5432, "localhost", "hermez", pass, "l2", 10, 512, 24*time.Hour) + l2DB, err = NewL2DB(5432, "localhost", "hermez", pass, "l2", 10, 100, 24*time.Hour) if err != nil { panic(err) } @@ -43,7 +35,7 @@ func TestMain(m *testing.M) { func TestAddTx(t *testing.T) { const nInserts = 20 cleanDB() - txs := genTxs(nInserts) + txs := test.GenPoolTxs(nInserts) for _, tx := range txs { err := l2DB.AddTx(tx) assert.NoError(t, err) @@ -61,7 +53,7 @@ func TestAddTx(t *testing.T) { func BenchmarkAddTx(b *testing.B) { const nInserts = 20 cleanDB() - txs := genTxs(nInserts) + txs := test.GenPoolTxs(nInserts) now := time.Now() for _, tx := range txs { _ = l2DB.AddTx(tx) @@ -73,7 +65,7 @@ func BenchmarkAddTx(b *testing.B) { func TestGetPending(t *testing.T) { const nInserts = 20 cleanDB() - txs := genTxs(nInserts) + txs := test.GenPoolTxs(nInserts) var pendingTxs []*common.PoolL2Tx for _, tx := range txs { err := l2DB.AddTx(tx) @@ -95,81 +87,296 @@ func TestGetPending(t *testing.T) { } func TestStartForging(t *testing.T) { - const nInserts = 24 - const fakeBlockNum = 33 + // Generate txs + const nInserts = 60 + const fakeBatchNum common.BatchNum = 33 cleanDB() - txs := genTxs(nInserts) - var startForgingTxs []*common.PoolL2Tx + txs := test.GenPoolTxs(nInserts) var startForgingTxIDs []common.TxID randomizer := 0 + // Add txs to DB for _, tx := range txs { err := l2DB.AddTx(tx) assert.NoError(t, err) if tx.State == common.PoolL2TxStatePending && randomizer%2 == 0 { randomizer++ - startForgingTxs = append(startForgingTxs, tx) startForgingTxIDs = append(startForgingTxIDs, tx.TxID) } - if tx.State == common.PoolL2TxStateForging { - startForgingTxs = append(startForgingTxs, tx) + } + // Start forging txs + err := l2DB.StartForging(startForgingTxIDs, fakeBatchNum) + assert.NoError(t, err) + // Fetch txs and check that they've been updated correctly + for _, id := range startForgingTxIDs { + fetchedTx, err := l2DB.GetTx(id) + assert.NoError(t, err) + assert.Equal(t, common.PoolL2TxStateForging, fetchedTx.State) + assert.Equal(t, fakeBatchNum, fetchedTx.BatchNum) + } +} + +func TestDoneForging(t *testing.T) { + // Generate txs + const nInserts = 60 + const fakeBatchNum common.BatchNum = 33 + cleanDB() + txs := test.GenPoolTxs(nInserts) + var doneForgingTxIDs []common.TxID + randomizer := 0 + // Add txs to DB + for _, tx := range txs { + err := l2DB.AddTx(tx) + assert.NoError(t, err) + if tx.State == common.PoolL2TxStateForging && randomizer%2 == 0 { + randomizer++ + doneForgingTxIDs = append(doneForgingTxIDs, tx.TxID) } } - fmt.Println(startForgingTxs) // TODO added print here to avoid lint complaining about startForgingTxs not being used - err := l2DB.StartForging(startForgingTxIDs, fakeBlockNum) + // Start forging txs + err := l2DB.DoneForging(doneForgingTxIDs, fakeBatchNum) assert.NoError(t, err) - // TODO: Fetch txs and check that they've been updated correctly + // Fetch txs and check that they've been updated correctly + for _, id := range doneForgingTxIDs { + fetchedTx, err := l2DB.GetTx(id) + assert.NoError(t, err) + assert.Equal(t, common.PoolL2TxStateForged, fetchedTx.State) + assert.Equal(t, fakeBatchNum, fetchedTx.BatchNum) + } } -func genTxs(n int) []*common.PoolL2Tx { - // WARNING: This tx doesn't follow the protocol (signature, txID, ...) - // it's just to test getting/setting from/to the DB. - // Type and RqTxCompressedData: not initialized because it's not stored - // on the DB and add noise when checking results. - txs := make([]*common.PoolL2Tx, 0, n) - privK := babyjub.NewRandPrivKey() - for i := 0; i < n; i++ { - var state common.PoolL2TxState - if i%4 == 0 { - state = common.PoolL2TxStatePending - } else if i%4 == 1 { - state = common.PoolL2TxStateInvalid - } else if i%4 == 2 { - state = common.PoolL2TxStateForging - } else if i%4 == 3 { - state = common.PoolL2TxStateForged +func TestInvalidate(t *testing.T) { + // Generate txs + const nInserts = 60 + const fakeBatchNum common.BatchNum = 33 + cleanDB() + txs := test.GenPoolTxs(nInserts) + var invalidTxIDs []common.TxID + randomizer := 0 + // Add txs to DB + for _, tx := range txs { + err := l2DB.AddTx(tx) + assert.NoError(t, err) + if tx.State != common.PoolL2TxStateInvalid && randomizer%2 == 0 { + randomizer++ + invalidTxIDs = append(invalidTxIDs, tx.TxID) } - tx := &common.PoolL2Tx{ - TxID: common.TxID(common.Hash([]byte(strconv.Itoa(i)))), - FromIdx: 47, - ToIdx: 96, - ToEthAddr: eth.BigToAddress(big.NewInt(234523534)), - ToBJJ: privK.Public(), - TokenID: 73, - Amount: big.NewInt(3487762374627846747), - Fee: 99, - Nonce: 28, - State: state, - Signature: privK.SignPoseidon(big.NewInt(674238462)), - Timestamp: time.Now().UTC(), + } + // Start forging txs + err := l2DB.InvalidateTxs(invalidTxIDs, fakeBatchNum) + assert.NoError(t, err) + // Fetch txs and check that they've been updated correctly + for _, id := range invalidTxIDs { + fetchedTx, err := l2DB.GetTx(id) + assert.NoError(t, err) + assert.Equal(t, common.PoolL2TxStateInvalid, fetchedTx.State) + assert.Equal(t, fakeBatchNum, fetchedTx.BatchNum) + } +} + +func TestCheckNonces(t *testing.T) { + // Generate txs + const nInserts = 60 + const fakeBatchNum common.BatchNum = 33 + cleanDB() + txs := test.GenPoolTxs(nInserts) + var invalidTxIDs []common.TxID + // Generate accounts + const nAccoutns = 2 + const currentNonce = 2 + accs := []common.Account{} + for i := 0; i < nAccoutns; i++ { + accs = append(accs, common.Account{ + Idx: common.Idx(i), + Nonce: currentNonce, + }) + } + // Add txs to DB + for i := 0; i < len(txs); i++ { + if txs[i].State != common.PoolL2TxStateInvalid { + if i%2 == 0 { // Ensure transaction will be marked as invalid due to old nonce + txs[i].Nonce = accs[i%len(accs)].Nonce + txs[i].FromIdx = accs[i%len(accs)].Idx + invalidTxIDs = append(invalidTxIDs, txs[i].TxID) + } else { // Ensure transaction will NOT be marked as invalid due to old nonce + txs[i].Nonce = currentNonce + 1 + } } - if i%2 == 0 { // Optional parameters: rq - tx.RqFromIdx = 893 - tx.RqToIdx = 334 - tx.RqToEthAddr = eth.BigToAddress(big.NewInt(239457111187)) - tx.RqToBJJ = privK.Public() - tx.RqTokenID = 222 - tx.RqAmount = big.NewInt(3487762374627846747) - tx.RqFee = 11 - tx.RqNonce = 78 + err := l2DB.AddTx(txs[i]) + assert.NoError(t, err) + } + // Start forging txs + err := l2DB.InvalidateTxs(invalidTxIDs, fakeBatchNum) + assert.NoError(t, err) + // Fetch txs and check that they've been updated correctly + for _, id := range invalidTxIDs { + fetchedTx, err := l2DB.GetTx(id) + assert.NoError(t, err) + assert.Equal(t, common.PoolL2TxStateInvalid, fetchedTx.State) + assert.Equal(t, fakeBatchNum, fetchedTx.BatchNum) + } +} + +func TestUpdateTxValue(t *testing.T) { + // Generate txs + const nInserts = 255 // Force all possible fee selector values + cleanDB() + txs := test.GenPoolTxs(nInserts) + // Generate tokens + const nTokens = 2 + tokens := []common.Token{} + for i := 0; i < nTokens; i++ { + tokens = append(tokens, common.Token{ + TokenID: common.TokenID(i), + USD: float64(i) * 1.3, + }) + } + // Add txs to DB + for i := 0; i < len(txs); i++ { + // Set Token + token := tokens[i%len(tokens)] + txs[i].TokenID = token.TokenID + // Insert to DB + err := l2DB.AddTx(txs[i]) + assert.NoError(t, err) + // Set USD values (for comparing results when fetching from DB) + txs[i].USD = txs[i].AmountFloat * token.USD + if txs[i].Fee == 0 { + txs[i].AbsoluteFee = 0 + } else if txs[i].Fee <= 32 { + txs[i].AbsoluteFee = txs[i].USD * math.Pow(10, -24+(float64(txs[i].Fee)/2)) + } else if txs[i].Fee <= 223 { + txs[i].AbsoluteFee = txs[i].USD * math.Pow(10, -8+(0.041666666666667*(float64(txs[i].Fee)-32))) + } else { + txs[i].AbsoluteFee = txs[i].USD * math.Pow(10, float64(txs[i].Fee)-224) } - if i%3 == 0 { // Optional parameters: things that get updated "a posteriori" - tx.BatchNum = 489 - tx.AbsoluteFee = 39.12345 - tx.AbsoluteFeeUpdate = time.Now().UTC() + } + // Update token value + err := l2DB.UpdateTxValue(tokens) + assert.NoError(t, err) + // Fetch txs and check that they've been updated correctly + for _, tx := range txs { + fetchedTx, err := l2DB.GetTx(tx.TxID) + assert.NoError(t, err) + if fetchedTx.USD > tx.USD { + assert.Less(t, 0.999, tx.USD/fetchedTx.USD) + } else if fetchedTx.USD < tx.USD { + assert.Less(t, 0.999, fetchedTx.USD/tx.USD) } - txs = append(txs, tx) + if fetchedTx.AbsoluteFee > tx.AbsoluteFee { + assert.Less(t, 0.999, tx.AbsoluteFee/fetchedTx.AbsoluteFee) + } else if fetchedTx.AbsoluteFee < tx.AbsoluteFee { + assert.Less(t, 0.999, fetchedTx.AbsoluteFee/tx.AbsoluteFee) + } + // Time is set in the DB, so it cannot be compared exactly + assert.Greater(t, float64(15*time.Second), time.Since(fetchedTx.AbsoluteFeeUpdate).Seconds()) + } +} + +func TestReorg(t *testing.T) { + // Generate txs + const nInserts = 20 + const lastValidBatch common.BatchNum = 20 + const reorgBatch common.BatchNum = lastValidBatch + 1 + cleanDB() + txs := test.GenPoolTxs(nInserts) + // Add txs to the DB + reorgedTxIDs := []common.TxID{} + nonReorgedTxIDs := []common.TxID{} + for i := 0; i < len(txs); i++ { + if txs[i].State == common.PoolL2TxStateForged || txs[i].State == common.PoolL2TxStateInvalid { + txs[i].BatchNum = reorgBatch + reorgedTxIDs = append(reorgedTxIDs, txs[i].TxID) + } else { + txs[i].BatchNum = lastValidBatch + nonReorgedTxIDs = append(nonReorgedTxIDs, txs[i].TxID) + } + err := l2DB.AddTx(txs[i]) + assert.NoError(t, err) + } + err := l2DB.Reorg(lastValidBatch) + assert.NoError(t, err) + var nullBatchNum common.BatchNum + for _, id := range reorgedTxIDs { + tx, err := l2DB.GetTx(id) + assert.NoError(t, err) + assert.Equal(t, nullBatchNum, tx.BatchNum) + assert.Equal(t, common.PoolL2TxStatePending, tx.State) + } + for _, id := range nonReorgedTxIDs { + tx, err := l2DB.GetTx(id) + assert.NoError(t, err) + assert.Equal(t, lastValidBatch, tx.BatchNum) + } +} + +func TestPurge(t *testing.T) { + // Generate txs + nInserts := l2DB.maxTxs + 20 + cleanDB() + txs := test.GenPoolTxs(int(nInserts)) + deletedIDs := []common.TxID{} + keepedIDs := []common.TxID{} + const toDeleteBatchNum common.BatchNum = 30 + safeBatchNum := toDeleteBatchNum + l2DB.safetyPeriod + 1 + // Add txs to the DB + for i := 0; i < int(l2DB.maxTxs); i++ { + if i%1 == 0 { // keep tx + txs[i].BatchNum = safeBatchNum + keepedIDs = append(keepedIDs, txs[i].TxID) + } else if i%2 == 0 { // delete after safety period + txs[i].BatchNum = toDeleteBatchNum + if i%3 == 0 { + txs[i].State = common.PoolL2TxStateForged + } else { + txs[i].State = common.PoolL2TxStateInvalid + } + deletedIDs = append(deletedIDs, txs[i].TxID) + } + err := l2DB.AddTx(txs[i]) + assert.NoError(t, err) + } + for i := int(l2DB.maxTxs); i < len(txs); i++ { + // Delete after TTL + txs[i].Timestamp = time.Unix(time.Now().UTC().Unix()-int64(l2DB.ttl.Seconds()+float64(4*time.Second)), 0) + deletedIDs = append(deletedIDs, txs[i].TxID) + err := l2DB.AddTx(txs[i]) + assert.NoError(t, err) + } + // Purge txs + err := l2DB.Purge(safeBatchNum - 1) + assert.NoError(t, err) + // Check results + for _, id := range deletedIDs { + tx, err := l2DB.GetTx(id) + if err == nil { + log.Debug(tx) + } + assert.Error(t, err) + } + for _, id := range keepedIDs { + _, err := l2DB.GetTx(id) + assert.NoError(t, err) + } +} + +func TestAuth(t *testing.T) { + cleanDB() + const nAuths = 5 + // Generate authorizations + auths := test.GenAuths(nAuths) + for i := 0; i < len(auths); i++ { + // Add to the DB + err := l2DB.AddAccountCreationAuth(auths[i]) + assert.NoError(t, err) + // Fetch from DB + auth, err := l2DB.GetAccountCreationAuth(auths[i].EthAddr) + assert.NoError(t, err) + // Check fetched vs generated + assert.Equal(t, auths[i].EthAddr, auth.EthAddr) + assert.Equal(t, auths[i].BJJ, auth.BJJ) + assert.Equal(t, auths[i].Signature, auth.Signature) + assert.Equal(t, auths[i].Timestamp.Unix(), auths[i].Timestamp.Unix()) } - return txs } func cleanDB() { diff --git a/db/l2db/migrations/001_init.sql b/db/l2db/migrations/001_init.sql index 80ac665..34cd8e3 100644 --- a/db/l2db/migrations/001_init.sql +++ b/db/l2db/migrations/001_init.sql @@ -7,9 +7,13 @@ CREATE TABLE tx_pool ( to_bjj BYTEA NOT NULL, token_id INT NOT NULL, amount BYTEA NOT NULL, + amount_f NUMERIC NOT NULL, + value_usd NUMERIC, fee SMALLINT NOT NULL, nonce BIGINT NOT NULL, state CHAR(4) NOT NULL, + signature BYTEA NOT NULL, + timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, batch_num BIGINT, rq_from_idx BIGINT, rq_to_idx BIGINT, @@ -19,17 +23,15 @@ CREATE TABLE tx_pool ( rq_amount BYTEA, rq_fee SMALLINT, rq_nonce BIGINT, - signature BYTEA NOT NULL, - timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, - absolute_fee NUMERIC, - absolute_fee_update TIMESTAMP WITHOUT TIME ZONE, + fee_usd NUMERIC, + usd_update TIMESTAMP WITHOUT TIME ZONE, tx_type VARCHAR(40) NOT NULL ); CREATE TABLE account_creation_auth ( eth_addr BYTEA PRIMARY KEY, bjj BYTEA NOT NULL, - account_creation_auth_sig BYTEA NOT NULL, + signature BYTEA NOT NULL, timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL ); diff --git a/go.sum b/go.sum index 2a27b50..fa8099f 100644 --- a/go.sum +++ b/go.sum @@ -285,6 +285,7 @@ github.com/iden3/go-circom-prover-verifier v0.0.1/go.mod h1:1FkpX4nUXxYcY2fpzqd2 github.com/iden3/go-circom-witnesscalc v0.0.1/go.mod h1:xjT1BlFZDBioHOlbD75SmZZLC1d1AfOycqbSa/1QRJU= github.com/iden3/go-iden3-core v0.0.8 h1:PLw7iCiX7Pw1dqBkR+JaLQWqB5RKd+vgu25UBdvFXGQ= github.com/iden3/go-iden3-core v0.0.8/go.mod h1:URNjIhMql6sEbWubIGrjJdw5wHCE1Pk1XghxjBOtA3s= +github.com/iden3/go-iden3-crypto v0.0.5 h1:inCSm5a+ry+nbpVTL/9+m6UcIwSv6nhUm0tnIxEbcps= github.com/iden3/go-iden3-crypto v0.0.5/go.mod h1:XKw1oDwYn2CIxKOtr7m/mL5jMn4mLOxAxtZBRxQBev8= github.com/iden3/go-iden3-crypto v0.0.6-0.20200723082457-29a66457f0bf h1:/7L5dEqctuzJY2g8OEQct+1Y+n2sMKyd4JoYhw2jy1s= github.com/iden3/go-iden3-crypto v0.0.6-0.20200723082457-29a66457f0bf/go.mod h1:XKw1oDwYn2CIxKOtr7m/mL5jMn4mLOxAxtZBRxQBev8= diff --git a/test/l2db.go b/test/l2db.go index 3bc8306..47f6227 100644 --- a/test/l2db.go +++ b/test/l2db.go @@ -1,6 +1,15 @@ package test -import "github.com/jmoiron/sqlx" +import ( + "math/big" + "strconv" + "time" + + ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/common" + "github.com/iden3/go-iden3-crypto/babyjub" + "github.com/jmoiron/sqlx" +) // CleanL2DB deletes 'tx_pool' and 'account_creation_auth' from the given DB func CleanL2DB(db *sqlx.DB) { @@ -11,3 +20,77 @@ func CleanL2DB(db *sqlx.DB) { panic(err) } } + +// GenPoolTxs generates L2 pool txs. +// WARNING: This tx doesn't follow the protocol (signature, txID, ...) +// it's just to test getting/setting from/to the DB. +func GenPoolTxs(n int) []*common.PoolL2Tx { + txs := make([]*common.PoolL2Tx, 0, n) + privK := babyjub.NewRandPrivKey() + for i := 0; i < n; i++ { + var state common.PoolL2TxState + //nolint:gomnd + if i%4 == 0 { + state = common.PoolL2TxStatePending + //nolint:gomnd + } else if i%4 == 1 { + state = common.PoolL2TxStateInvalid + //nolint:gomnd + } else if i%4 == 2 { + state = common.PoolL2TxStateForging + //nolint:gomnd + } else if i%4 == 3 { + state = common.PoolL2TxStateForged + } + f := new(big.Float).SetInt(big.NewInt(int64(i))) + amountF, _ := f.Float64() + tx := &common.PoolL2Tx{ + TxID: common.TxID(common.Hash([]byte(strconv.Itoa(i)))), + FromIdx: common.Idx(i), + ToIdx: common.Idx(i + 1), + ToEthAddr: ethCommon.BigToAddress(big.NewInt(int64(i))), + ToBJJ: privK.Public(), + TokenID: common.TokenID(i), + Amount: big.NewInt(int64(i)), + AmountFloat: amountF, + //nolint:gomnd + Fee: common.FeeSelector(i % 255), + Nonce: common.Nonce(i), + State: state, + Signature: privK.SignPoseidon(big.NewInt(int64(i))), + Timestamp: time.Now().UTC(), + } + if i%2 == 0 { // Optional parameters: rq + tx.RqFromIdx = common.Idx(i) + tx.RqToIdx = common.Idx(i + 1) + tx.RqToEthAddr = ethCommon.BigToAddress(big.NewInt(int64(i))) + tx.RqToBJJ = privK.Public() + tx.RqTokenID = common.TokenID(i) + tx.RqAmount = big.NewInt(int64(i)) + tx.RqFee = common.FeeSelector(i) + tx.RqNonce = uint64(i) + } + if i%3 == 0 { // Optional parameters: things that get updated "a posteriori" + tx.BatchNum = 489 + tx.AbsoluteFee = 39.12345 + tx.AbsoluteFeeUpdate = time.Now().UTC() + } + txs = append(txs, tx) + } + return txs +} + +// GenAuths generates account creation authorizations +func GenAuths(nAuths int) []*common.AccountCreationAuth { + auths := []*common.AccountCreationAuth{} + for i := 0; i < nAuths; i++ { + privK := babyjub.NewRandPrivKey() + auths = append(auths, &common.AccountCreationAuth{ + EthAddr: ethCommon.BigToAddress(big.NewInt(int64(i))), + BJJ: privK.Public(), + Signature: []byte(strconv.Itoa(i)), + Timestamp: time.Now(), + }) + } + return auths +}