Feature/merge history l2 tables (#156)

* WIP rebase

* Combine both SQL DBs

* API and DB refactor
This commit is contained in:
a_bennassar
2020-09-29 18:27:07 +02:00
committed by GitHub
parent 8efbb7ab18
commit c6f70f3177
34 changed files with 1493 additions and 990 deletions

View File

@@ -1,19 +1,16 @@
package l2db
import (
"fmt"
"math/big"
"time"
ethCommon "github.com/ethereum/go-ethereum/common"
"github.com/gobuffalo/packr/v2"
"github.com/hermeznetwork/hermez-node/common"
"github.com/hermeznetwork/hermez-node/db"
"github.com/hermeznetwork/hermez-node/log"
"github.com/iden3/go-iden3-crypto/babyjub"
"github.com/jmoiron/sqlx"
//nolint:errcheck // driver for postgres DB
_ "github.com/lib/pq"
migrate "github.com/rubenv/sql-migrate"
"github.com/russross/meddler"
)
@@ -29,40 +26,15 @@ type L2DB struct {
}
// NewL2DB creates a L2DB.
// To create it, it's needed postgres configuration, safety period expressed in batches,
// To create it, it's needed db connection, safety period expressed in batches,
// maxTxs that the DB should have and TTL (time to live) for pending txs.
func NewL2DB(
port int, host, user, password, dbname string,
safetyPeriod common.BatchNum,
maxTxs uint32,
TTL time.Duration,
) (*L2DB, error) {
// init meddler
db.InitMeddler()
meddler.Default = meddler.PostgreSQL
// Stablish DB connection
psqlconn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", host, port, user, password, dbname)
db, err := sqlx.Connect("postgres", psqlconn)
if err != nil {
return nil, err
}
// Run DB migrations
migrations := &migrate.PackrMigrationSource{
Box: packr.New("l2db-migrations", "./migrations"),
}
nMigrations, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up)
if err != nil {
return nil, err
}
log.Debug("L2DB applied ", nMigrations, " migrations for ", dbname, " database")
func NewL2DB(db *sqlx.DB, safetyPeriod common.BatchNum, maxTxs uint32, TTL time.Duration) *L2DB {
return &L2DB{
db: db,
safetyPeriod: safetyPeriod,
ttl: TTL,
maxTxs: maxTxs,
}, nil
}
}
// DB returns a pointer to the L2DB.db. This method should be used only for
@@ -86,27 +58,82 @@ func (l2db *L2DB) GetAccountCreationAuth(addr ethCommon.Address) (*common.Accoun
)
}
// AddTx inserts a tx into the L2DB
func (l2db *L2DB) AddTx(tx *common.PoolL2Tx) error {
return meddler.Insert(l2db.db, "tx_pool", tx)
// AddTxTest inserts a tx into the L2DB. This is useful for test purposes,
// but in production txs will only be inserted through the API (method TBD)
func (l2db *L2DB) AddTxTest(tx *common.PoolL2Tx) error {
type withouUSD struct {
TxID common.TxID `meddler:"tx_id"`
FromIdx common.Idx `meddler:"from_idx"`
ToIdx common.Idx `meddler:"to_idx"`
ToEthAddr ethCommon.Address `meddler:"to_eth_addr"`
ToBJJ *babyjub.PublicKey `meddler:"to_bjj"`
TokenID common.TokenID `meddler:"token_id"`
Amount *big.Int `meddler:"amount,bigint"`
AmountFloat float64 `meddler:"amount_f"`
Fee common.FeeSelector `meddler:"fee"`
Nonce common.Nonce `meddler:"nonce"`
State common.PoolL2TxState `meddler:"state"`
Signature *babyjub.Signature `meddler:"signature"`
Timestamp time.Time `meddler:"timestamp,utctime"`
BatchNum common.BatchNum `meddler:"batch_num,zeroisnull"`
RqFromIdx common.Idx `meddler:"rq_from_idx,zeroisnull"`
RqToIdx common.Idx `meddler:"rq_to_idx,zeroisnull"`
RqToEthAddr ethCommon.Address `meddler:"rq_to_eth_addr"`
RqToBJJ *babyjub.PublicKey `meddler:"rq_to_bjj"`
RqTokenID common.TokenID `meddler:"rq_token_id,zeroisnull"`
RqAmount *big.Int `meddler:"rq_amount,bigintnull"`
RqFee common.FeeSelector `meddler:"rq_fee,zeroisnull"`
RqNonce uint64 `meddler:"rq_nonce,zeroisnull"`
Type common.TxType `meddler:"tx_type"`
}
return meddler.Insert(l2db.db, "tx_pool", &withouUSD{
TxID: tx.TxID,
FromIdx: tx.FromIdx,
ToIdx: tx.ToIdx,
ToEthAddr: tx.ToEthAddr,
ToBJJ: tx.ToBJJ,
TokenID: tx.TokenID,
Amount: tx.Amount,
AmountFloat: tx.AmountFloat,
Fee: tx.Fee,
Nonce: tx.Nonce,
State: tx.State,
Signature: tx.Signature,
Timestamp: tx.Timestamp,
BatchNum: tx.BatchNum,
RqFromIdx: tx.RqFromIdx,
RqToIdx: tx.RqToIdx,
RqToEthAddr: tx.RqToEthAddr,
RqToBJJ: tx.RqToBJJ,
RqTokenID: tx.RqTokenID,
RqAmount: tx.RqAmount,
RqFee: tx.RqFee,
RqNonce: tx.RqNonce,
Type: tx.Type,
})
}
// selectPoolTx select part of queries to get common.PoolL2Tx
const selectPoolTx = `SELECT tx_pool.*, token.usd * tx_pool.amount_f AS value_usd,
fee_percentage(tx_pool.fee::NUMERIC) * token.usd * tx_pool.amount_f AS fee_usd, token.usd_update,
token.symbol AS token_symbol FROM tx_pool INNER JOIN token ON tx_pool.token_id = token.token_id `
// GetTx return the specified Tx
func (l2db *L2DB) GetTx(txID common.TxID) (*common.PoolL2Tx, error) {
tx := new(common.PoolL2Tx)
return tx, meddler.QueryRow(
l2db.db, tx,
"SELECT * FROM tx_pool WHERE tx_id = $1;",
selectPoolTx+"WHERE tx_id = $1;",
txID,
)
}
// GetPendingTxs return all the pending txs of the L2DB
// GetPendingTxs return all the pending txs of the L2DB, that have a non NULL AbsoluteFee
func (l2db *L2DB) GetPendingTxs() ([]*common.PoolL2Tx, error) {
var txs []*common.PoolL2Tx
err := meddler.QueryAll(
l2db.db, &txs,
"SELECT * FROM tx_pool WHERE state = $1",
selectPoolTx+"WHERE state = $1 AND token.usd IS NOT NULL",
common.PoolL2TxStatePending,
)
return txs, err
@@ -202,40 +229,6 @@ func (l2db *L2DB) CheckNonces(updatedAccounts []common.Account, batchNum common.
return txn.Commit()
}
// UpdateTxValue updates the absolute fee and value of txs given a token list that include their price in USD
func (l2db *L2DB) UpdateTxValue(tokens []common.Token) (err error) {
// WARNING: this is very slow and should be optimized
txn, err := l2db.db.Begin()
if err != nil {
return err
}
defer func() {
// Rollback the transaction if there was an error.
if err != nil {
err = txn.Rollback()
}
}()
now := time.Now()
for i := 0; i < len(tokens); i++ {
_, err = txn.Exec(
`UPDATE tx_pool
SET usd_update = $1, value_usd = amount_f * $2, fee_usd = $2 * amount_f * CASE
WHEN fee = 0 THEN 0
WHEN fee >= 1 AND fee <= 32 THEN POWER(10,-24+(fee::float/2))
WHEN fee >= 33 AND fee <= 223 THEN POWER(10,-8+(0.041666666666667*(fee::float-32)))
WHEN fee >= 224 AND fee <= 255 THEN POWER(10,fee-224) END
WHERE token_id = $3;`,
now,
tokens[i].USD,
tokens[i].TokenID,
)
if err != nil {
return err
}
}
return txn.Commit()
}
// Reorg updates the state of txs that were updated in a batch that has been discarted due to a blockchain reorg.
// The state of the affected txs can change form Forged -> Pending or from Invalid -> Pending
func (l2db *L2DB) Reorg(lastValidBatch common.BatchNum) error {
@@ -286,8 +279,3 @@ func (l2db *L2DB) Purge(currentBatchNum common.BatchNum) (err error) {
}
return txn.Commit()
}
// Close frees the resources used by the L2DB
func (l2db *L2DB) Close() error {
return l2db.db.Close()
}

View File

@@ -1,79 +1,110 @@
package l2db
import (
"fmt"
"math"
"os"
"testing"
"time"
"github.com/hermeznetwork/hermez-node/common"
dbUtils "github.com/hermeznetwork/hermez-node/db"
"github.com/hermeznetwork/hermez-node/db/historydb"
"github.com/hermeznetwork/hermez-node/log"
"github.com/hermeznetwork/hermez-node/test"
"github.com/jmoiron/sqlx"
"github.com/stretchr/testify/assert"
)
var l2DB *L2DB
var tokens []common.Token
func TestMain(m *testing.M) {
// init DB
var err error
pass := os.Getenv("POSTGRES_PASS")
l2DB, err = NewL2DB(5432, "localhost", "hermez", pass, "l2", 10, 100, 24*time.Hour)
db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
if err != nil {
panic(err)
}
l2DB = NewL2DB(db, 10, 100, 24*time.Hour)
tokens, err = prepareHistoryDB(db)
if err != nil {
log.Error("L2DB migration failed: " + err.Error())
panic(err)
} else {
log.Debug("L2DB migration succed")
}
// Run tests
result := m.Run()
// Close DB
if err := l2DB.Close(); err != nil {
fmt.Println("Error closing the history DB:", err)
if err := db.Close(); err != nil {
log.Error("Error closing the history DB:", err)
}
os.Exit(result)
}
func TestAddTx(t *testing.T) {
func prepareHistoryDB(db *sqlx.DB) ([]common.Token, error) {
historyDB := historydb.NewHistoryDB(db)
const fromBlock int64 = 1
const toBlock int64 = 5
// Clean historyDB
if err := historyDB.Reorg(-1); err != nil {
panic(err)
}
// Store blocks to historyDB
blocks := test.GenBlocks(fromBlock, toBlock)
if err := historyDB.AddBlocks(blocks); err != nil {
panic(err)
}
// Store tokens to historyDB
const nTokens = 5
tokens := test.GenTokens(nTokens, blocks)
return tokens, historyDB.AddTokens(tokens)
}
func TestAddTxTest(t *testing.T) {
// Gen poolTxs
const nInserts = 20
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
for _, tx := range txs {
err := l2DB.AddTx(tx)
err := l2DB.AddTxTest(tx)
assert.NoError(t, err)
fetchedTx, err := l2DB.GetTx(tx.TxID)
assert.NoError(t, err)
assert.Equal(t, tx.Timestamp.Unix(), fetchedTx.Timestamp.Unix())
tx.Timestamp = fetchedTx.Timestamp
assert.Equal(t, tx.AbsoluteFeeUpdate.Unix(), fetchedTx.AbsoluteFeeUpdate.Unix())
tx.Timestamp = fetchedTx.Timestamp
tx.AbsoluteFeeUpdate = fetchedTx.AbsoluteFeeUpdate
assert.Equal(t, tx, fetchedTx)
assertTx(t, tx, fetchedTx)
}
}
func BenchmarkAddTx(b *testing.B) {
func assertTx(t *testing.T, expected, actual *common.PoolL2Tx) {
assert.Equal(t, expected.Timestamp.Unix(), actual.Timestamp.Unix())
expected.Timestamp = actual.Timestamp
if expected.AbsoluteFeeUpdate != nil {
assert.Equal(t, expected.AbsoluteFeeUpdate.Unix(), actual.AbsoluteFeeUpdate.Unix())
expected.AbsoluteFeeUpdate = actual.AbsoluteFeeUpdate
} else {
assert.Equal(t, expected.AbsoluteFeeUpdate, actual.AbsoluteFeeUpdate)
}
test.AssertUSD(t, expected.AbsoluteFee, actual.AbsoluteFee)
assert.Equal(t, expected, actual)
}
func BenchmarkAddTxTest(b *testing.B) {
const nInserts = 20
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
now := time.Now()
for _, tx := range txs {
_ = l2DB.AddTx(tx)
_ = l2DB.AddTxTest(tx)
}
elapsedTime := time.Since(now)
fmt.Println("Time to insert 2048 txs:", elapsedTime)
log.Info("Time to insert 2048 txs:", elapsedTime)
}
func TestGetPending(t *testing.T) {
const nInserts = 20
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
var pendingTxs []*common.PoolL2Tx
for _, tx := range txs {
err := l2DB.AddTx(tx)
err := l2DB.AddTxTest(tx)
assert.NoError(t, err)
if tx.State == common.PoolL2TxStatePending {
if tx.State == common.PoolL2TxStatePending && tx.AbsoluteFee != nil {
pendingTxs = append(pendingTxs, tx)
}
}
@@ -81,11 +112,7 @@ func TestGetPending(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, len(pendingTxs), len(fetchedTxs))
for i, fetchedTx := range fetchedTxs {
assert.Equal(t, pendingTxs[i].Timestamp.Unix(), fetchedTx.Timestamp.Unix())
pendingTxs[i].Timestamp = fetchedTx.Timestamp
assert.Equal(t, pendingTxs[i].AbsoluteFeeUpdate.Unix(), fetchedTx.AbsoluteFeeUpdate.Unix())
pendingTxs[i].AbsoluteFeeUpdate = fetchedTx.AbsoluteFeeUpdate
assert.Equal(t, pendingTxs[i], fetchedTx)
assertTx(t, pendingTxs[i], fetchedTx)
}
}
@@ -94,12 +121,12 @@ func TestStartForging(t *testing.T) {
const nInserts = 60
const fakeBatchNum common.BatchNum = 33
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
var startForgingTxIDs []common.TxID
randomizer := 0
// Add txs to DB
for _, tx := range txs {
err := l2DB.AddTx(tx)
err := l2DB.AddTxTest(tx)
assert.NoError(t, err)
if tx.State == common.PoolL2TxStatePending && randomizer%2 == 0 {
randomizer++
@@ -123,12 +150,12 @@ func TestDoneForging(t *testing.T) {
const nInserts = 60
const fakeBatchNum common.BatchNum = 33
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
var doneForgingTxIDs []common.TxID
randomizer := 0
// Add txs to DB
for _, tx := range txs {
err := l2DB.AddTx(tx)
err := l2DB.AddTxTest(tx)
assert.NoError(t, err)
if tx.State == common.PoolL2TxStateForging && randomizer%2 == 0 {
randomizer++
@@ -152,12 +179,12 @@ func TestInvalidate(t *testing.T) {
const nInserts = 60
const fakeBatchNum common.BatchNum = 33
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
var invalidTxIDs []common.TxID
randomizer := 0
// Add txs to DB
for _, tx := range txs {
err := l2DB.AddTx(tx)
err := l2DB.AddTxTest(tx)
assert.NoError(t, err)
if tx.State != common.PoolL2TxStateInvalid && randomizer%2 == 0 {
randomizer++
@@ -181,7 +208,7 @@ func TestCheckNonces(t *testing.T) {
const nInserts = 60
const fakeBatchNum common.BatchNum = 33
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
var invalidTxIDs []common.TxID
// Generate accounts
const nAccoutns = 2
@@ -204,7 +231,7 @@ func TestCheckNonces(t *testing.T) {
txs[i].Nonce = currentNonce + 1
}
}
err := l2DB.AddTx(txs[i])
err := l2DB.AddTxTest(txs[i])
assert.NoError(t, err)
}
// Start forging txs
@@ -219,69 +246,13 @@ func TestCheckNonces(t *testing.T) {
}
}
func TestUpdateTxValue(t *testing.T) {
// Generate txs
const nInserts = 255 // Force all possible fee selector values
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
// Generate tokens
const nTokens = 2
tokens := []common.Token{}
for i := 0; i < nTokens; i++ {
tokens = append(tokens, common.Token{
TokenID: common.TokenID(i),
USD: float64(i) * 1.3,
})
}
// Add txs to DB
for i := 0; i < len(txs); i++ {
// Set Token
token := tokens[i%len(tokens)]
txs[i].TokenID = token.TokenID
// Insert to DB
err := l2DB.AddTx(txs[i])
assert.NoError(t, err)
// Set USD values (for comparing results when fetching from DB)
txs[i].USD = txs[i].AmountFloat * token.USD
if txs[i].Fee == 0 {
txs[i].AbsoluteFee = 0
} else if txs[i].Fee <= 32 {
txs[i].AbsoluteFee = txs[i].USD * math.Pow(10, -24+(float64(txs[i].Fee)/2))
} else if txs[i].Fee <= 223 {
txs[i].AbsoluteFee = txs[i].USD * math.Pow(10, -8+(0.041666666666667*(float64(txs[i].Fee)-32)))
} else {
txs[i].AbsoluteFee = txs[i].USD * math.Pow(10, float64(txs[i].Fee)-224)
}
}
// Update token value
err := l2DB.UpdateTxValue(tokens)
assert.NoError(t, err)
// Fetch txs and check that they've been updated correctly
for _, tx := range txs {
fetchedTx, err := l2DB.GetTx(tx.TxID)
assert.NoError(t, err)
if fetchedTx.USD > tx.USD {
assert.Less(t, 0.999, tx.USD/fetchedTx.USD)
} else if fetchedTx.USD < tx.USD {
assert.Less(t, 0.999, fetchedTx.USD/tx.USD)
}
if fetchedTx.AbsoluteFee > tx.AbsoluteFee {
assert.Less(t, 0.999, tx.AbsoluteFee/fetchedTx.AbsoluteFee)
} else if fetchedTx.AbsoluteFee < tx.AbsoluteFee {
assert.Less(t, 0.999, fetchedTx.AbsoluteFee/tx.AbsoluteFee)
}
// Time is set in the DB, so it cannot be compared exactly
assert.Greater(t, float64(15*time.Second), time.Since(fetchedTx.AbsoluteFeeUpdate).Seconds())
}
}
func TestReorg(t *testing.T) {
// Generate txs
const nInserts = 20
const lastValidBatch common.BatchNum = 20
const reorgBatch common.BatchNum = lastValidBatch + 1
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(nInserts)
txs := test.GenPoolTxs(nInserts, tokens)
// Add txs to the DB
reorgedTxIDs := []common.TxID{}
nonReorgedTxIDs := []common.TxID{}
@@ -293,7 +264,7 @@ func TestReorg(t *testing.T) {
txs[i].BatchNum = lastValidBatch
nonReorgedTxIDs = append(nonReorgedTxIDs, txs[i].TxID)
}
err := l2DB.AddTx(txs[i])
err := l2DB.AddTxTest(txs[i])
assert.NoError(t, err)
}
err := l2DB.Reorg(lastValidBatch)
@@ -316,7 +287,7 @@ func TestPurge(t *testing.T) {
// Generate txs
nInserts := l2DB.maxTxs + 20
test.CleanL2DB(l2DB.DB())
txs := test.GenPoolTxs(int(nInserts))
txs := test.GenPoolTxs(int(nInserts), tokens)
deletedIDs := []common.TxID{}
keepedIDs := []common.TxID{}
const toDeleteBatchNum common.BatchNum = 30
@@ -335,14 +306,14 @@ func TestPurge(t *testing.T) {
}
deletedIDs = append(deletedIDs, txs[i].TxID)
}
err := l2DB.AddTx(txs[i])
err := l2DB.AddTxTest(txs[i])
assert.NoError(t, err)
}
for i := int(l2DB.maxTxs); i < len(txs); i++ {
// Delete after TTL
txs[i].Timestamp = time.Unix(time.Now().UTC().Unix()-int64(l2DB.ttl.Seconds()+float64(4*time.Second)), 0)
deletedIDs = append(deletedIDs, txs[i].TxID)
err := l2DB.AddTx(txs[i])
err := l2DB.AddTxTest(txs[i])
assert.NoError(t, err)
}
// Purge txs

View File

@@ -1,40 +0,0 @@
-- +migrate Up
CREATE TABLE tx_pool (
tx_id BYTEA PRIMARY KEY,
from_idx BIGINT NOT NULL,
to_idx BIGINT NOT NULL,
to_eth_addr BYTEA NOT NULL,
to_bjj BYTEA NOT NULL,
token_id INT NOT NULL,
amount BYTEA NOT NULL,
amount_f NUMERIC NOT NULL,
value_usd NUMERIC,
fee SMALLINT NOT NULL,
nonce BIGINT NOT NULL,
state CHAR(4) NOT NULL,
signature BYTEA NOT NULL,
timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL,
batch_num BIGINT,
rq_from_idx BIGINT,
rq_to_idx BIGINT,
rq_to_eth_addr BYTEA,
rq_to_bjj BYTEA,
rq_token_id INT,
rq_amount BYTEA,
rq_fee SMALLINT,
rq_nonce BIGINT,
fee_usd NUMERIC,
usd_update TIMESTAMP WITHOUT TIME ZONE,
tx_type VARCHAR(40) NOT NULL
);
CREATE TABLE account_creation_auth (
eth_addr BYTEA PRIMARY KEY,
bjj BYTEA NOT NULL,
signature BYTEA NOT NULL,
timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL
);
-- +migrate Down
DROP TABLE account_creation_auth;
DROP TABLE tx_pool;