Add GET histroy-transactions endpoint

2026-02-07 03:16:45 +01:00 · 2020-09-21 00:11:20 +02:00
parent 35a558f6c9
commit 85fe885265
21 changed files with 4114 additions and 88 deletions
--- a/db/historydb/historydb.go
+++ b/db/historydb/historydb.go
@@ -1,11 +1,16 @@
 package historydb

 import (
+	"database/sql"
+	"errors"
 	"fmt"

+	ethCommon "github.com/ethereum/go-ethereum/common"
 	"github.com/gobuffalo/packr/v2"
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db"
+	"github.com/hermeznetwork/hermez-node/log"
+	"github.com/iden3/go-iden3-crypto/babyjub"
 	"github.com/jmoiron/sqlx"

 	//nolint:errcheck // driver for postgres DB
@@ -37,9 +42,11 @@ func NewHistoryDB(port int, host, user, password, dbname string) (*HistoryDB, er
 	migrations := &migrate.PackrMigrationSource{
 		Box: packr.New("history-migrations", "./migrations"),
 	}
-	if _, err := migrate.Exec(hdb.DB, "postgres", migrations, migrate.Up); err != nil {
+	nMigrations, err := migrate.Exec(hdb.DB, "postgres", migrations, migrate.Up)
+	if err != nil {
 		return nil, err
 	}
+	log.Debug("HistoryDB applied ", nMigrations, " migrations for ", dbname, " database")

 	return &HistoryDB{hdb}, nil
 }
@@ -320,6 +327,107 @@ func (hdb *HistoryDB) GetTxs() ([]*common.Tx, error) {
 	return txs, err
 }

+// GetHistoryTxs returns a list of txs from the DB using the HistoryTx struct
+func (hdb *HistoryDB) GetHistoryTxs(
+	ethAddr *ethCommon.Address, bjj *babyjub.PublicKey,
+	tokenID, idx, batchNum *uint, txType *common.TxType,
+	offset, limit *uint, last bool,
+) ([]*HistoryTx, int, error) {
+	if ethAddr != nil && bjj != nil {
+		return nil, 0, errors.New("ethAddr and bjj are incompatible")
+	}
+	var query string
+	var args []interface{}
+	queryStr := `SELECT tx.*, tx.amount_f * token.usd AS current_usd,
+	token.symbol, token.usd_update, block.timestamp, count(*) OVER() AS total_items FROM tx 
+	INNER JOIN token ON tx.token_id = token.token_id 
+	INNER JOIN block ON tx.eth_block_num = block.eth_block_num `
+	// Apply filters
+	nextIsAnd := false
+	// ethAddr filter
+	if ethAddr != nil {
+		queryStr = `WITH acc AS 
+		(select idx from account where eth_addr = ?) ` + queryStr
+		queryStr += ", acc WHERE (tx.from_idx IN(acc.idx) OR tx.to_idx IN(acc.idx)) "
+		nextIsAnd = true
+		args = append(args, ethAddr)
+	} else if bjj != nil { // bjj filter
+		queryStr = `WITH acc AS 
+		(select idx from account where bjj = ?) ` + queryStr
+		queryStr += ", acc WHERE (tx.from_idx IN(acc.idx) OR tx.to_idx IN(acc.idx)) "
+		nextIsAnd = true
+		args = append(args, bjj)
+	}
+	// tokenID filter
+	if tokenID != nil {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		queryStr += "tx.token_id = ? "
+		args = append(args, tokenID)
+		nextIsAnd = true
+	}
+	// idx filter
+	if idx != nil {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		queryStr += "(tx.from_idx = ? OR tx.to_idx = ?) "
+		args = append(args, idx, idx)
+		nextIsAnd = true
+	}
+	// batchNum filter
+	if batchNum != nil {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		queryStr += "tx.batch_num = ? "
+		args = append(args, batchNum)
+		nextIsAnd = true
+	}
+	// txType filter
+	if txType != nil {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		queryStr += "tx.type = ? "
+		args = append(args, txType)
+		// nextIsAnd = true
+	}
+	// pagination
+	if last {
+		queryStr += "ORDER BY (batch_num, position) DESC NULLS FIRST "
+	} else {
+		queryStr += "ORDER BY (batch_num, position) ASC NULLS LAST "
+		queryStr += fmt.Sprintf("OFFSET %d ", *offset)
+	}
+	queryStr += fmt.Sprintf("LIMIT %d ", *limit)
+	query = hdb.db.Rebind(queryStr)
+	// log.Debug(query)
+	txs := []*HistoryTx{}
+	if err := meddler.QueryAll(hdb.db, &txs, query, args...); err != nil {
+		return nil, 0, err
+	}
+	if len(txs) == 0 {
+		return nil, 0, sql.ErrNoRows
+	} else if last {
+		tmp := []*HistoryTx{}
+		for i := len(txs) - 1; i >= 0; i-- {
+			tmp = append(tmp, txs[i])
+		}
+		txs = tmp
+	}
+	return txs, txs[0].TotalItems, nil
+}
+
 // GetTx returns a tx from the DB
 func (hdb *HistoryDB) GetTx(txID common.TxID) (*common.Tx, error) {
 	tx := new(common.Tx)
--- a/db/historydb/historydb_test.go
+++ b/db/historydb/historydb_test.go
@@ -2,7 +2,6 @@ package historydb

 import (
 	"fmt"
-	"math"
 	"math/big"
 	"os"
 	"testing"
@@ -186,7 +185,7 @@ func TestAccounts(t *testing.T) {
 	assert.NoError(t, err)
 	// Generate fake accounts
 	const nAccounts = 3
-	accs := test.GenAccounts(nAccounts, 0, tokens, nil, batches)
+	accs := test.GenAccounts(nAccounts, 0, tokens, nil, nil, batches)
 	err = historyDB.AddAccounts(accs)
 	assert.NoError(t, err)
 	// Fetch accounts
@@ -219,7 +218,7 @@ func TestTxs(t *testing.T) {
 	assert.NoError(t, err)
 	// Generate fake accounts
 	const nAccounts = 3
-	accs := test.GenAccounts(nAccounts, 0, tokens, nil, batches)
+	accs := test.GenAccounts(nAccounts, 0, tokens, nil, nil, batches)
 	err = historyDB.AddAccounts(accs)
 	assert.NoError(t, err)
 	// Generate fake L1 txs
@@ -265,15 +264,7 @@ func TestTxs(t *testing.T) {
 		} else {
 			assert.Less(t, 0.999, fetchedTx.USD/tx.USD)
 		}
-		if tx.Fee == 0 {
-			tx.FeeUSD = 0
-		} else if tx.Fee <= 32 {
-			tx.FeeUSD = tx.USD * math.Pow(10, -24+(float64(tx.Fee)/2))
-		} else if tx.Fee <= 223 {
-			tx.FeeUSD = tx.USD * math.Pow(10, -8+(0.041666666666667*(float64(tx.Fee)-32)))
-		} else {
-			tx.FeeUSD = tx.USD * math.Pow(10, float64(tx.Fee)-224)
-		}
+		tx.FeeUSD = tx.USD * tx.Fee.Percentage()
 		if fetchedTx.FeeUSD > tx.FeeUSD {
 			assert.Less(t, 0.999, tx.FeeUSD/fetchedTx.FeeUSD)
 		} else if fetchedTx.FeeUSD < tx.FeeUSD {
--- a/db/historydb/migrations/001_init.sql
+++ b/db/historydb/migrations/001_init.sql
@@ -5,7 +5,7 @@ CREATE TABLE block (
    hash BYTEA NOT NULL
 );

-CREATE TABLE coordianator (
+CREATE TABLE coordinator (
    forger_addr BYTEA NOT NULL,
    eth_block_num BIGINT NOT NULL REFERENCES block (eth_block_num) ON DELETE CASCADE,
    withdraw_addr BYTEA NOT NULL,
@@ -99,6 +99,8 @@ CREATE TABLE tx (
    nonce BIGINT
 );

+CREATE INDEX tx_order ON tx (batch_num, position);
+
 -- +migrate StatementBegin
 CREATE FUNCTION set_tx()
    RETURNS TRIGGER 
@@ -197,11 +199,10 @@ CREATE TABLE consensus_vars (
 DROP TABLE consensus_vars;
 DROP TABLE rollup_vars;
 DROP TABLE account;
-DROP TABLE l2tx;
-DROP TABLE l1tx;
+DROP TABLE tx;
 DROP TABLE token;
 DROP TABLE bid;
 DROP TABLE exit_tree;
 DROP TABLE batch;
-DROP TABLE coordianator;
+DROP TABLE coordinator;
 DROP TABLE block;
--- a/db/historydb/views.go
+++ b/db/historydb/views.go
@@ -0,0 +1,46 @@
+package historydb
+
+import (
+	"math/big"
+	"time"
+
+	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/iden3/go-iden3-crypto/babyjub"
+)
+
+// HistoryTx is a representation of a generic Tx with additional information
+// required by the API, and extracted by joining block and token tables
+type HistoryTx struct {
+	// Generic
+	IsL1        bool            `meddler:"is_l1"`
+	TxID        common.TxID     `meddler:"id"`
+	Type        common.TxType   `meddler:"type"`
+	Position    int             `meddler:"position"`
+	FromIdx     common.Idx      `meddler:"from_idx"`
+	ToIdx       common.Idx      `meddler:"to_idx"`
+	Amount      *big.Int        `meddler:"amount,bigint"`
+	AmountFloat float64         `meddler:"amount_f"`
+	TokenID     common.TokenID  `meddler:"token_id"`
+	USD         float64         `meddler:"amount_usd,zeroisnull"`
+	BatchNum    common.BatchNum `meddler:"batch_num,zeroisnull"` // batchNum in which this tx was forged. If the tx is L2, this must be != 0
+	EthBlockNum int64           `meddler:"eth_block_num"`        // Ethereum Block Number in which this L1Tx was added to the queue
+	// L1
+	ToForgeL1TxsNum int64              `meddler:"to_forge_l1_txs_num"` // toForgeL1TxsNum in which the tx was forged / will be forged
+	UserOrigin      bool               `meddler:"user_origin"`         // true if the tx was originated by a user, false if it was aoriginated by a coordinator. Note that this differ from the spec for implementation simplification purpposes
+	FromEthAddr     ethCommon.Address  `meddler:"from_eth_addr"`
+	FromBJJ         *babyjub.PublicKey `meddler:"from_bjj"`
+	LoadAmount      *big.Int           `meddler:"load_amount,bigintnull"`
+	LoadAmountFloat float64            `meddler:"load_amount_f"`
+	LoadAmountUSD   float64            `meddler:"load_amount_usd,zeroisnull"`
+	// L2
+	Fee    common.FeeSelector `meddler:"fee,zeroisnull"`
+	FeeUSD float64            `meddler:"fee_usd,zeroisnull"`
+	Nonce  common.Nonce       `meddler:"nonce,zeroisnull"`
+	// API extras
+	Timestamp   time.Time `meddler:"timestamp,utctime"`
+	TokenSymbol string    `meddler:"symbol"`
+	CurrentUSD  float64   `meddler:"current_usd"`
+	USDUpdate   time.Time `meddler:"usd_update,utctime"`
+	TotalItems  int       `meddler:"total_items"`
+}
--- a/db/l2db/l2db.go
+++ b/db/l2db/l2db.go
@@ -8,6 +8,7 @@ import (
 	"github.com/gobuffalo/packr/v2"
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db"
+	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/jmoiron/sqlx"

 	//nolint:errcheck // driver for postgres DB
@@ -48,11 +49,13 @@ func NewL2DB(

 	// Run DB migrations
 	migrations := &migrate.PackrMigrationSource{
-		Box: packr.New("history-migrations", "./migrations"),
+		Box: packr.New("l2db-migrations", "./migrations"),
 	}
-	if _, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up); err != nil {
+	nMigrations, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up)
+	if err != nil {
 		return nil, err
 	}
+	log.Debug("L2DB applied ", nMigrations, " migrations for ", dbname, " database")

 	return &L2DB{
 		db:           db,
--- a/db/l2db/l2db_test.go
+++ b/db/l2db/l2db_test.go
@@ -21,7 +21,10 @@ func TestMain(m *testing.M) {
 	pass := os.Getenv("POSTGRES_PASS")
 	l2DB, err = NewL2DB(5432, "localhost", "hermez", pass, "l2", 10, 100, 24*time.Hour)
 	if err != nil {
+		log.Error("L2DB migration failed: " + err.Error())
 		panic(err)
+	} else {
+		log.Debug("L2DB migration succed")
 	}
 	// Run tests
 	result := m.Run()
@@ -34,7 +37,7 @@ func TestMain(m *testing.M) {

 func TestAddTx(t *testing.T) {
 	const nInserts = 20
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	for _, tx := range txs {
 		err := l2DB.AddTx(tx)
@@ -52,7 +55,7 @@ func TestAddTx(t *testing.T) {

 func BenchmarkAddTx(b *testing.B) {
 	const nInserts = 20
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	now := time.Now()
 	for _, tx := range txs {
@@ -64,7 +67,7 @@ func BenchmarkAddTx(b *testing.B) {

 func TestGetPending(t *testing.T) {
 	const nInserts = 20
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	var pendingTxs []*common.PoolL2Tx
 	for _, tx := range txs {
@@ -90,7 +93,7 @@ func TestStartForging(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
 	const fakeBatchNum common.BatchNum = 33
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	var startForgingTxIDs []common.TxID
 	randomizer := 0
@@ -119,7 +122,7 @@ func TestDoneForging(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
 	const fakeBatchNum common.BatchNum = 33
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	var doneForgingTxIDs []common.TxID
 	randomizer := 0
@@ -148,7 +151,7 @@ func TestInvalidate(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
 	const fakeBatchNum common.BatchNum = 33
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	var invalidTxIDs []common.TxID
 	randomizer := 0
@@ -177,7 +180,7 @@ func TestCheckNonces(t *testing.T) {
 	// Generate txs
 	const nInserts = 60
 	const fakeBatchNum common.BatchNum = 33
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	var invalidTxIDs []common.TxID
 	// Generate accounts
@@ -219,7 +222,7 @@ func TestCheckNonces(t *testing.T) {
 func TestUpdateTxValue(t *testing.T) {
 	// Generate txs
 	const nInserts = 255 // Force all possible fee selector values
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	// Generate tokens
 	const nTokens = 2
@@ -277,7 +280,7 @@ func TestReorg(t *testing.T) {
 	const nInserts = 20
 	const lastValidBatch common.BatchNum = 20
 	const reorgBatch common.BatchNum = lastValidBatch + 1
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(nInserts)
 	// Add txs to the DB
 	reorgedTxIDs := []common.TxID{}
@@ -312,7 +315,7 @@ func TestReorg(t *testing.T) {
 func TestPurge(t *testing.T) {
 	// Generate txs
 	nInserts := l2DB.maxTxs + 20
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	txs := test.GenPoolTxs(int(nInserts))
 	deletedIDs := []common.TxID{}
 	keepedIDs := []common.TxID{}
@@ -360,7 +363,7 @@ func TestPurge(t *testing.T) {
 }

 func TestAuth(t *testing.T) {
-	cleanDB()
+	test.CleanL2DB(l2DB.DB())
 	const nAuths = 5
 	// Generate authorizations
 	auths := test.GenAuths(nAuths)
@@ -378,12 +381,3 @@ func TestAuth(t *testing.T) {
 		assert.Equal(t, auths[i].Timestamp.Unix(), auths[i].Timestamp.Unix())
 	}
 }
-
-func cleanDB() {
-	if _, err := l2DB.db.Exec("DELETE FROM tx_pool"); err != nil {
-		panic(err)
-	}
-	if _, err := l2DB.db.Exec("DELETE FROM account_creation_auth"); err != nil {
-		panic(err)
-	}
-}