1package historydb
  2
  3import (
  4	"database/sql"
  5	"errors"
  6	"fmt"
  7
  8	ethCommon "github.com/ethereum/go-ethereum/common"
  9	"github.com/hermeznetwork/hermez-node/common"
 10	"github.com/hermeznetwork/hermez-node/db"
 11	"github.com/iden3/go-iden3-crypto/babyjub"
 12	"github.com/jmoiron/sqlx"
 13
 14	//nolint:errcheck // driver for postgres DB
 15	_ "github.com/lib/pq"
 16	"github.com/russross/meddler"
 17)
 18
 19// TODO(Edu): Document here how HistoryDB is kept consistent
 20
 21// HistoryDB persist the historic of the rollup
 22type HistoryDB struct {
 23	db *sqlx.DB
 24}
 25
 26// BlockData contains the information of a Block
 27type BlockData struct {
 28	block *common.Block
 29	// Rollup
 30	// L1UserTxs that were submitted in the block
 31	L1UserTxs        []common.L1Tx
 32	Batches          []BatchData
 33	RegisteredTokens []common.Token
 34	RollupVars       *common.RollupVars
 35	// Auction
 36	Bids         []common.Bid
 37	Coordinators []common.Coordinator
 38	AuctionVars  *common.AuctionVars
 39	// WithdrawalDelayer
 40	// TODO: enable when common.WithdrawalDelayerVars is Merged from Synchronizer PR
 41	// WithdrawalDelayerVars *common.WithdrawalDelayerVars
 42}
 43
 44// BatchData contains the information of a Batch
 45type BatchData struct {
 46	// L1UserTxs that were forged in the batch
 47	L1Batch          bool // TODO: Remove once Batch.ForgeL1TxsNum is a pointer
 48	L1UserTxs        []common.L1Tx
 49	L1CoordinatorTxs []common.L1Tx
 50	L2Txs            []common.L2Tx
 51	CreatedAccounts  []common.Account
 52	ExitTree         []common.ExitInfo
 53	Batch            *common.Batch
 54}
 55
 56// NewHistoryDB initialize the DB
 57func NewHistoryDB(db *sqlx.DB) *HistoryDB {
 58	return &HistoryDB{db: db}
 59}
 60
 61// AddBlock insert a block into the DB
 62func (hdb *HistoryDB) AddBlock(block *common.Block) error { return hdb.addBlock(hdb.db, block) }
 63func (hdb *HistoryDB) addBlock(d meddler.DB, block *common.Block) error {
 64	return meddler.Insert(d, "block", block)
 65}
 66
 67// AddBlocks inserts blocks into the DB
 68func (hdb *HistoryDB) AddBlocks(blocks []common.Block) error {
 69	return hdb.addBlocks(hdb.db, blocks)
 70}
 71
 72func (hdb *HistoryDB) addBlocks(d meddler.DB, blocks []common.Block) error {
 73	return db.BulkInsert(
 74		d,
 75		`INSERT INTO block (
 76			eth_block_num,
 77			timestamp,
 78			hash
 79		) VALUES %s;`,
 80		blocks[:],
 81	)
 82}
 83
 84// GetBlock retrieve a block from the DB, given a block number
 85func (hdb *HistoryDB) GetBlock(blockNum int64) (*common.Block, error) {
 86	block := &common.Block{}
 87	err := meddler.QueryRow(
 88		hdb.db, block,
 89		"SELECT * FROM block WHERE eth_block_num = $1;", blockNum,
 90	)
 91	return block, err
 92}
 93
 94// GetBlocks retrieve blocks from the DB, given a range of block numbers defined by from and to
 95func (hdb *HistoryDB) GetBlocks(from, to int64) ([]*common.Block, error) {
 96	var blocks []*common.Block
 97	err := meddler.QueryAll(
 98		hdb.db, &blocks,
 99		"SELECT * FROM block WHERE $1 <= eth_block_num AND eth_block_num < $2;",
100		from, to,
101	)
102	return blocks, err
103}
104
105// GetLastBlock retrieve the block with the highest block number from the DB
106func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) {
107	block := &common.Block{}
108	err := meddler.QueryRow(
109		hdb.db, block, "SELECT * FROM block ORDER BY eth_block_num DESC LIMIT 1;",
110	)
111	return block, err
112}
113
114// AddBatch insert a Batch into the DB
115func (hdb *HistoryDB) AddBatch(batch *common.Batch) error { return hdb.addBatch(hdb.db, batch) }
116func (hdb *HistoryDB) addBatch(d meddler.DB, batch *common.Batch) error {
117	return meddler.Insert(d, "batch", batch)
118}
119
120// AddBatches insert Bids into the DB
121func (hdb *HistoryDB) AddBatches(batches []common.Batch) error {
122	return hdb.addBatches(hdb.db, batches)
123}
124func (hdb *HistoryDB) addBatches(d meddler.DB, batches []common.Batch) error {
125	return db.BulkInsert(
126		d,
127		`INSERT INTO batch (
128			batch_num,
129			eth_block_num,
130			forger_addr,
131			fees_collected,
132			state_root,
133			num_accounts,
134			exit_root,
135			forge_l1_txs_num,
136			slot_num
137		) VALUES %s;`,
138		batches[:],
139	)
140}
141
142// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to
143func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]*common.Batch, error) {
144	var batches []*common.Batch
145	err := meddler.QueryAll(
146		hdb.db, &batches,
147		"SELECT * FROM batch WHERE $1 <= batch_num AND batch_num < $2;",
148		from, to,
149	)
150	return batches, err
151}
152
153// GetLastBatchNum returns the BatchNum of the latest forged batch
154func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) {
155	row := hdb.db.QueryRow("SELECT batch_num FROM batch ORDER BY batch_num DESC LIMIT 1;")
156	var batchNum common.BatchNum
157	return batchNum, row.Scan(&batchNum)
158}
159
160// GetLastL1TxsNum returns the greatest ForgeL1TxsNum in the DB.  If there's no
161// batch in the DB (nil, nil) is returned.
162func (hdb *HistoryDB) GetLastL1TxsNum() (*int64, error) {
163	row := hdb.db.QueryRow("SELECT MAX(forge_l1_txs_num) FROM batch;")
164	lastL1TxsNum := new(int64)
165	return lastL1TxsNum, row.Scan(&lastL1TxsNum)
166}
167
168// Reorg deletes all the information that was added into the DB after the
169// lastValidBlock.  If lastValidBlock is negative, all block information is
170// deleted.
171func (hdb *HistoryDB) Reorg(lastValidBlock int64) error {
172	var err error
173	if lastValidBlock < 0 {
174		_, err = hdb.db.Exec("DELETE FROM block;")
175	} else {
176		_, err = hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock)
177	}
178	return err
179}
180
181// SyncPoD stores all the data that can be changed / added on a block in the PoD SC
182func (hdb *HistoryDB) SyncPoD(
183	blockNum uint64,
184	bids []common.Bid,
185	coordinators []common.Coordinator,
186	vars *common.AuctionVars,
187) error {
188	return nil
189}
190
191// AddBids insert Bids into the DB
192func (hdb *HistoryDB) AddBids(bids []common.Bid) error { return hdb.addBids(hdb.db, bids) }
193func (hdb *HistoryDB) addBids(d meddler.DB, bids []common.Bid) error {
194	// TODO: check the coordinator info
195	return db.BulkInsert(
196		d,
197		"INSERT INTO bid (slot_num, forger_addr, bid_value, eth_block_num) VALUES %s;",
198		bids[:],
199	)
200}
201
202// GetBids return the bids
203func (hdb *HistoryDB) GetBids() ([]*common.Bid, error) {
204	var bids []*common.Bid
205	err := meddler.QueryAll(
206		hdb.db, &bids,
207		"SELECT * FROM bid;",
208	)
209	return bids, err
210}
211
212// AddCoordinators insert Coordinators into the DB
213func (hdb *HistoryDB) AddCoordinators(coordinators []common.Coordinator) error {
214	return hdb.addCoordinators(hdb.db, coordinators)
215}
216func (hdb *HistoryDB) addCoordinators(d meddler.DB, coordinators []common.Coordinator) error {
217	return db.BulkInsert(
218		d,
219		"INSERT INTO coordinator (forger_addr, eth_block_num, withdraw_addr, url) VALUES %s;",
220		coordinators[:],
221	)
222}
223
224// AddExitTree insert Exit tree into the DB
225func (hdb *HistoryDB) AddExitTree(exitTree []common.ExitInfo) error {
226	return hdb.addExitTree(hdb.db, exitTree)
227}
228func (hdb *HistoryDB) addExitTree(d meddler.DB, exitTree []common.ExitInfo) error {
229	return db.BulkInsert(
230		d,
231		"INSERT INTO exit_tree (batch_num, account_idx, merkle_proof, balance, "+
232			"instant_withdrawn, delayed_withdraw_request, delayed_withdrawn) VALUES %s;",
233		exitTree[:],
234	)
235}
236
237// AddToken insert a token into the DB
238func (hdb *HistoryDB) AddToken(token *common.Token) error {
239	return meddler.Insert(hdb.db, "token", token)
240}
241
242// AddTokens insert tokens into the DB
243func (hdb *HistoryDB) AddTokens(tokens []common.Token) error { return hdb.addTokens(hdb.db, tokens) }
244func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
245	return db.BulkInsert(
246		d,
247		`INSERT INTO token (
248			token_id,
249			eth_block_num,
250			eth_addr,
251			name,
252			symbol,
253			decimals,
254			usd,
255			usd_update
256		) VALUES %s;`,
257		tokens[:],
258	)
259}
260
261// UpdateTokenValue updates the USD value of a token
262func (hdb *HistoryDB) UpdateTokenValue(tokenID common.TokenID, value float64) error {
263	_, err := hdb.db.Exec(
264		"UPDATE token SET usd = $1 WHERE token_id = $2;",
265		value, tokenID,
266	)
267	return err
268}
269
270// GetTokens returns a list of tokens from the DB
271func (hdb *HistoryDB) GetTokens() ([]*common.Token, error) {
272	var tokens []*common.Token
273	err := meddler.QueryAll(
274		hdb.db, &tokens,
275		"SELECT * FROM token ORDER BY token_id;",
276	)
277	return tokens, err
278}
279
280// AddAccounts insert accounts into the DB
281func (hdb *HistoryDB) AddAccounts(accounts []common.Account) error {
282	return hdb.addAccounts(hdb.db, accounts)
283}
284func (hdb *HistoryDB) addAccounts(d meddler.DB, accounts []common.Account) error {
285	return db.BulkInsert(
286		d,
287		`INSERT INTO account (
288			idx,
289			token_id,
290			batch_num,
291			bjj,
292			eth_addr
293		) VALUES %s;`,
294		accounts[:],
295	)
296}
297
298// GetAccounts returns a list of accounts from the DB
299func (hdb *HistoryDB) GetAccounts() ([]*common.Account, error) {
300	var accs []*common.Account
301	err := meddler.QueryAll(
302		hdb.db, &accs,
303		"SELECT * FROM account ORDER BY idx;",
304	)
305	return accs, err
306}
307
308// AddL1Txs inserts L1 txs to the DB. USD and LoadAmountUSD will be set automatically before storing the tx.
309// If the tx is originated by a coordinator, BatchNum must be provided. If it's originated by a user,
310// BatchNum should be null, and the value will be setted by a trigger when a batch forges the tx.
311func (hdb *HistoryDB) AddL1Txs(l1txs []common.L1Tx) error { return hdb.addL1Txs(hdb.db, l1txs) }
312
313// addL1Txs inserts L1 txs to the DB. USD and LoadAmountUSD will be set automatically before storing the tx.
314// If the tx is originated by a coordinator, BatchNum must be provided. If it's originated by a user,
315// BatchNum should be null, and the value will be setted by a trigger when a batch forges the tx.
316func (hdb *HistoryDB) addL1Txs(d meddler.DB, l1txs []common.L1Tx) error {
317	txs := []common.Tx{}
318	for _, tx := range l1txs {
319		txs = append(txs, *(tx.Tx()))
320	}
321	return hdb.addTxs(d, txs)
322}
323
324// AddL2Txs inserts L2 txs to the DB. USD and FeeUSD will be set automatically before storing the tx.
325func (hdb *HistoryDB) AddL2Txs(l2txs []common.L2Tx) error { return hdb.addL2Txs(hdb.db, l2txs) }
326
327// addL2Txs inserts L2 txs to the DB. USD and FeeUSD will be set automatically before storing the tx.
328func (hdb *HistoryDB) addL2Txs(d meddler.DB, l2txs []common.L2Tx) error {
329	txs := []common.Tx{}
330	for _, tx := range l2txs {
331		txs = append(txs, *(tx.Tx()))
332	}
333	return hdb.addTxs(d, txs)
334}
335
336func (hdb *HistoryDB) addTxs(d meddler.DB, txs []common.Tx) error {
337	return db.BulkInsert(
338		d,
339		`INSERT INTO tx (
340			is_l1,
341			id,
342			type,
343			position,
344			from_idx,
345			to_idx,
346			amount,
347			amount_f,
348			token_id,
349			amount_usd,
350			batch_num,
351			eth_block_num,
352			to_forge_l1_txs_num,
353			user_origin,
354			from_eth_addr,
355			from_bjj,
356			load_amount,
357			load_amount_f,
358			load_amount_usd,
359			fee,
360			fee_usd,
361			nonce
362		) VALUES %s;`,
363		txs[:],
364	)
365}
366
367// GetTxs returns a list of txs from the DB
368func (hdb *HistoryDB) GetTxs() ([]*common.Tx, error) {
369	var txs []*common.Tx
370	err := meddler.QueryAll(
371		hdb.db, &txs,
372		`SELECT * FROM tx 
373		ORDER BY (batch_num, position) ASC`,
374	)
375	return txs, err
376}
377
378// GetHistoryTxs returns a list of txs from the DB using the HistoryTx struct
379func (hdb *HistoryDB) GetHistoryTxs(
380	ethAddr *ethCommon.Address, bjj *babyjub.PublicKey,
381	tokenID, idx, batchNum *uint, txType *common.TxType,
382	offset, limit *uint, last bool,
383) ([]*HistoryTx, int, error) {
384	if ethAddr != nil && bjj != nil {
385		return nil, 0, errors.New("ethAddr and bjj are incompatible")
386	}
387	var query string
388	var args []interface{}
389	queryStr := `SELECT tx.*, token.token_id, token.eth_block_num AS token_block,
390	token.eth_addr, token.name, token.symbol, token.decimals, token.usd,
391	token.usd_update, block.timestamp, count(*) OVER() AS total_items 
392	FROM tx 
393	INNER JOIN token ON tx.token_id = token.token_id 
394	INNER JOIN block ON tx.eth_block_num = block.eth_block_num `
395	// Apply filters
396	nextIsAnd := false
397	// ethAddr filter
398	if ethAddr != nil {
399		queryStr = `WITH acc AS 
400		(select idx from account where eth_addr = ?) ` + queryStr
401		queryStr += ", acc WHERE (tx.from_idx IN(acc.idx) OR tx.to_idx IN(acc.idx)) "
402		nextIsAnd = true
403		args = append(args, ethAddr)
404	} else if bjj != nil { // bjj filter
405		queryStr = `WITH acc AS 
406		(select idx from account where bjj = ?) ` + queryStr
407		queryStr += ", acc WHERE (tx.from_idx IN(acc.idx) OR tx.to_idx IN(acc.idx)) "
408		nextIsAnd = true
409		args = append(args, bjj)
410	}
411	// tokenID filter
412	if tokenID != nil {
413		if nextIsAnd {
414			queryStr += "AND "
415		} else {
416			queryStr += "WHERE "
417		}
418		queryStr += "tx.token_id = ? "
419		args = append(args, tokenID)
420		nextIsAnd = true
421	}
422	// idx filter
423	if idx != nil {
424		if nextIsAnd {
425			queryStr += "AND "
426		} else {
427			queryStr += "WHERE "
428		}
429		queryStr += "(tx.from_idx = ? OR tx.to_idx = ?) "
430		args = append(args, idx, idx)
431		nextIsAnd = true
432	}
433	// batchNum filter
434	if batchNum != nil {
435		if nextIsAnd {
436			queryStr += "AND "
437		} else {
438			queryStr += "WHERE "
439		}
440		queryStr += "tx.batch_num = ? "
441		args = append(args, batchNum)
442		nextIsAnd = true
443	}
444	// txType filter
445	if txType != nil {
446		if nextIsAnd {
447			queryStr += "AND "
448		} else {
449			queryStr += "WHERE "
450		}
451		queryStr += "tx.type = ? "
452		args = append(args, txType)
453		// nextIsAnd = true
454	}
455	// pagination
456	if last {
457		queryStr += "ORDER BY (batch_num, position) DESC NULLS FIRST "
458	} else {
459		queryStr += "ORDER BY (batch_num, position) ASC NULLS LAST "
460		queryStr += fmt.Sprintf("OFFSET %d ", *offset)
461	}
462	queryStr += fmt.Sprintf("LIMIT %d;", *limit)
463	query = hdb.db.Rebind(queryStr)
464	// log.Debug(query)
465	txs := []*HistoryTx{}
466	if err := meddler.QueryAll(hdb.db, &txs, query, args...); err != nil {
467		return nil, 0, err
468	}
469	if len(txs) == 0 {
470		return nil, 0, sql.ErrNoRows
471	} else if last {
472		tmp := []*HistoryTx{}
473		for i := len(txs) - 1; i >= 0; i-- {
474			tmp = append(tmp, txs[i])
475		}
476		txs = tmp
477	}
478	return txs, txs[0].TotalItems, nil
479}
480
481// GetTx returns a tx from the DB
482func (hdb *HistoryDB) GetTx(txID common.TxID) (*common.Tx, error) {
483	tx := new(common.Tx)
484	return tx, meddler.QueryRow(
485		hdb.db, tx,
486		"SELECT * FROM tx WHERE id = $1;",
487		txID,
488	)
489}
490
491// // GetL1UserTxs gets L1 User Txs to be forged in a batch that will create an account
492// // TODO: This is currently not used.  Figure out if it should be used somewhere or removed.
493// func (hdb *HistoryDB) GetL1UserTxs(toForgeL1TxsNum int64) ([]*common.Tx, error) {
494// 	var txs []*common.Tx
495// 	err := meddler.QueryAll(
496// 		hdb.db, &txs,
497// 		"SELECT * FROM tx WHERE to_forge_l1_txs_num = $1 AND is_l1 = TRUE AND user_origin = TRUE;",
498// 		toForgeL1TxsNum,
499// 	)
500// 	return txs, err
501// }
502
503// TODO: Think about chaning all the queries that return a last value, to queries that return the next valid value.
504
505// GetLastTxsPosition for a given to_forge_l1_txs_num
506func (hdb *HistoryDB) GetLastTxsPosition(toForgeL1TxsNum int64) (int, error) {
507	row := hdb.db.QueryRow("SELECT MAX(position) FROM tx WHERE to_forge_l1_txs_num = $1;", toForgeL1TxsNum)
508	var lastL1TxsPosition int
509	return lastL1TxsPosition, row.Scan(&lastL1TxsPosition)
510}
511
512// AddBlockSCData stores all the information of a block retrieved by the Synchronizer
513func (hdb *HistoryDB) AddBlockSCData(blockData *BlockData) (err error) {
514	txn, err := hdb.db.Begin()
515	if err != nil {
516		return err
517	}
518	defer func() {
519		if err != nil {
520			err = txn.Rollback()
521		}
522	}()
523
524	// Add block
525	err = hdb.addBlock(txn, blockData.block)
526	if err != nil {
527		return err
528	}
529
530	// Add l1 Txs
531	if len(blockData.L1UserTxs) > 0 {
532		err = hdb.addL1Txs(txn, blockData.L1UserTxs)
533		if err != nil {
534			return err
535		}
536	}
537
538	// Add Tokens
539	if len(blockData.RegisteredTokens) > 0 {
540		err = hdb.addTokens(txn, blockData.RegisteredTokens)
541		if err != nil {
542			return err
543		}
544	}
545
546	// Add Bids
547	if len(blockData.Bids) > 0 {
548		err = hdb.addBids(txn, blockData.Bids)
549		if err != nil {
550			return err
551		}
552	}
553
554	// Add Coordinators
555	if len(blockData.Coordinators) > 0 {
556		err = hdb.addCoordinators(txn, blockData.Coordinators)
557		if err != nil {
558			return err
559		}
560	}
561
562	// Add Batches
563	for _, batch := range blockData.Batches {
564		// Add Batch: this will trigger an update on the DB
565		// that will set the batch num of forged L1 txs in this batch
566		err = hdb.addBatch(txn, batch.Batch)
567		if err != nil {
568			return err
569		}
570
571		// Add unforged l1 Txs
572		if batch.L1Batch {
573			if len(batch.L1CoordinatorTxs) > 0 {
574				err = hdb.addL1Txs(txn, batch.L1CoordinatorTxs)
575				if err != nil {
576					return err
577				}
578			}
579		}
580
581		// Add l2 Txs
582		if len(batch.L2Txs) > 0 {
583			err = hdb.addL2Txs(txn, batch.L2Txs)
584			if err != nil {
585				return err
586			}
587		}
588
589		// Add accounts
590		if len(batch.CreatedAccounts) > 0 {
591			err = hdb.addAccounts(txn, batch.CreatedAccounts)
592			if err != nil {
593				return err
594			}
595		}
596
597		// Add exit tree
598		if len(batch.ExitTree) > 0 {
599			err = hdb.addExitTree(txn, batch.ExitTree)
600			if err != nil {
601				return err
602			}
603		}
604
605		// TODO: INSERT CONTRACTS VARS
606	}
607
608	return txn.Commit()
609}