1package historydb
  2
  3import (
  4	"database/sql"
  5	"errors"
  6	"fmt"
  7
  8	ethCommon "github.com/ethereum/go-ethereum/common"
  9	"github.com/hermeznetwork/hermez-node/common"
 10	"github.com/hermeznetwork/hermez-node/db"
 11	"github.com/iden3/go-iden3-crypto/babyjub"
 12	"github.com/jmoiron/sqlx"
 13
 14	//nolint:errcheck // driver for postgres DB
 15	_ "github.com/lib/pq"
 16	"github.com/russross/meddler"
 17)
 18
 19// TODO(Edu): Document here how HistoryDB is kept consistent
 20
 21// HistoryDB persist the historic of the rollup
 22type HistoryDB struct {
 23	db *sqlx.DB
 24}
 25
 26// BlockData contains the information of a Block
 27type BlockData struct {
 28	block *common.Block
 29	// Rollup
 30	// L1UserTxs that were submitted in the block
 31	L1UserTxs        []common.L1Tx
 32	Batches          []BatchData
 33	RegisteredTokens []common.Token
 34	RollupVars       *common.RollupVars
 35	// Auction
 36	Bids         []common.Bid
 37	Coordinators []common.Coordinator
 38	AuctionVars  *common.AuctionVars
 39	// WithdrawalDelayer
 40	// TODO: enable when common.WithdrawalDelayerVars is Merged from Synchronizer PR
 41	// WithdrawalDelayerVars *common.WithdrawalDelayerVars
 42}
 43
 44// BatchData contains the information of a Batch
 45type BatchData struct {
 46	// L1UserTxs that were forged in the batch
 47	L1Batch          bool // TODO: Remove once Batch.ForgeL1TxsNum is a pointer
 48	L1UserTxs        []common.L1Tx
 49	L1CoordinatorTxs []common.L1Tx
 50	L2Txs            []common.L2Tx
 51	CreatedAccounts  []common.Account
 52	ExitTree         []common.ExitInfo
 53	Batch            *common.Batch
 54}
 55
 56// NewHistoryDB initialize the DB
 57func NewHistoryDB(db *sqlx.DB) *HistoryDB {
 58	return &HistoryDB{db: db}
 59}
 60
 61// AddBlock insert a block into the DB
 62func (hdb *HistoryDB) AddBlock(block *common.Block) error { return hdb.addBlock(hdb.db, block) }
 63func (hdb *HistoryDB) addBlock(d meddler.DB, block *common.Block) error {
 64	return meddler.Insert(d, "block", block)
 65}
 66
 67// AddBlocks inserts blocks into the DB
 68func (hdb *HistoryDB) AddBlocks(blocks []common.Block) error {
 69	return hdb.addBlocks(hdb.db, blocks)
 70}
 71
 72func (hdb *HistoryDB) addBlocks(d meddler.DB, blocks []common.Block) error {
 73	return db.BulkInsert(
 74		d,
 75		`INSERT INTO block (
 76			eth_block_num,
 77			timestamp,
 78			hash
 79		) VALUES %s;`,
 80		blocks[:],
 81	)
 82}
 83
 84// GetBlock retrieve a block from the DB, given a block number
 85func (hdb *HistoryDB) GetBlock(blockNum int64) (*common.Block, error) {
 86	block := &common.Block{}
 87	err := meddler.QueryRow(
 88		hdb.db, block,
 89		"SELECT * FROM block WHERE eth_block_num = $1;", blockNum,
 90	)
 91	return block, err
 92}
 93
 94// GetBlocks retrieve blocks from the DB, given a range of block numbers defined by from and to
 95func (hdb *HistoryDB) GetBlocks(from, to int64) ([]*common.Block, error) {
 96	var blocks []*common.Block
 97	err := meddler.QueryAll(
 98		hdb.db, &blocks,
 99		"SELECT * FROM block WHERE $1 <= eth_block_num AND eth_block_num < $2;",
100		from, to,
101	)
102	return blocks, err
103}
104
105// GetLastBlock retrieve the block with the highest block number from the DB
106func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) {
107	block := &common.Block{}
108	err := meddler.QueryRow(
109		hdb.db, block, "SELECT * FROM block ORDER BY eth_block_num DESC LIMIT 1;",
110	)
111	return block, err
112}
113
114// AddBatch insert a Batch into the DB
115func (hdb *HistoryDB) AddBatch(batch *common.Batch) error { return hdb.addBatch(hdb.db, batch) }
116func (hdb *HistoryDB) addBatch(d meddler.DB, batch *common.Batch) error {
117	return meddler.Insert(d, "batch", batch)
118}
119
120// AddBatches insert Bids into the DB
121func (hdb *HistoryDB) AddBatches(batches []common.Batch) error {
122	return hdb.addBatches(hdb.db, batches)
123}
124func (hdb *HistoryDB) addBatches(d meddler.DB, batches []common.Batch) error {
125	return db.BulkInsert(
126		d,
127		`INSERT INTO batch (
128			batch_num,
129			eth_block_num,
130			forger_addr,
131			fees_collected,
132			state_root,
133			num_accounts,
134			exit_root,
135			forge_l1_txs_num,
136			slot_num
137		) VALUES %s;`,
138		batches[:],
139	)
140}
141
142// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to
143func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]*common.Batch, error) {
144	var batches []*common.Batch
145	err := meddler.QueryAll(
146		hdb.db, &batches,
147		"SELECT * FROM batch WHERE $1 <= batch_num AND batch_num < $2;",
148		from, to,
149	)
150	return batches, err
151}
152
153// GetLastBatchNum returns the BatchNum of the latest forged batch
154func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) {
155	row := hdb.db.QueryRow("SELECT batch_num FROM batch ORDER BY batch_num DESC LIMIT 1;")
156	var batchNum common.BatchNum
157	return batchNum, row.Scan(&batchNum)
158}
159
160// GetLastL1TxsNum returns the greatest ForgeL1TxsNum in the DB.  If there's no
161// batch in the DB (nil, nil) is returned.
162func (hdb *HistoryDB) GetLastL1TxsNum() (*int64, error) {
163	row := hdb.db.QueryRow("SELECT MAX(forge_l1_txs_num) FROM batch;")
164	lastL1TxsNum := new(int64)
165	return lastL1TxsNum, row.Scan(&lastL1TxsNum)
166}
167
168// Reorg deletes all the information that was added into the DB after the
169// lastValidBlock.  If lastValidBlock is negative, all block information is
170// deleted.
171func (hdb *HistoryDB) Reorg(lastValidBlock int64) error {
172	var err error
173	if lastValidBlock < 0 {
174		_, err = hdb.db.Exec("DELETE FROM block;")
175	} else {
176		_, err = hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock)
177	}
178	return err
179}
180
181// SyncPoD stores all the data that can be changed / added on a block in the PoD SC
182func (hdb *HistoryDB) SyncPoD(
183	blockNum uint64,
184	bids []common.Bid,
185	coordinators []common.Coordinator,
186	vars *common.AuctionVars,
187) error {
188	return nil
189}
190
191// AddBids insert Bids into the DB
192func (hdb *HistoryDB) AddBids(bids []common.Bid) error { return hdb.addBids(hdb.db, bids) }
193func (hdb *HistoryDB) addBids(d meddler.DB, bids []common.Bid) error {
194	// TODO: check the coordinator info
195	return db.BulkInsert(
196		d,
197		"INSERT INTO bid (slot_num, forger_addr, bid_value, eth_block_num) VALUES %s;",
198		bids[:],
199	)
200}
201
202// GetBids return the bids
203func (hdb *HistoryDB) GetBids() ([]*common.Bid, error) {
204	var bids []*common.Bid
205	err := meddler.QueryAll(
206		hdb.db, &bids,
207		"SELECT * FROM bid;",
208	)
209	return bids, err
210}
211
212// AddCoordinators insert Coordinators into the DB
213func (hdb *HistoryDB) AddCoordinators(coordinators []common.Coordinator) error {
214	return hdb.addCoordinators(hdb.db, coordinators)
215}
216func (hdb *HistoryDB) addCoordinators(d meddler.DB, coordinators []common.Coordinator) error {
217	return db.BulkInsert(
218		d,
219		"INSERT INTO coordinator (forger_addr, eth_block_num, withdraw_addr, url) VALUES %s;",
220		coordinators[:],
221	)
222}
223
224// AddExitTree insert Exit tree into the DB
225func (hdb *HistoryDB) AddExitTree(exitTree []common.ExitInfo) error {
226	return hdb.addExitTree(hdb.db, exitTree)
227}
228func (hdb *HistoryDB) addExitTree(d meddler.DB, exitTree []common.ExitInfo) error {
229	return db.BulkInsert(
230		d,
231		"INSERT INTO exit_tree (batch_num, account_idx, merkle_proof, balance, "+
232			"instant_withdrawn, delayed_withdraw_request, delayed_withdrawn) VALUES %s;",
233		exitTree[:],
234	)
235}
236
237// AddToken insert a token into the DB
238func (hdb *HistoryDB) AddToken(token *common.Token) error {
239	return meddler.Insert(hdb.db, "token", token)
240}
241
242// AddTokens insert tokens into the DB
243func (hdb *HistoryDB) AddTokens(tokens []common.Token) error { return hdb.addTokens(hdb.db, tokens) }
244func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
245	return db.BulkInsert(
246		d,
247		`INSERT INTO token (
248			token_id,
249			eth_block_num,
250			eth_addr,
251			name,
252			symbol,
253			decimals,
254			usd,
255			usd_update
256		) VALUES %s;`,
257		tokens[:],
258	)
259}
260
261// UpdateTokenValue updates the USD value of a token
262func (hdb *HistoryDB) UpdateTokenValue(tokenSymbol string, value float64) error {
263	_, err := hdb.db.Exec(
264		"UPDATE token SET usd = $1 WHERE symbol = $2;",
265		value, tokenSymbol,
266	)
267	return err
268}
269
270// GetTokens returns a list of tokens from the DB
271func (hdb *HistoryDB) GetTokens() ([]*common.Token, error) {
272	var tokens []*common.Token
273	err := meddler.QueryAll(
274		hdb.db, &tokens,
275		"SELECT * FROM token ORDER BY token_id;",
276	)
277	return tokens, err
278}
279
280// GetTokenSymbols returns all the token symbols from the DB
281func (hdb *HistoryDB) GetTokenSymbols() ([]string, error) {
282	var tokenSymbols []string
283	rows, err := hdb.db.Query("SELECT symbol FROM token;")
284	if err != nil {
285		return nil, err
286	}
287	sym := new(string)
288	for rows.Next() {
289		err = rows.Scan(sym)
290		if err != nil {
291			return nil, err
292		}
293		tokenSymbols = append(tokenSymbols, *sym)
294	}
295	return tokenSymbols, nil
296}
297
298// AddAccounts insert accounts into the DB
299func (hdb *HistoryDB) AddAccounts(accounts []common.Account) error {
300	return hdb.addAccounts(hdb.db, accounts)
301}
302func (hdb *HistoryDB) addAccounts(d meddler.DB, accounts []common.Account) error {
303	return db.BulkInsert(
304		d,
305		`INSERT INTO account (
306			idx,
307			token_id,
308			batch_num,
309			bjj,
310			eth_addr
311		) VALUES %s;`,
312		accounts[:],
313	)
314}
315
316// GetAccounts returns a list of accounts from the DB
317func (hdb *HistoryDB) GetAccounts() ([]*common.Account, error) {
318	var accs []*common.Account
319	err := meddler.QueryAll(
320		hdb.db, &accs,
321		"SELECT * FROM account ORDER BY idx;",
322	)
323	return accs, err
324}
325
326// AddL1Txs inserts L1 txs to the DB. USD and LoadAmountUSD will be set automatically before storing the tx.
327// If the tx is originated by a coordinator, BatchNum must be provided. If it's originated by a user,
328// BatchNum should be null, and the value will be setted by a trigger when a batch forges the tx.
329func (hdb *HistoryDB) AddL1Txs(l1txs []common.L1Tx) error { return hdb.addL1Txs(hdb.db, l1txs) }
330
331// addL1Txs inserts L1 txs to the DB. USD and LoadAmountUSD will be set automatically before storing the tx.
332// If the tx is originated by a coordinator, BatchNum must be provided. If it's originated by a user,
333// BatchNum should be null, and the value will be setted by a trigger when a batch forges the tx.
334func (hdb *HistoryDB) addL1Txs(d meddler.DB, l1txs []common.L1Tx) error {
335	txs := []common.Tx{}
336	for _, tx := range l1txs {
337		txs = append(txs, *(tx.Tx()))
338	}
339	return hdb.addTxs(d, txs)
340}
341
342// AddL2Txs inserts L2 txs to the DB. USD and FeeUSD will be set automatically before storing the tx.
343func (hdb *HistoryDB) AddL2Txs(l2txs []common.L2Tx) error { return hdb.addL2Txs(hdb.db, l2txs) }
344
345// addL2Txs inserts L2 txs to the DB. USD and FeeUSD will be set automatically before storing the tx.
346func (hdb *HistoryDB) addL2Txs(d meddler.DB, l2txs []common.L2Tx) error {
347	txs := []common.Tx{}
348	for _, tx := range l2txs {
349		txs = append(txs, *(tx.Tx()))
350	}
351	return hdb.addTxs(d, txs)
352}
353
354func (hdb *HistoryDB) addTxs(d meddler.DB, txs []common.Tx) error {
355	return db.BulkInsert(
356		d,
357		`INSERT INTO tx (
358			is_l1,
359			id,
360			type,
361			position,
362			from_idx,
363			to_idx,
364			amount,
365			amount_f,
366			token_id,
367			amount_usd,
368			batch_num,
369			eth_block_num,
370			to_forge_l1_txs_num,
371			user_origin,
372			from_eth_addr,
373			from_bjj,
374			load_amount,
375			load_amount_f,
376			load_amount_usd,
377			fee,
378			fee_usd,
379			nonce
380		) VALUES %s;`,
381		txs[:],
382	)
383}
384
385// GetTxs returns a list of txs from the DB
386func (hdb *HistoryDB) GetTxs() ([]*common.Tx, error) {
387	var txs []*common.Tx
388	err := meddler.QueryAll(
389		hdb.db, &txs,
390		`SELECT * FROM tx 
391		ORDER BY (batch_num, position) ASC`,
392	)
393	return txs, err
394}
395
396// GetHistoryTxs returns a list of txs from the DB using the HistoryTx struct
397func (hdb *HistoryDB) GetHistoryTxs(
398	ethAddr *ethCommon.Address, bjj *babyjub.PublicKey,
399	tokenID, idx, batchNum *uint, txType *common.TxType,
400	offset, limit *uint, last bool,
401) ([]*HistoryTx, int, error) {
402	if ethAddr != nil && bjj != nil {
403		return nil, 0, errors.New("ethAddr and bjj are incompatible")
404	}
405	var query string
406	var args []interface{}
407	queryStr := `SELECT tx.*, token.token_id, token.eth_block_num AS token_block,
408	token.eth_addr, token.name, token.symbol, token.decimals, token.usd,
409	token.usd_update, block.timestamp, count(*) OVER() AS total_items 
410	FROM tx 
411	INNER JOIN token ON tx.token_id = token.token_id 
412	INNER JOIN block ON tx.eth_block_num = block.eth_block_num `
413	// Apply filters
414	nextIsAnd := false
415	// ethAddr filter
416	if ethAddr != nil {
417		queryStr = `WITH acc AS 
418		(select idx from account where eth_addr = ?) ` + queryStr
419		queryStr += ", acc WHERE (tx.from_idx IN(acc.idx) OR tx.to_idx IN(acc.idx)) "
420		nextIsAnd = true
421		args = append(args, ethAddr)
422	} else if bjj != nil { // bjj filter
423		queryStr = `WITH acc AS 
424		(select idx from account where bjj = ?) ` + queryStr
425		queryStr += ", acc WHERE (tx.from_idx IN(acc.idx) OR tx.to_idx IN(acc.idx)) "
426		nextIsAnd = true
427		args = append(args, bjj)
428	}
429	// tokenID filter
430	if tokenID != nil {
431		if nextIsAnd {
432			queryStr += "AND "
433		} else {
434			queryStr += "WHERE "
435		}
436		queryStr += "tx.token_id = ? "
437		args = append(args, tokenID)
438		nextIsAnd = true
439	}
440	// idx filter
441	if idx != nil {
442		if nextIsAnd {
443			queryStr += "AND "
444		} else {
445			queryStr += "WHERE "
446		}
447		queryStr += "(tx.from_idx = ? OR tx.to_idx = ?) "
448		args = append(args, idx, idx)
449		nextIsAnd = true
450	}
451	// batchNum filter
452	if batchNum != nil {
453		if nextIsAnd {
454			queryStr += "AND "
455		} else {
456			queryStr += "WHERE "
457		}
458		queryStr += "tx.batch_num = ? "
459		args = append(args, batchNum)
460		nextIsAnd = true
461	}
462	// txType filter
463	if txType != nil {
464		if nextIsAnd {
465			queryStr += "AND "
466		} else {
467			queryStr += "WHERE "
468		}
469		queryStr += "tx.type = ? "
470		args = append(args, txType)
471		// nextIsAnd = true
472	}
473	// pagination
474	if last {
475		queryStr += "ORDER BY (batch_num, position) DESC NULLS FIRST "
476	} else {
477		queryStr += "ORDER BY (batch_num, position) ASC NULLS LAST "
478		queryStr += fmt.Sprintf("OFFSET %d ", *offset)
479	}
480	queryStr += fmt.Sprintf("LIMIT %d;", *limit)
481	query = hdb.db.Rebind(queryStr)
482	// log.Debug(query)
483	txs := []*HistoryTx{}
484	if err := meddler.QueryAll(hdb.db, &txs, query, args...); err != nil {
485		return nil, 0, err
486	}
487	if len(txs) == 0 {
488		return nil, 0, sql.ErrNoRows
489	} else if last {
490		tmp := []*HistoryTx{}
491		for i := len(txs) - 1; i >= 0; i-- {
492			tmp = append(tmp, txs[i])
493		}
494		txs = tmp
495	}
496	return txs, txs[0].TotalItems, nil
497}
498
499// GetTx returns a tx from the DB
500func (hdb *HistoryDB) GetTx(txID common.TxID) (*common.Tx, error) {
501	tx := new(common.Tx)
502	return tx, meddler.QueryRow(
503		hdb.db, tx,
504		"SELECT * FROM tx WHERE id = $1;",
505		txID,
506	)
507}
508
509// // GetL1UserTxs gets L1 User Txs to be forged in a batch that will create an account
510// // TODO: This is currently not used.  Figure out if it should be used somewhere or removed.
511// func (hdb *HistoryDB) GetL1UserTxs(toForgeL1TxsNum int64) ([]*common.Tx, error) {
512// 	var txs []*common.Tx
513// 	err := meddler.QueryAll(
514// 		hdb.db, &txs,
515// 		"SELECT * FROM tx WHERE to_forge_l1_txs_num = $1 AND is_l1 = TRUE AND user_origin = TRUE;",
516// 		toForgeL1TxsNum,
517// 	)
518// 	return txs, err
519// }
520
521// TODO: Think about chaning all the queries that return a last value, to queries that return the next valid value.
522
523// GetLastTxsPosition for a given to_forge_l1_txs_num
524func (hdb *HistoryDB) GetLastTxsPosition(toForgeL1TxsNum int64) (int, error) {
525	row := hdb.db.QueryRow("SELECT MAX(position) FROM tx WHERE to_forge_l1_txs_num = $1;", toForgeL1TxsNum)
526	var lastL1TxsPosition int
527	return lastL1TxsPosition, row.Scan(&lastL1TxsPosition)
528}
529
530// AddBlockSCData stores all the information of a block retrieved by the Synchronizer
531func (hdb *HistoryDB) AddBlockSCData(blockData *BlockData) (err error) {
532	txn, err := hdb.db.Begin()
533	if err != nil {
534		return err
535	}
536	defer func() {
537		if err != nil {
538			err = txn.Rollback()
539		}
540	}()
541
542	// Add block
543	err = hdb.addBlock(txn, blockData.block)
544	if err != nil {
545		return err
546	}
547
548	// Add l1 Txs
549	if len(blockData.L1UserTxs) > 0 {
550		err = hdb.addL1Txs(txn, blockData.L1UserTxs)
551		if err != nil {
552			return err
553		}
554	}
555
556	// Add Tokens
557	if len(blockData.RegisteredTokens) > 0 {
558		err = hdb.addTokens(txn, blockData.RegisteredTokens)
559		if err != nil {
560			return err
561		}
562	}
563
564	// Add Bids
565	if len(blockData.Bids) > 0 {
566		err = hdb.addBids(txn, blockData.Bids)
567		if err != nil {
568			return err
569		}
570	}
571
572	// Add Coordinators
573	if len(blockData.Coordinators) > 0 {
574		err = hdb.addCoordinators(txn, blockData.Coordinators)
575		if err != nil {
576			return err
577		}
578	}
579
580	// Add Batches
581	for _, batch := range blockData.Batches {
582		// Add Batch: this will trigger an update on the DB
583		// that will set the batch num of forged L1 txs in this batch
584		err = hdb.addBatch(txn, batch.Batch)
585		if err != nil {
586			return err
587		}
588
589		// Add unforged l1 Txs
590		if batch.L1Batch {
591			if len(batch.L1CoordinatorTxs) > 0 {
592				err = hdb.addL1Txs(txn, batch.L1CoordinatorTxs)
593				if err != nil {
594					return err
595				}
596			}
597		}
598
599		// Add l2 Txs
600		if len(batch.L2Txs) > 0 {
601			err = hdb.addL2Txs(txn, batch.L2Txs)
602			if err != nil {
603				return err
604			}
605		}
606
607		// Add accounts
608		if len(batch.CreatedAccounts) > 0 {
609			err = hdb.addAccounts(txn, batch.CreatedAccounts)
610			if err != nil {
611				return err
612			}
613		}
614
615		// Add exit tree
616		if len(batch.ExitTree) > 0 {
617			err = hdb.addExitTree(txn, batch.ExitTree)
618			if err != nil {
619				return err
620			}
621		}
622
623		// TODO: INSERT CONTRACTS VARS
624	}
625
626	return txn.Commit()
627}