hermez-node/db/historydb/historydb.go
db/historydb/historydb.goRaw
1package historydb
2
3import (
4 "math"
5 "math/big"
6 "strings"
7
8 ethCommon "github.com/ethereum/go-ethereum/common"
9 "github.com/hermeznetwork/hermez-node/common"
10 "github.com/hermeznetwork/hermez-node/db"
11 "github.com/hermeznetwork/tracerr"
12 "github.com/jmoiron/sqlx"
13
14 //nolint:errcheck // driver for postgres DB
15 _ "github.com/lib/pq"
16 "github.com/russross/meddler"
17)
18
19const (
20 // OrderAsc indicates ascending order when using pagination
21 OrderAsc = "ASC"
22 // OrderDesc indicates descending order when using pagination
23 OrderDesc = "DESC"
24)
25
26// TODO(Edu): Document here how HistoryDB is kept consistent
27
28// HistoryDB persist the historic of the rollup
29type HistoryDB struct {
30 db *sqlx.DB
31 apiConnCon *db.APIConnectionController
32}
33
34// NewHistoryDB initialize the DB
35func NewHistoryDB(db *sqlx.DB, apiConnCon *db.APIConnectionController) *HistoryDB {
36 return &HistoryDB{db: db, apiConnCon: apiConnCon}
37}
38
39// DB returns a pointer to the L2DB.db. This method should be used only for
40// internal testing purposes.
41func (hdb *HistoryDB) DB() *sqlx.DB {
42 return hdb.db
43}
44
45// AddBlock insert a block into the DB
46func (hdb *HistoryDB) AddBlock(block *common.Block) error { return hdb.addBlock(hdb.db, block) }
47func (hdb *HistoryDB) addBlock(d meddler.DB, block *common.Block) error {
48 return tracerr.Wrap(meddler.Insert(d, "block", block))
49}
50
51// AddBlocks inserts blocks into the DB
52func (hdb *HistoryDB) AddBlocks(blocks []common.Block) error {
53 return tracerr.Wrap(hdb.addBlocks(hdb.db, blocks))
54}
55
56func (hdb *HistoryDB) addBlocks(d meddler.DB, blocks []common.Block) error {
57 return tracerr.Wrap(db.BulkInsert(
58 d,
59 `INSERT INTO block (
60 eth_block_num,
61 timestamp,
62 hash
63 ) VALUES %s;`,
64 blocks[:],
65 ))
66}
67
68// GetBlock retrieve a block from the DB, given a block number
69func (hdb *HistoryDB) GetBlock(blockNum int64) (*common.Block, error) {
70 block := &common.Block{}
71 err := meddler.QueryRow(
72 hdb.db, block,
73 "SELECT * FROM block WHERE eth_block_num = $1;", blockNum,
74 )
75 return block, tracerr.Wrap(err)
76}
77
78// GetAllBlocks retrieve all blocks from the DB
79func (hdb *HistoryDB) GetAllBlocks() ([]common.Block, error) {
80 var blocks []*common.Block
81 err := meddler.QueryAll(
82 hdb.db, &blocks,
83 "SELECT * FROM block ORDER BY eth_block_num;",
84 )
85 return db.SlicePtrsToSlice(blocks).([]common.Block), tracerr.Wrap(err)
86}
87
88// getBlocks retrieve blocks from the DB, given a range of block numbers defined by from and to
89func (hdb *HistoryDB) getBlocks(from, to int64) ([]common.Block, error) {
90 var blocks []*common.Block
91 err := meddler.QueryAll(
92 hdb.db, &blocks,
93 "SELECT * FROM block WHERE $1 <= eth_block_num AND eth_block_num < $2 ORDER BY eth_block_num;",
94 from, to,
95 )
96 return db.SlicePtrsToSlice(blocks).([]common.Block), tracerr.Wrap(err)
97}
98
99// GetLastBlock retrieve the block with the highest block number from the DB
100func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) {
101 block := &common.Block{}
102 err := meddler.QueryRow(
103 hdb.db, block, "SELECT * FROM block ORDER BY eth_block_num DESC LIMIT 1;",
104 )
105 return block, tracerr.Wrap(err)
106}
107
108// AddBatch insert a Batch into the DB
109func (hdb *HistoryDB) AddBatch(batch *common.Batch) error { return hdb.addBatch(hdb.db, batch) }
110func (hdb *HistoryDB) addBatch(d meddler.DB, batch *common.Batch) error {
111 // Calculate total collected fees in USD
112 // Get IDs of collected tokens for fees
113 tokenIDs := []common.TokenID{}
114 for id := range batch.CollectedFees {
115 tokenIDs = append(tokenIDs, id)
116 }
117 // Get USD value of the tokens
118 type tokenPrice struct {
119 ID common.TokenID `meddler:"token_id"`
120 USD *float64 `meddler:"usd"`
121 Decimals int `meddler:"decimals"`
122 }
123 var tokenPrices []*tokenPrice
124 if len(tokenIDs) > 0 {
125 query, args, err := sqlx.In(
126 "SELECT token_id, usd, decimals FROM token WHERE token_id IN (?);",
127 tokenIDs,
128 )
129 if err != nil {
130 return tracerr.Wrap(err)
131 }
132 query = hdb.db.Rebind(query)
133 if err := meddler.QueryAll(
134 hdb.db, &tokenPrices, query, args...,
135 ); err != nil {
136 return tracerr.Wrap(err)
137 }
138 }
139 // Calculate total collected
140 var total float64
141 for _, tokenPrice := range tokenPrices {
142 if tokenPrice.USD == nil {
143 continue
144 }
145 f := new(big.Float).SetInt(batch.CollectedFees[tokenPrice.ID])
146 amount, _ := f.Float64()
147 total += *tokenPrice.USD * (amount / math.Pow(10, float64(tokenPrice.Decimals))) //nolint decimals have to be ^10
148 }
149 batch.TotalFeesUSD = &total
150 // Insert to DB
151 return tracerr.Wrap(meddler.Insert(d, "batch", batch))
152}
153
154// AddBatches insert Bids into the DB
155func (hdb *HistoryDB) AddBatches(batches []common.Batch) error {
156 return tracerr.Wrap(hdb.addBatches(hdb.db, batches))
157}
158func (hdb *HistoryDB) addBatches(d meddler.DB, batches []common.Batch) error {
159 for i := 0; i < len(batches); i++ {
160 if err := hdb.addBatch(d, &batches[i]); err != nil {
161 return tracerr.Wrap(err)
162 }
163 }
164 return nil
165}
166
167// GetAllBatches retrieve all batches from the DB
168func (hdb *HistoryDB) GetAllBatches() ([]common.Batch, error) {
169 var batches []*common.Batch
170 err := meddler.QueryAll(
171 hdb.db, &batches,
172 `SELECT batch.batch_num, batch.eth_block_num, batch.forger_addr, batch.fees_collected,
173 batch.fee_idxs_coordinator, batch.state_root, batch.num_accounts, batch.last_idx, batch.exit_root,
174 batch.forge_l1_txs_num, batch.slot_num, batch.total_fees_usd FROM batch
175 ORDER BY item_id;`,
176 )
177 return db.SlicePtrsToSlice(batches).([]common.Batch), tracerr.Wrap(err)
178}
179
180// GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to
181func (hdb *HistoryDB) GetBatches(from, to common.BatchNum) ([]common.Batch, error) {
182 var batches []*common.Batch
183 err := meddler.QueryAll(
184 hdb.db, &batches,
185 `SELECT batch_num, eth_block_num, forger_addr, fees_collected, fee_idxs_coordinator,
186 state_root, num_accounts, last_idx, exit_root, forge_l1_txs_num, slot_num, total_fees_usd
187 FROM batch WHERE $1 <= batch_num AND batch_num < $2 ORDER BY batch_num;`,
188 from, to,
189 )
190 return db.SlicePtrsToSlice(batches).([]common.Batch), tracerr.Wrap(err)
191}
192
193// GetFirstBatchBlockNumBySlot returns the ethereum block number of the first
194// batch within a slot
195func (hdb *HistoryDB) GetFirstBatchBlockNumBySlot(slotNum int64) (int64, error) {
196 row := hdb.db.QueryRow(
197 `SELECT eth_block_num FROM batch
198 WHERE slot_num = $1 ORDER BY batch_num ASC LIMIT 1;`, slotNum,
199 )
200 var blockNum int64
201 return blockNum, tracerr.Wrap(row.Scan(&blockNum))
202}
203
204// GetLastBatchNum returns the BatchNum of the latest forged batch
205func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) {
206 row := hdb.db.QueryRow("SELECT batch_num FROM batch ORDER BY batch_num DESC LIMIT 1;")
207 var batchNum common.BatchNum
208 return batchNum, tracerr.Wrap(row.Scan(&batchNum))
209}
210
211// GetLastL1BatchBlockNum returns the blockNum of the latest forged l1Batch
212func (hdb *HistoryDB) GetLastL1BatchBlockNum() (int64, error) {
213 row := hdb.db.QueryRow(`SELECT eth_block_num FROM batch
214 WHERE forge_l1_txs_num IS NOT NULL
215 ORDER BY batch_num DESC LIMIT 1;`)
216 var blockNum int64
217 return blockNum, tracerr.Wrap(row.Scan(&blockNum))
218}
219
220// GetLastL1TxsNum returns the greatest ForgeL1TxsNum in the DB from forged
221// batches. If there's no batch in the DB (nil, nil) is returned.
222func (hdb *HistoryDB) GetLastL1TxsNum() (*int64, error) {
223 row := hdb.db.QueryRow("SELECT MAX(forge_l1_txs_num) FROM batch;")
224 lastL1TxsNum := new(int64)
225 return lastL1TxsNum, tracerr.Wrap(row.Scan(&lastL1TxsNum))
226}
227
228// Reorg deletes all the information that was added into the DB after the
229// lastValidBlock. If lastValidBlock is negative, all block information is
230// deleted.
231func (hdb *HistoryDB) Reorg(lastValidBlock int64) error {
232 var err error
233 if lastValidBlock < 0 {
234 _, err = hdb.db.Exec("DELETE FROM block;")
235 } else {
236 _, err = hdb.db.Exec("DELETE FROM block WHERE eth_block_num > $1;", lastValidBlock)
237 }
238 return tracerr.Wrap(err)
239}
240
241// AddBids insert Bids into the DB
242func (hdb *HistoryDB) AddBids(bids []common.Bid) error { return hdb.addBids(hdb.db, bids) }
243func (hdb *HistoryDB) addBids(d meddler.DB, bids []common.Bid) error {
244 if len(bids) == 0 {
245 return nil
246 }
247 // TODO: check the coordinator info
248 return tracerr.Wrap(db.BulkInsert(
249 d,
250 "INSERT INTO bid (slot_num, bid_value, eth_block_num, bidder_addr) VALUES %s;",
251 bids[:],
252 ))
253}
254
255// GetAllBids retrieve all bids from the DB
256func (hdb *HistoryDB) GetAllBids() ([]common.Bid, error) {
257 var bids []*common.Bid
258 err := meddler.QueryAll(
259 hdb.db, &bids,
260 `SELECT bid.slot_num, bid.bid_value, bid.eth_block_num, bid.bidder_addr FROM bid
261 ORDER BY item_id;`,
262 )
263 return db.SlicePtrsToSlice(bids).([]common.Bid), tracerr.Wrap(err)
264}
265
266// GetBestBidCoordinator returns the forger address of the highest bidder in a slot by slotNum
267func (hdb *HistoryDB) GetBestBidCoordinator(slotNum int64) (*common.BidCoordinator, error) {
268 bidCoord := &common.BidCoordinator{}
269 err := meddler.QueryRow(
270 hdb.db, bidCoord,
271 `SELECT (
272 SELECT default_slot_set_bid
273 FROM auction_vars
274 WHERE default_slot_set_bid_slot_num <= $1
275 ORDER BY eth_block_num DESC LIMIT 1
276 ),
277 bid.slot_num, bid.bid_value, bid.bidder_addr,
278 coordinator.forger_addr, coordinator.url
279 FROM bid
280 INNER JOIN (
281 SELECT bidder_addr, MAX(item_id) AS item_id FROM coordinator
282 GROUP BY bidder_addr
283 ) c ON bid.bidder_addr = c.bidder_addr
284 INNER JOIN coordinator ON c.item_id = coordinator.item_id
285 WHERE bid.slot_num = $1 ORDER BY bid.item_id DESC LIMIT 1;`,
286 slotNum)
287
288 return bidCoord, tracerr.Wrap(err)
289}
290
291// AddCoordinators insert Coordinators into the DB
292func (hdb *HistoryDB) AddCoordinators(coordinators []common.Coordinator) error {
293 return tracerr.Wrap(hdb.addCoordinators(hdb.db, coordinators))
294}
295func (hdb *HistoryDB) addCoordinators(d meddler.DB, coordinators []common.Coordinator) error {
296 if len(coordinators) == 0 {
297 return nil
298 }
299 return tracerr.Wrap(db.BulkInsert(
300 d,
301 "INSERT INTO coordinator (bidder_addr, forger_addr, eth_block_num, url) VALUES %s;",
302 coordinators[:],
303 ))
304}
305
306// AddExitTree insert Exit tree into the DB
307func (hdb *HistoryDB) AddExitTree(exitTree []common.ExitInfo) error {
308 return tracerr.Wrap(hdb.addExitTree(hdb.db, exitTree))
309}
310func (hdb *HistoryDB) addExitTree(d meddler.DB, exitTree []common.ExitInfo) error {
311 if len(exitTree) == 0 {
312 return nil
313 }
314 return tracerr.Wrap(db.BulkInsert(
315 d,
316 "INSERT INTO exit_tree (batch_num, account_idx, merkle_proof, balance, "+
317 "instant_withdrawn, delayed_withdraw_request, delayed_withdrawn) VALUES %s;",
318 exitTree[:],
319 ))
320}
321
322func (hdb *HistoryDB) updateExitTree(d sqlx.Ext, blockNum int64,
323 rollupWithdrawals []common.WithdrawInfo, wDelayerWithdrawals []common.WDelayerTransfer) error {
324 if len(rollupWithdrawals) == 0 && len(wDelayerWithdrawals) == 0 {
325 return nil
326 }
327 type withdrawal struct {
328 BatchNum int64 `db:"batch_num"`
329 AccountIdx int64 `db:"account_idx"`
330 InstantWithdrawn *int64 `db:"instant_withdrawn"`
331 DelayedWithdrawRequest *int64 `db:"delayed_withdraw_request"`
332 DelayedWithdrawn *int64 `db:"delayed_withdrawn"`
333 Owner *ethCommon.Address `db:"owner"`
334 Token *ethCommon.Address `db:"token"`
335 }
336 withdrawals := make([]withdrawal, len(rollupWithdrawals)+len(wDelayerWithdrawals))
337 for i := range rollupWithdrawals {
338 info := &rollupWithdrawals[i]
339 withdrawals[i] = withdrawal{
340 BatchNum: int64(info.NumExitRoot),
341 AccountIdx: int64(info.Idx),
342 }
343 if info.InstantWithdraw {
344 withdrawals[i].InstantWithdrawn = &blockNum
345 } else {
346 withdrawals[i].DelayedWithdrawRequest = &blockNum
347 withdrawals[i].Owner = &info.Owner
348 withdrawals[i].Token = &info.Token
349 }
350 }
351 for i := range wDelayerWithdrawals {
352 info := &wDelayerWithdrawals[i]
353 withdrawals[len(rollupWithdrawals)+i] = withdrawal{
354 DelayedWithdrawn: &blockNum,
355 Owner: &info.Owner,
356 Token: &info.Token,
357 }
358 }
359 // In VALUES we set an initial row of NULLs to set the types of each
360 // variable passed as argument
361 const query string = `
362 UPDATE exit_tree e SET
363 instant_withdrawn = d.instant_withdrawn,
364 delayed_withdraw_request = CASE
365 WHEN e.delayed_withdraw_request IS NOT NULL THEN e.delayed_withdraw_request
366 ELSE d.delayed_withdraw_request
367 END,
368 delayed_withdrawn = d.delayed_withdrawn,
369 owner = d.owner,
370 token = d.token
371 FROM (VALUES
372 (NULL::::BIGINT, NULL::::BIGINT, NULL::::BIGINT, NULL::::BIGINT, NULL::::BIGINT, NULL::::BYTEA, NULL::::BYTEA),
373 (:batch_num,
374 :account_idx,
375 :instant_withdrawn,
376 :delayed_withdraw_request,
377 :delayed_withdrawn,
378 :owner,
379 :token)
380 ) as d (batch_num, account_idx, instant_withdrawn, delayed_withdraw_request, delayed_withdrawn, owner, token)
381 WHERE
382 (d.batch_num IS NOT NULL AND e.batch_num = d.batch_num AND e.account_idx = d.account_idx) OR
383 (d.delayed_withdrawn IS NOT NULL AND e.delayed_withdrawn IS NULL AND e.owner = d.owner AND e.token = d.token);
384 `
385 if len(withdrawals) > 0 {
386 if _, err := sqlx.NamedExec(d, query, withdrawals); err != nil {
387 return tracerr.Wrap(err)
388 }
389 }
390
391 return nil
392}
393
394// AddToken insert a token into the DB
395func (hdb *HistoryDB) AddToken(token *common.Token) error {
396 return tracerr.Wrap(meddler.Insert(hdb.db, "token", token))
397}
398
399// AddTokens insert tokens into the DB
400func (hdb *HistoryDB) AddTokens(tokens []common.Token) error { return hdb.addTokens(hdb.db, tokens) }
401func (hdb *HistoryDB) addTokens(d meddler.DB, tokens []common.Token) error {
402 if len(tokens) == 0 {
403 return nil
404 }
405 // Sanitize name and symbol
406 for i, token := range tokens {
407 token.Name = strings.ToValidUTF8(token.Name, " ")
408 token.Symbol = strings.ToValidUTF8(token.Symbol, " ")
409 tokens[i] = token
410 }
411 return tracerr.Wrap(db.BulkInsert(
412 d,
413 `INSERT INTO token (
414 token_id,
415 eth_block_num,
416 eth_addr,
417 name,
418 symbol,
419 decimals
420 ) VALUES %s;`,
421 tokens[:],
422 ))
423}
424
425// UpdateTokenValue updates the USD value of a token
426func (hdb *HistoryDB) UpdateTokenValue(tokenSymbol string, value float64) error {
427 // Sanitize symbol
428 tokenSymbol = strings.ToValidUTF8(tokenSymbol, " ")
429
430 _, err := hdb.db.Exec(
431 "UPDATE token SET usd = $1 WHERE symbol = $2;",
432 value, tokenSymbol,
433 )
434 return tracerr.Wrap(err)
435}
436
437// GetToken returns a token from the DB given a TokenID
438func (hdb *HistoryDB) GetToken(tokenID common.TokenID) (*TokenWithUSD, error) {
439 token := &TokenWithUSD{}
440 err := meddler.QueryRow(
441 hdb.db, token, `SELECT * FROM token WHERE token_id = $1;`, tokenID,
442 )
443 return token, tracerr.Wrap(err)
444}
445
446// GetAllTokens returns all tokens from the DB
447func (hdb *HistoryDB) GetAllTokens() ([]TokenWithUSD, error) {
448 var tokens []*TokenWithUSD
449 err := meddler.QueryAll(
450 hdb.db, &tokens,
451 "SELECT * FROM token ORDER BY token_id;",
452 )
453 return db.SlicePtrsToSlice(tokens).([]TokenWithUSD), tracerr.Wrap(err)
454}
455
456// GetTokenSymbols returns all the token symbols from the DB
457func (hdb *HistoryDB) GetTokenSymbols() ([]string, error) {
458 var tokenSymbols []string
459 rows, err := hdb.db.Query("SELECT symbol FROM token;")
460 if err != nil {
461 return nil, tracerr.Wrap(err)
462 }
463 defer db.RowsClose(rows)
464 sym := new(string)
465 for rows.Next() {
466 err = rows.Scan(sym)
467 if err != nil {
468 return nil, tracerr.Wrap(err)
469 }
470 tokenSymbols = append(tokenSymbols, *sym)
471 }
472 return tokenSymbols, nil
473}
474
475// AddAccounts insert accounts into the DB
476func (hdb *HistoryDB) AddAccounts(accounts []common.Account) error {
477 return tracerr.Wrap(hdb.addAccounts(hdb.db, accounts))
478}
479func (hdb *HistoryDB) addAccounts(d meddler.DB, accounts []common.Account) error {
480 if len(accounts) == 0 {
481 return nil
482 }
483 return tracerr.Wrap(db.BulkInsert(
484 d,
485 `INSERT INTO account (
486 idx,
487 token_id,
488 batch_num,
489 bjj,
490 eth_addr
491 ) VALUES %s;`,
492 accounts[:],
493 ))
494}
495
496// GetAllAccounts returns a list of accounts from the DB
497func (hdb *HistoryDB) GetAllAccounts() ([]common.Account, error) {
498 var accs []*common.Account
499 err := meddler.QueryAll(
500 hdb.db, &accs,
501 "SELECT idx, token_id, batch_num, bjj, eth_addr FROM account ORDER BY idx;",
502 )
503 return db.SlicePtrsToSlice(accs).([]common.Account), tracerr.Wrap(err)
504}
505
506// AddL1Txs inserts L1 txs to the DB. USD and DepositAmountUSD will be set automatically before storing the tx.
507// If the tx is originated by a coordinator, BatchNum must be provided. If it's originated by a user,
508// BatchNum should be null, and the value will be setted by a trigger when a batch forges the tx.
509// EffectiveAmount and EffectiveDepositAmount are seted with default values by the DB.
510func (hdb *HistoryDB) AddL1Txs(l1txs []common.L1Tx) error {
511 return tracerr.Wrap(hdb.addL1Txs(hdb.db, l1txs))
512}
513
514// addL1Txs inserts L1 txs to the DB. USD and DepositAmountUSD will be set automatically before storing the tx.
515// If the tx is originated by a coordinator, BatchNum must be provided. If it's originated by a user,
516// BatchNum should be null, and the value will be setted by a trigger when a batch forges the tx.
517// EffectiveAmount and EffectiveDepositAmount are seted with default values by the DB.
518func (hdb *HistoryDB) addL1Txs(d meddler.DB, l1txs []common.L1Tx) error {
519 if len(l1txs) == 0 {
520 return nil
521 }
522 txs := []txWrite{}
523 for i := 0; i < len(l1txs); i++ {
524 af := new(big.Float).SetInt(l1txs[i].Amount)
525 amountFloat, _ := af.Float64()
526 laf := new(big.Float).SetInt(l1txs[i].DepositAmount)
527 depositAmountFloat, _ := laf.Float64()
528 var effectiveFromIdx *common.Idx
529 if l1txs[i].UserOrigin {
530 if l1txs[i].Type != common.TxTypeCreateAccountDeposit &&
531 l1txs[i].Type != common.TxTypeCreateAccountDepositTransfer {
532 effectiveFromIdx = &l1txs[i].FromIdx
533 }
534 } else {
535 effectiveFromIdx = &l1txs[i].EffectiveFromIdx
536 }
537 txs = append(txs, txWrite{
538 // Generic
539 IsL1: true,
540 TxID: l1txs[i].TxID,
541 Type: l1txs[i].Type,
542 Position: l1txs[i].Position,
543 FromIdx: &l1txs[i].FromIdx,
544 EffectiveFromIdx: effectiveFromIdx,
545 ToIdx: l1txs[i].ToIdx,
546 Amount: l1txs[i].Amount,
547 AmountFloat: amountFloat,
548 TokenID: l1txs[i].TokenID,
549 BatchNum: l1txs[i].BatchNum,
550 EthBlockNum: l1txs[i].EthBlockNum,
551 // L1
552 ToForgeL1TxsNum: l1txs[i].ToForgeL1TxsNum,
553 UserOrigin: &l1txs[i].UserOrigin,
554 FromEthAddr: &l1txs[i].FromEthAddr,
555 FromBJJ: &l1txs[i].FromBJJ,
556 DepositAmount: l1txs[i].DepositAmount,
557 DepositAmountFloat: &depositAmountFloat,
558 })
559 }
560 return tracerr.Wrap(hdb.addTxs(d, txs))
561}
562
563// AddL2Txs inserts L2 txs to the DB. TokenID, USD and FeeUSD will be set automatically before storing the tx.
564func (hdb *HistoryDB) AddL2Txs(l2txs []common.L2Tx) error {
565 return tracerr.Wrap(hdb.addL2Txs(hdb.db, l2txs))
566}
567
568// addL2Txs inserts L2 txs to the DB. TokenID, USD and FeeUSD will be set automatically before storing the tx.
569func (hdb *HistoryDB) addL2Txs(d meddler.DB, l2txs []common.L2Tx) error {
570 txs := []txWrite{}
571 for i := 0; i < len(l2txs); i++ {
572 f := new(big.Float).SetInt(l2txs[i].Amount)
573 amountFloat, _ := f.Float64()
574 txs = append(txs, txWrite{
575 // Generic
576 IsL1: false,
577 TxID: l2txs[i].TxID,
578 Type: l2txs[i].Type,
579 Position: l2txs[i].Position,
580 FromIdx: &l2txs[i].FromIdx,
581 EffectiveFromIdx: &l2txs[i].FromIdx,
582 ToIdx: l2txs[i].ToIdx,
583 TokenID: l2txs[i].TokenID,
584 Amount: l2txs[i].Amount,
585 AmountFloat: amountFloat,
586 BatchNum: &l2txs[i].BatchNum,
587 EthBlockNum: l2txs[i].EthBlockNum,
588 // L2
589 Fee: &l2txs[i].Fee,
590 Nonce: &l2txs[i].Nonce,
591 })
592 }
593 return tracerr.Wrap(hdb.addTxs(d, txs))
594}
595
596func (hdb *HistoryDB) addTxs(d meddler.DB, txs []txWrite) error {
597 if len(txs) == 0 {
598 return nil
599 }
600 return tracerr.Wrap(db.BulkInsert(
601 d,
602 `INSERT INTO tx (
603 is_l1,
604 id,
605 type,
606 position,
607 from_idx,
608 effective_from_idx,
609 to_idx,
610 amount,
611 amount_f,
612 token_id,
613 batch_num,
614 eth_block_num,
615 to_forge_l1_txs_num,
616 user_origin,
617 from_eth_addr,
618 from_bjj,
619 deposit_amount,
620 deposit_amount_f,
621 fee,
622 nonce
623 ) VALUES %s;`,
624 txs[:],
625 ))
626}
627
628// GetAllExits returns all exit from the DB
629func (hdb *HistoryDB) GetAllExits() ([]common.ExitInfo, error) {
630 var exits []*common.ExitInfo
631 err := meddler.QueryAll(
632 hdb.db, &exits,
633 `SELECT exit_tree.batch_num, exit_tree.account_idx, exit_tree.merkle_proof,
634 exit_tree.balance, exit_tree.instant_withdrawn, exit_tree.delayed_withdraw_request,
635 exit_tree.delayed_withdrawn FROM exit_tree ORDER BY item_id;`,
636 )
637 return db.SlicePtrsToSlice(exits).([]common.ExitInfo), tracerr.Wrap(err)
638}
639
640// GetAllL1UserTxs returns all L1UserTxs from the DB
641func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) {
642 var txs []*common.L1Tx
643 err := meddler.QueryAll(
644 hdb.db, &txs, // Note that '\x' gets parsed as a big.Int with value = 0
645 `SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
646 tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
647 tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE '\x' END) AS effective_amount,
648 tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE '\x' END) AS effective_deposit_amount,
649 tx.eth_block_num, tx.type, tx.batch_num
650 FROM tx WHERE is_l1 = TRUE AND user_origin = TRUE ORDER BY item_id;`,
651 )
652 return db.SlicePtrsToSlice(txs).([]common.L1Tx), tracerr.Wrap(err)
653}
654
655// GetAllL1CoordinatorTxs returns all L1CoordinatorTxs from the DB
656func (hdb *HistoryDB) GetAllL1CoordinatorTxs() ([]common.L1Tx, error) {
657 var txs []*common.L1Tx
658 // Since the query specifies that only coordinator txs are returned, it's safe to assume
659 // that returned txs will always have effective amounts
660 err := meddler.QueryAll(
661 hdb.db, &txs,
662 `SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
663 tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
664 tx.amount, tx.amount AS effective_amount,
665 tx.deposit_amount, tx.deposit_amount AS effective_deposit_amount,
666 tx.eth_block_num, tx.type, tx.batch_num
667 FROM tx WHERE is_l1 = TRUE AND user_origin = FALSE ORDER BY item_id;`,
668 )
669 return db.SlicePtrsToSlice(txs).([]common.L1Tx), tracerr.Wrap(err)
670}
671
672// GetAllL2Txs returns all L2Txs from the DB
673func (hdb *HistoryDB) GetAllL2Txs() ([]common.L2Tx, error) {
674 var txs []*common.L2Tx
675 err := meddler.QueryAll(
676 hdb.db, &txs,
677 `SELECT tx.id, tx.batch_num, tx.position,
678 tx.from_idx, tx.to_idx, tx.amount, tx.token_id,
679 tx.fee, tx.nonce, tx.type, tx.eth_block_num
680 FROM tx WHERE is_l1 = FALSE ORDER BY item_id;`,
681 )
682 return db.SlicePtrsToSlice(txs).([]common.L2Tx), tracerr.Wrap(err)
683}
684
685// GetUnforgedL1UserTxs gets L1 User Txs to be forged in the L1Batch with toForgeL1TxsNum.
686func (hdb *HistoryDB) GetUnforgedL1UserTxs(toForgeL1TxsNum int64) ([]common.L1Tx, error) {
687 var txs []*common.L1Tx
688 err := meddler.QueryAll(
689 hdb.db, &txs, // only L1 user txs can have batch_num set to null
690 `SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
691 tx.from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
692 tx.amount, NULL AS effective_amount,
693 tx.deposit_amount, NULL AS effective_deposit_amount,
694 tx.eth_block_num, tx.type, tx.batch_num
695 FROM tx WHERE batch_num IS NULL AND to_forge_l1_txs_num = $1
696 ORDER BY position;`,
697 toForgeL1TxsNum,
698 )
699 return db.SlicePtrsToSlice(txs).([]common.L1Tx), tracerr.Wrap(err)
700}
701
702// TODO: Think about chaning all the queries that return a last value, to queries that return the next valid value.
703
704// GetLastTxsPosition for a given to_forge_l1_txs_num
705func (hdb *HistoryDB) GetLastTxsPosition(toForgeL1TxsNum int64) (int, error) {
706 row := hdb.db.QueryRow(
707 "SELECT position FROM tx WHERE to_forge_l1_txs_num = $1 ORDER BY position DESC;",
708 toForgeL1TxsNum,
709 )
710 var lastL1TxsPosition int
711 return lastL1TxsPosition, tracerr.Wrap(row.Scan(&lastL1TxsPosition))
712}
713
714// GetSCVars returns the rollup, auction and wdelayer smart contracts variables at their last update.
715func (hdb *HistoryDB) GetSCVars() (*common.RollupVariables, *common.AuctionVariables,
716 *common.WDelayerVariables, error) {
717 var rollup common.RollupVariables
718 var auction common.AuctionVariables
719 var wDelayer common.WDelayerVariables
720 if err := meddler.QueryRow(hdb.db, &rollup,
721 "SELECT * FROM rollup_vars ORDER BY eth_block_num DESC LIMIT 1;"); err != nil {
722 return nil, nil, nil, tracerr.Wrap(err)
723 }
724 if err := meddler.QueryRow(hdb.db, &auction,
725 "SELECT * FROM auction_vars ORDER BY eth_block_num DESC LIMIT 1;"); err != nil {
726 return nil, nil, nil, tracerr.Wrap(err)
727 }
728 if err := meddler.QueryRow(hdb.db, &wDelayer,
729 "SELECT * FROM wdelayer_vars ORDER BY eth_block_num DESC LIMIT 1;"); err != nil {
730 return nil, nil, nil, tracerr.Wrap(err)
731 }
732 return &rollup, &auction, &wDelayer, nil
733}
734
735func (hdb *HistoryDB) setRollupVars(d meddler.DB, rollup *common.RollupVariables) error {
736 return tracerr.Wrap(meddler.Insert(d, "rollup_vars", rollup))
737}
738
739func (hdb *HistoryDB) setAuctionVars(d meddler.DB, auction *common.AuctionVariables) error {
740 return tracerr.Wrap(meddler.Insert(d, "auction_vars", auction))
741}
742
743func (hdb *HistoryDB) setWDelayerVars(d meddler.DB, wDelayer *common.WDelayerVariables) error {
744 return tracerr.Wrap(meddler.Insert(d, "wdelayer_vars", wDelayer))
745}
746
747func (hdb *HistoryDB) addBucketUpdates(d meddler.DB, bucketUpdates []common.BucketUpdate) error {
748 if len(bucketUpdates) == 0 {
749 return nil
750 }
751 return tracerr.Wrap(db.BulkInsert(
752 d,
753 `INSERT INTO bucket_update (
754 eth_block_num,
755 num_bucket,
756 block_stamp,
757 withdrawals
758 ) VALUES %s;`,
759 bucketUpdates[:],
760 ))
761}
762
763// AddBucketUpdatesTest allows call to unexported method
764// only for internal testing purposes
765func (hdb *HistoryDB) AddBucketUpdatesTest(d meddler.DB, bucketUpdates []common.BucketUpdate) error {
766 return hdb.addBucketUpdates(d, bucketUpdates)
767}
768
769// GetAllBucketUpdates retrieves all the bucket updates
770func (hdb *HistoryDB) GetAllBucketUpdates() ([]common.BucketUpdate, error) {
771 var bucketUpdates []*common.BucketUpdate
772 err := meddler.QueryAll(
773 hdb.db, &bucketUpdates,
774 `SELECT eth_block_num, num_bucket, block_stamp, withdrawals
775 FROM bucket_update ORDER BY item_id;`,
776 )
777 return db.SlicePtrsToSlice(bucketUpdates).([]common.BucketUpdate), tracerr.Wrap(err)
778}
779
780func (hdb *HistoryDB) addTokenExchanges(d meddler.DB, tokenExchanges []common.TokenExchange) error {
781 if len(tokenExchanges) == 0 {
782 return nil
783 }
784 return tracerr.Wrap(db.BulkInsert(
785 d,
786 `INSERT INTO token_exchange (
787 eth_block_num,
788 eth_addr,
789 value_usd
790 ) VALUES %s;`,
791 tokenExchanges[:],
792 ))
793}
794
795// GetAllTokenExchanges retrieves all the token exchanges
796func (hdb *HistoryDB) GetAllTokenExchanges() ([]common.TokenExchange, error) {
797 var tokenExchanges []*common.TokenExchange
798 err := meddler.QueryAll(
799 hdb.db, &tokenExchanges,
800 "SELECT eth_block_num, eth_addr, value_usd FROM token_exchange ORDER BY item_id;",
801 )
802 return db.SlicePtrsToSlice(tokenExchanges).([]common.TokenExchange), tracerr.Wrap(err)
803}
804
805func (hdb *HistoryDB) addEscapeHatchWithdrawals(d meddler.DB,
806 escapeHatchWithdrawals []common.WDelayerEscapeHatchWithdrawal) error {
807 if len(escapeHatchWithdrawals) == 0 {
808 return nil
809 }
810 return tracerr.Wrap(db.BulkInsert(
811 d,
812 `INSERT INTO escape_hatch_withdrawal (
813 eth_block_num,
814 who_addr,
815 to_addr,
816 token_addr,
817 amount
818 ) VALUES %s;`,
819 escapeHatchWithdrawals[:],
820 ))
821}
822
823// GetAllEscapeHatchWithdrawals retrieves all the escape hatch withdrawals
824func (hdb *HistoryDB) GetAllEscapeHatchWithdrawals() ([]common.WDelayerEscapeHatchWithdrawal, error) {
825 var escapeHatchWithdrawals []*common.WDelayerEscapeHatchWithdrawal
826 err := meddler.QueryAll(
827 hdb.db, &escapeHatchWithdrawals,
828 "SELECT eth_block_num, who_addr, to_addr, token_addr, amount FROM escape_hatch_withdrawal ORDER BY item_id;",
829 )
830 return db.SlicePtrsToSlice(escapeHatchWithdrawals).([]common.WDelayerEscapeHatchWithdrawal),
831 tracerr.Wrap(err)
832}
833
834// SetInitialSCVars sets the initial state of rollup, auction, wdelayer smart
835// contract variables. This initial state is stored linked to block 0, which
836// always exist in the DB and is used to store initialization data that always
837// exist in the smart contracts.
838func (hdb *HistoryDB) SetInitialSCVars(rollup *common.RollupVariables,
839 auction *common.AuctionVariables, wDelayer *common.WDelayerVariables) error {
840 txn, err := hdb.db.Beginx()
841 if err != nil {
842 return tracerr.Wrap(err)
843 }
844 defer func() {
845 if err != nil {
846 db.Rollback(txn)
847 }
848 }()
849 // Force EthBlockNum to be 0 because it's the block used to link data
850 // that belongs to the creation of the smart contracts
851 rollup.EthBlockNum = 0
852 auction.EthBlockNum = 0
853 wDelayer.EthBlockNum = 0
854 auction.DefaultSlotSetBidSlotNum = 0
855 if err := hdb.setRollupVars(txn, rollup); err != nil {
856 return tracerr.Wrap(err)
857 }
858 if err := hdb.setAuctionVars(txn, auction); err != nil {
859 return tracerr.Wrap(err)
860 }
861 if err := hdb.setWDelayerVars(txn, wDelayer); err != nil {
862 return tracerr.Wrap(err)
863 }
864
865 return tracerr.Wrap(txn.Commit())
866}
867
868// setExtraInfoForgedL1UserTxs sets the EffectiveAmount, EffectiveDepositAmount
869// and EffectiveFromIdx of the given l1UserTxs (with an UPDATE)
870func (hdb *HistoryDB) setExtraInfoForgedL1UserTxs(d sqlx.Ext, txs []common.L1Tx) error {
871 if len(txs) == 0 {
872 return nil
873 }
874 // Effective amounts are stored as success flags in the DB, with true value by default
875 // to reduce the amount of updates. Therefore, only amounts that became uneffective should be
876 // updated to become false. At the same time, all the txs that contain
877 // accounts (FromIdx == 0) are updated to set the EffectiveFromIdx.
878 type txUpdate struct {
879 ID common.TxID `db:"id"`
880 AmountSuccess bool `db:"amount_success"`
881 DepositAmountSuccess bool `db:"deposit_amount_success"`
882 EffectiveFromIdx common.Idx `db:"effective_from_idx"`
883 }
884 txUpdates := []txUpdate{}
885 equal := func(a *big.Int, b *big.Int) bool {
886 return a.Cmp(b) == 0
887 }
888 for i := range txs {
889 amountSuccess := equal(txs[i].Amount, txs[i].EffectiveAmount)
890 depositAmountSuccess := equal(txs[i].DepositAmount, txs[i].EffectiveDepositAmount)
891 if !amountSuccess || !depositAmountSuccess || txs[i].FromIdx == 0 {
892 txUpdates = append(txUpdates, txUpdate{
893 ID: txs[i].TxID,
894 AmountSuccess: amountSuccess,
895 DepositAmountSuccess: depositAmountSuccess,
896 EffectiveFromIdx: txs[i].EffectiveFromIdx,
897 })
898 }
899 }
900 const query string = `
901 UPDATE tx SET
902 amount_success = tx_update.amount_success,
903 deposit_amount_success = tx_update.deposit_amount_success,
904 effective_from_idx = tx_update.effective_from_idx
905 FROM (VALUES
906 (NULL::::BYTEA, NULL::::BOOL, NULL::::BOOL, NULL::::BIGINT),
907 (:id, :amount_success, :deposit_amount_success, :effective_from_idx)
908 ) as tx_update (id, amount_success, deposit_amount_success, effective_from_idx)
909 WHERE tx.id = tx_update.id;
910 `
911 if len(txUpdates) > 0 {
912 if _, err := sqlx.NamedExec(d, query, txUpdates); err != nil {
913 return tracerr.Wrap(err)
914 }
915 }
916 return nil
917}
918
919// AddBlockSCData stores all the information of a block retrieved by the
920// Synchronizer. Blocks should be inserted in order, leaving no gaps because
921// the pagination system of the API/DB depends on this. Within blocks, all
922// items should also be in the correct order (Accounts, Tokens, Txs, etc.)
923func (hdb *HistoryDB) AddBlockSCData(blockData *common.BlockData) (err error) {
924 txn, err := hdb.db.Beginx()
925 if err != nil {
926 return tracerr.Wrap(err)
927 }
928 defer func() {
929 if err != nil {
930 db.Rollback(txn)
931 }
932 }()
933
934 // Add block
935 if err := hdb.addBlock(txn, &blockData.Block); err != nil {
936 return tracerr.Wrap(err)
937 }
938
939 // Add Coordinators
940 if err := hdb.addCoordinators(txn, blockData.Auction.Coordinators); err != nil {
941 return tracerr.Wrap(err)
942 }
943
944 // Add Bids
945 if err := hdb.addBids(txn, blockData.Auction.Bids); err != nil {
946 return tracerr.Wrap(err)
947 }
948
949 // Add Tokens
950 if err := hdb.addTokens(txn, blockData.Rollup.AddedTokens); err != nil {
951 return tracerr.Wrap(err)
952 }
953
954 // Prepare user L1 txs to be added.
955 // They must be added before the batch that will forge them (which can be in the same block)
956 // and after the account that will be sent to (also can be in the same block).
957 // Note: insert order is not relevant since item_id will be updated by a DB trigger when
958 // the batch that forges those txs is inserted
959 userL1s := make(map[common.BatchNum][]common.L1Tx)
960 for i := range blockData.Rollup.L1UserTxs {
961 batchThatForgesIsInTheBlock := false
962 for _, batch := range blockData.Rollup.Batches {
963 if batch.Batch.ForgeL1TxsNum != nil &&
964 *batch.Batch.ForgeL1TxsNum == *blockData.Rollup.L1UserTxs[i].ToForgeL1TxsNum {
965 // Tx is forged in this block. It's guaranteed that:
966 // * the first batch of the block won't forge user L1 txs that have been added in this block
967 // * batch nums are sequential therefore it's safe to add the tx at batch.BatchNum -1
968 batchThatForgesIsInTheBlock = true
969 addAtBatchNum := batch.Batch.BatchNum - 1
970 userL1s[addAtBatchNum] = append(userL1s[addAtBatchNum], blockData.Rollup.L1UserTxs[i])
971 break
972 }
973 }
974 if !batchThatForgesIsInTheBlock {
975 // User artificial batchNum 0 to add txs that are not forge in this block
976 // after all the accounts of the block have been added
977 userL1s[0] = append(userL1s[0], blockData.Rollup.L1UserTxs[i])
978 }
979 }
980
981 // Add Batches
982 for i := range blockData.Rollup.Batches {
983 batch := &blockData.Rollup.Batches[i]
984
985 // Add Batch: this will trigger an update on the DB
986 // that will set the batch num of forged L1 txs in this batch
987 if err = hdb.addBatch(txn, &batch.Batch); err != nil {
988 return tracerr.Wrap(err)
989 }
990
991 // Add accounts
992 if err := hdb.addAccounts(txn, batch.CreatedAccounts); err != nil {
993 return tracerr.Wrap(err)
994 }
995
996 // Set the EffectiveAmount and EffectiveDepositAmount of all the
997 // L1UserTxs that have been forged in this batch
998 if err = hdb.setExtraInfoForgedL1UserTxs(txn, batch.L1UserTxs); err != nil {
999 return tracerr.Wrap(err)
1000 }
1001
1002 // Add forged l1 coordinator Txs
1003 if err := hdb.addL1Txs(txn, batch.L1CoordinatorTxs); err != nil {
1004 return tracerr.Wrap(err)
1005 }
1006
1007 // Add l2 Txs
1008 if err := hdb.addL2Txs(txn, batch.L2Txs); err != nil {
1009 return tracerr.Wrap(err)
1010 }
1011
1012 // Add user L1 txs that will be forged in next batch
1013 if userlL1s, ok := userL1s[batch.Batch.BatchNum]; ok {
1014 if err := hdb.addL1Txs(txn, userlL1s); err != nil {
1015 return tracerr.Wrap(err)
1016 }
1017 }
1018
1019 // Add exit tree
1020 if err := hdb.addExitTree(txn, batch.ExitTree); err != nil {
1021 return tracerr.Wrap(err)
1022 }
1023 }
1024 // Add user L1 txs that won't be forged in this block
1025 if userL1sNotForgedInThisBlock, ok := userL1s[0]; ok {
1026 if err := hdb.addL1Txs(txn, userL1sNotForgedInThisBlock); err != nil {
1027 return tracerr.Wrap(err)
1028 }
1029 }
1030
1031 // Set SC Vars if there was an update
1032 if blockData.Rollup.Vars != nil {
1033 if err := hdb.setRollupVars(txn, blockData.Rollup.Vars); err != nil {
1034 return tracerr.Wrap(err)
1035 }
1036 }
1037 if blockData.Auction.Vars != nil {
1038 if err := hdb.setAuctionVars(txn, blockData.Auction.Vars); err != nil {
1039 return tracerr.Wrap(err)
1040 }
1041 }
1042 if blockData.WDelayer.Vars != nil {
1043 if err := hdb.setWDelayerVars(txn, blockData.WDelayer.Vars); err != nil {
1044 return tracerr.Wrap(err)
1045 }
1046 }
1047
1048 // Update withdrawals in exit tree table
1049 if err := hdb.updateExitTree(txn, blockData.Block.Num,
1050 blockData.Rollup.Withdrawals, blockData.WDelayer.Withdrawals); err != nil {
1051 return tracerr.Wrap(err)
1052 }
1053
1054 // Add Escape Hatch Withdrawals
1055 if err := hdb.addEscapeHatchWithdrawals(txn,
1056 blockData.WDelayer.EscapeHatchWithdrawals); err != nil {
1057 return tracerr.Wrap(err)
1058 }
1059
1060 // Add Buckets withdrawals updates
1061 if err := hdb.addBucketUpdates(txn, blockData.Rollup.UpdateBucketWithdraw); err != nil {
1062 return tracerr.Wrap(err)
1063 }
1064
1065 // Add Token exchange updates
1066 if err := hdb.addTokenExchanges(txn, blockData.Rollup.TokenExchanges); err != nil {
1067 return tracerr.Wrap(err)
1068 }
1069
1070 return tracerr.Wrap(txn.Commit())
1071}
1072
1073// GetCoordinatorAPI returns a coordinator by its bidderAddr
1074func (hdb *HistoryDB) GetCoordinatorAPI(bidderAddr ethCommon.Address) (*CoordinatorAPI, error) {
1075 coordinator := &CoordinatorAPI{}
1076 err := meddler.QueryRow(
1077 hdb.db, coordinator,
1078 "SELECT * FROM coordinator WHERE bidder_addr = $1 ORDER BY item_id DESC LIMIT 1;",
1079 bidderAddr,
1080 )
1081 return coordinator, tracerr.Wrap(err)
1082}
1083
1084// AddAuctionVars insert auction vars into the DB
1085func (hdb *HistoryDB) AddAuctionVars(auctionVars *common.AuctionVariables) error {
1086 return tracerr.Wrap(meddler.Insert(hdb.db, "auction_vars", auctionVars))
1087}
1088
1089// GetTokensTest used to get tokens in a testing context
1090func (hdb *HistoryDB) GetTokensTest() ([]TokenWithUSD, error) {
1091 tokens := []*TokenWithUSD{}
1092 if err := meddler.QueryAll(
1093 hdb.db, &tokens,
1094 "SELECT * FROM TOKEN",
1095 ); err != nil {
1096 return nil, tracerr.Wrap(err)
1097 }
1098 if len(tokens) == 0 {
1099 return []TokenWithUSD{}, nil
1100 }
1101 return db.SlicePtrsToSlice(tokens).([]TokenWithUSD), nil
1102}