From 9f5af0aa07ab620225f1faa47194fecb468144e2 Mon Sep 17 00:00:00 2001 From: Eduard S Date: Mon, 9 Nov 2020 12:39:52 +0100 Subject: [PATCH] Process withdrawals in HistoryDB, test withdrawals and vars in sync --- db/historydb/historydb.go | 67 +++++++++++++++++++++- db/historydb/historydb_test.go | 94 +++++++++++++++++++++++++++++++ eth/rollup.go | 7 ++- synchronizer/synchronizer_test.go | 74 ++++++++++++++++++++++-- test/ethclient.go | 83 +++++++++++++++++++++++---- test/til/txs.go | 43 ++++++++------ 6 files changed, 332 insertions(+), 36 deletions(-) diff --git a/db/historydb/historydb.go b/db/historydb/historydb.go index bc9655f..ff25a0a 100644 --- a/db/historydb/historydb.go +++ b/db/historydb/historydb.go @@ -511,6 +511,51 @@ func (hdb *HistoryDB) addExitTree(d meddler.DB, exitTree []common.ExitInfo) erro ) } +type exitID struct { + batchNum int64 + idx int64 +} + +func (hdb *HistoryDB) updateExitTree(d meddler.DB, blockNum int64, + instantWithdrawn []exitID, delayedWithdrawRequest []exitID) error { + // helperQueryExitIDTuples is a helper function to build the query with + // an array of tuples in the arguments side built from []exitID + helperQueryExitIDTuples := func(queryTmpl string, blockNum int64, exits []exitID) (string, []interface{}) { + args := make([]interface{}, len(exits)*2+1) + holder := "" + args[0] = blockNum + for i, v := range exits { + args[1+i*2+0] = v.batchNum + args[1+i*2+1] = v.idx + holder += "(?, ?)," + } + query := fmt.Sprintf(queryTmpl, holder[:len(holder)-1]) + return hdb.db.Rebind(query), args + } + + if len(instantWithdrawn) > 0 { + query, args := helperQueryExitIDTuples( + `UPDATE exit_tree SET instant_withdrawn = ? WHERE (batch_num, account_idx) IN (%s);`, + blockNum, + instantWithdrawn, + ) + if _, err := hdb.db.DB.Exec(query, args...); err != nil { + return err + } + } + if len(delayedWithdrawRequest) > 0 { + query, args := helperQueryExitIDTuples( + `UPDATE exit_tree SET delayed_withdraw_request = ? WHERE (batch_num, account_idx) IN (%s);`, + blockNum, + delayedWithdrawRequest, + ) + if _, err := hdb.db.DB.Exec(query, args...); err != nil { + return err + } + } + return nil +} + // AddToken insert a token into the DB func (hdb *HistoryDB) AddToken(token *common.Token) error { return meddler.Insert(hdb.db, "token", token) @@ -1324,7 +1369,27 @@ func (hdb *HistoryDB) AddBlockSCData(blockData *common.BlockData) (err error) { } } - // TODO: Process withdrawals + if len(blockData.Rollup.Withdrawals) > 0 { + instantWithdrawn := []exitID{} + delayedWithdrawRequest := []exitID{} + for _, withdraw := range blockData.Rollup.Withdrawals { + exitID := exitID{ + batchNum: int64(withdraw.NumExitRoot), + idx: int64(withdraw.Idx), + } + if withdraw.InstantWithdraw { + instantWithdrawn = append(instantWithdrawn, exitID) + } else { + delayedWithdrawRequest = append(delayedWithdrawRequest, exitID) + } + } + if err := hdb.updateExitTree(txn, blockData.Block.EthBlockNum, + instantWithdrawn, delayedWithdrawRequest); err != nil { + return err + } + } + + // TODO: Process WDelayer withdrawals return txn.Commit() } diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index 92af258..0105614 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -504,6 +504,100 @@ func TestSetInitialSCVars(t *testing.T) { require.Equal(t, wDelayer, dbWDelayer) } +func TestUpdateExitTree(t *testing.T) { + test.WipeDB(historyDB.DB()) + + set := ` + Type: Blockchain + + AddToken(1) + + CreateAccountDeposit(1) C: 2000 // Idx=256+2=258 + CreateAccountDeposit(1) D: 500 // Idx=256+3=259 + + CreateAccountCoordinator(1) A // Idx=256+0=256 + CreateAccountCoordinator(1) B // Idx=256+1=257 + + > batchL1 // forge L1UserTxs{nil}, freeze defined L1UserTxs{5} + > batchL1 // forge defined L1UserTxs{5}, freeze L1UserTxs{nil} + > block // blockNum=2 + + ForceExit(1) A: 100 + ForceExit(1) B: 80 + + Exit(1) C: 50 (200) + Exit(1) D: 30 (200) + + > batchL1 // forge L1UserTxs{nil}, freeze defined L1UserTxs{3} + > batchL1 // forge L1UserTxs{3}, freeze defined L1UserTxs{nil} + > block // blockNum=3 + + > block // blockNum=4 (empty block) + ` + + tc := til.NewContext(common.RollupConstMaxL1UserTx) + tilCfgExtra := til.ConfigExtra{ + BootCoordAddr: ethCommon.HexToAddress("0xE39fEc6224708f0772D2A74fd3f9055A90E0A9f2"), + CoordUser: "A", + } + blocks, err := tc.GenerateBlocks(set) + require.Nil(t, err) + err = tc.FillBlocksExtra(blocks, &tilCfgExtra) + assert.Nil(t, err) + + // Add all blocks except for the last one + for i := range blocks[:len(blocks)-1] { + err = historyDB.AddBlockSCData(&blocks[i]) + require.Nil(t, err) + } + + // Add withdraws to the last block, and insert block into the DB + block := &blocks[len(blocks)-1] + require.Equal(t, int64(4), block.Block.EthBlockNum) + block.Rollup.Withdrawals = append(block.Rollup.Withdrawals, + common.WithdrawInfo{Idx: 256, NumExitRoot: 4, InstantWithdraw: true}, + common.WithdrawInfo{Idx: 257, NumExitRoot: 4, InstantWithdraw: false}, + common.WithdrawInfo{Idx: 258, NumExitRoot: 3, InstantWithdraw: true}, + common.WithdrawInfo{Idx: 259, NumExitRoot: 3, InstantWithdraw: false}, + ) + err = historyDB.addBlock(historyDB.db, &block.Block) + require.Nil(t, err) + + // update exit trees in DB + instantWithdrawn := []exitID{} + delayedWithdrawRequest := []exitID{} + for _, withdraw := range block.Rollup.Withdrawals { + exitID := exitID{ + batchNum: int64(withdraw.NumExitRoot), + idx: int64(withdraw.Idx), + } + if withdraw.InstantWithdraw { + instantWithdrawn = append(instantWithdrawn, exitID) + } else { + delayedWithdrawRequest = append(delayedWithdrawRequest, exitID) + } + } + err = historyDB.updateExitTree(historyDB.db, block.Block.EthBlockNum, instantWithdrawn, delayedWithdrawRequest) + require.Nil(t, err) + + // Check that exits in DB match with the expected values + dbExits, err := historyDB.GetAllExits() + require.Nil(t, err) + assert.Equal(t, 4, len(dbExits)) + dbExitsByIdx := make(map[common.Idx]common.ExitInfo) + for _, dbExit := range dbExits { + dbExitsByIdx[dbExit.AccountIdx] = dbExit + } + for _, withdraw := range block.Rollup.Withdrawals { + assert.Equal(t, withdraw.NumExitRoot, dbExitsByIdx[withdraw.Idx].BatchNum) + if withdraw.InstantWithdraw { + assert.Equal(t, &block.Block.EthBlockNum, dbExitsByIdx[withdraw.Idx].InstantWithdrawn) + } else { + assert.Equal(t, &block.Block.EthBlockNum, dbExitsByIdx[withdraw.Idx].DelayedWithdrawRequest) + } + } +} + // setTestBlocks WARNING: this will delete the blocks and recreate them func setTestBlocks(from, to int64) []common.Block { test.WipeDB(historyDB.DB()) diff --git a/eth/rollup.go b/eth/rollup.go index a695f8d..23d14bc 100644 --- a/eth/rollup.go +++ b/eth/rollup.go @@ -36,9 +36,10 @@ func NewQueueStruct() *QueueStruct { // RollupState represents the state of the Rollup in the Smart Contract type RollupState struct { - StateRoot *big.Int - ExitRoots []*big.Int - ExitNullifierMap map[[256 / 8]byte]bool + StateRoot *big.Int + ExitRoots []*big.Int + // ExitNullifierMap map[[256 / 8]byte]bool + ExitNullifierMap map[int64]map[int64]bool // batchNum -> idx -> bool TokenList []ethCommon.Address TokenMap map[ethCommon.Address]bool MapL1TxQueue map[int64]*QueueStruct diff --git a/synchronizer/synchronizer_test.go b/synchronizer/synchronizer_test.go index 9743490..9e596dc 100644 --- a/synchronizer/synchronizer_test.go +++ b/synchronizer/synchronizer_test.go @@ -287,13 +287,15 @@ func TestSync(t *testing.T) { CreateAccountDepositTransfer(1) E-A: 1000, 200 // Idx=256+7=263 ForceExit(1) A: 100 + ForceExit(1) B: 80 ForceTransfer(1) A-D: 100 Transfer(1) C-A: 100 (200) + Exit(1) C: 50 (200) Exit(1) D: 30 (200) - > batchL1 // forge L1UserTxs{nil}, freeze defined L1UserTxs{2} - > batchL1 // forge L1UserTxs{2}, freeze defined L1UserTxs{nil} + > batchL1 // forge L1UserTxs{nil}, freeze defined L1UserTxs{3} + > batchL1 // forge L1UserTxs{3}, freeze defined L1UserTxs{nil} > block // blockNum=3 ` tc := til.NewContext(common.RollupConstMaxL1UserTx) @@ -315,9 +317,9 @@ func TestSync(t *testing.T) { // blocks 1 (blockNum=3) i = 1 require.Equal(t, 3, int(blocks[i].Block.EthBlockNum)) - require.Equal(t, 3, len(blocks[i].Rollup.L1UserTxs)) + require.Equal(t, 4, len(blocks[i].Rollup.L1UserTxs)) require.Equal(t, 2, len(blocks[i].Rollup.Batches)) - require.Equal(t, 2, len(blocks[i].Rollup.Batches[0].L2Txs)) + require.Equal(t, 3, len(blocks[i].Rollup.Batches[0].L2Txs)) // Generate extra required data for _, block := range blocks { @@ -334,6 +336,7 @@ func TestSync(t *testing.T) { err = tc.FillBlocksExtra(blocks, &tilCfgExtra) assert.Nil(t, err) + tc.FillBlocksL1UserTxsBatchNum(blocks) // Add block data to the smart contracts for _, block := range blocks { @@ -401,6 +404,69 @@ func TestSync(t *testing.T) { checkSyncBlock(t, s, 3, &blocks[1], syncBlock) + // Block 4 + // Generate 2 withdraws manually + _, err = client.RollupWithdrawMerkleProof(tc.Users["A"].BJJ.Public(), 1, 4, 256, big.NewInt(100), []*big.Int{}, true) + require.Nil(t, err) + _, err = client.RollupWithdrawMerkleProof(tc.Users["C"].BJJ.Public(), 1, 3, 258, big.NewInt(50), []*big.Int{}, false) + require.Nil(t, err) + client.CtlMineBlock() + + syncBlock, discards, err = s.Sync2(ctx, nil) + require.Nil(t, err) + require.Nil(t, discards) + require.NotNil(t, syncBlock) + assert.Equal(t, int64(4), syncBlock.Block.EthBlockNum) + dbExits, err := s.historyDB.GetAllExits() + require.Nil(t, err) + foundA1, foundC1 := false, false + for _, exit := range dbExits { + if exit.AccountIdx == 256 && exit.BatchNum == 4 { + foundA1 = true + assert.Equal(t, int64(4), *exit.InstantWithdrawn) + } + if exit.AccountIdx == 258 && exit.BatchNum == 3 { + foundC1 = true + assert.Equal(t, int64(4), *exit.DelayedWithdrawRequest) + } + } + assert.True(t, foundA1) + assert.True(t, foundC1) + + // Block 5 + // Update variables manually + rollupVars, auctionVars, wDelayerVars, err := s.historyDB.GetSCVars() + require.Nil(t, err) + rollupVars.ForgeL1L2BatchTimeout = 42 + _, err = client.RollupUpdateForgeL1L2BatchTimeout(rollupVars.ForgeL1L2BatchTimeout) + require.Nil(t, err) + + auctionVars.OpenAuctionSlots = 17 + _, err = client.AuctionSetOpenAuctionSlots(auctionVars.OpenAuctionSlots) + require.Nil(t, err) + + wDelayerVars.WithdrawalDelay = 99 + _, err = client.WDelayerChangeWithdrawalDelay(wDelayerVars.WithdrawalDelay) + require.Nil(t, err) + + client.CtlMineBlock() + + syncBlock, discards, err = s.Sync2(ctx, nil) + require.Nil(t, err) + require.Nil(t, discards) + require.NotNil(t, syncBlock) + assert.Equal(t, int64(5), syncBlock.Block.EthBlockNum) + + dbRollupVars, dbAuctionVars, dbWDelayerVars, err := s.historyDB.GetSCVars() + require.Nil(t, err) + // Set EthBlockNum for Vars to the blockNum in which they were updated (should be 5) + rollupVars.EthBlockNum = syncBlock.Block.EthBlockNum + auctionVars.EthBlockNum = syncBlock.Block.EthBlockNum + wDelayerVars.EthBlockNum = syncBlock.Block.EthBlockNum + assert.Equal(t, rollupVars, dbRollupVars) + assert.Equal(t, auctionVars, dbAuctionVars) + assert.Equal(t, wDelayerVars, dbWDelayerVars) + // TODO: Reorg will be properly tested once we have the mock ethClient implemented /* // Force a Reorg diff --git a/test/ethclient.go b/test/ethclient.go index c12f851..536eee4 100644 --- a/test/ethclient.go +++ b/test/ethclient.go @@ -39,6 +39,12 @@ type WDelayerBlock struct { Eth *EthereumBlock } +func (w *WDelayerBlock) addTransaction(tx *types.Transaction) *types.Transaction { + txHash := tx.Hash() + w.Txs[txHash] = tx + return tx +} + // RollupBlock stores all the data related to the Rollup SC from an ethereum block type RollupBlock struct { State eth.RollupState @@ -369,8 +375,8 @@ func NewClient(l bool, timer Timer, addr *ethCommon.Address, setup *ClientSetup) Rollup: &RollupBlock{ State: eth.RollupState{ StateRoot: big.NewInt(0), - ExitRoots: make([]*big.Int, 0), - ExitNullifierMap: make(map[[256 / 8]byte]bool), + ExitRoots: make([]*big.Int, 1), + ExitNullifierMap: make(map[int64]map[int64]bool), // TokenID = 0 is ETH. Set first entry in TokenList with 0x0 address for ETH. TokenList: []ethCommon.Address{{}}, TokenMap: make(map[ethCommon.Address]bool), @@ -746,9 +752,46 @@ func (c *Client) RollupWithdrawCircuit(proofA, proofC [2]*big.Int, proofB [2][2] } // RollupWithdrawMerkleProof is the interface to call the smart contract function -func (c *Client) RollupWithdrawMerkleProof(babyPubKey *babyjub.PublicKey, tokenID uint32, numExitRoot, idx int64, amount *big.Int, siblings []*big.Int, instantWithdraw bool) (*types.Transaction, error) { - log.Error("TODO") - return nil, errTODO +func (c *Client) RollupWithdrawMerkleProof(babyPubKey *babyjub.PublicKey, tokenID uint32, numExitRoot, idx int64, amount *big.Int, siblings []*big.Int, instantWithdraw bool) (tx *types.Transaction, err error) { + c.rw.Lock() + defer c.rw.Unlock() + cpy := c.nextBlock().copy() + defer func() { c.revertIfErr(err, cpy) }() + + nextBlock := c.nextBlock() + r := nextBlock.Rollup + + if int(numExitRoot) >= len(r.State.ExitRoots) { + return nil, fmt.Errorf("numExitRoot >= len(r.State.ExitRoots)") + } + if _, ok := r.State.ExitNullifierMap[numExitRoot][idx]; ok { + return nil, fmt.Errorf("exit already withdrawn") + } + r.State.ExitNullifierMap[numExitRoot][idx] = true + + r.Events.Withdraw = append(r.Events.Withdraw, eth.RollupEventWithdraw{ + Idx: uint64(idx), + NumExitRoot: uint64(numExitRoot), + InstantWithdraw: instantWithdraw, + }) + type data struct { + babyPubKey *babyjub.PublicKey + tokenID uint32 + numExitRoot int64 + idx int64 + amount *big.Int + siblings []*big.Int + instantWithdraw bool + } + return r.addTransaction(newTransaction("withdrawMerkleProof", data{ + babyPubKey: babyPubKey, + tokenID: tokenID, + numExitRoot: numExitRoot, + idx: idx, + amount: amount, + siblings: siblings, + instantWithdraw: instantWithdraw, + })), nil } type transactionData struct { @@ -817,6 +860,7 @@ func (c *Client) addBatch(args *eth.RollupForgeBatchArgs) (*types.Transaction, e return nil, fmt.Errorf("args.NewLastIdx < r.State.CurrentIdx") } r.State.CurrentIdx = args.NewLastIdx + r.State.ExitNullifierMap[int64(len(r.State.ExitRoots))] = make(map[int64]bool) r.State.ExitRoots = append(r.State.ExitRoots, args.NewExitRoot) if args.L1Batch { r.State.CurrentToForgeL1TxsNum++ @@ -828,7 +872,7 @@ func (c *Client) addBatch(args *eth.RollupForgeBatchArgs) (*types.Transaction, e ethTx := r.addTransaction(newTransaction("forgebatch", args)) c.forgeBatchArgsPending[ethTx.Hash()] = &batch{*args, *c.addr} r.Events.ForgeBatch = append(r.Events.ForgeBatch, eth.RollupEventForgeBatch{ - BatchNum: int64(len(r.State.ExitRoots)), + BatchNum: int64(len(r.State.ExitRoots)) - 1, EthTxHash: ethTx.Hash(), }) @@ -887,8 +931,13 @@ func (c *Client) RollupUpdateForgeL1L2BatchTimeout(newForgeL1Timeout int64) (tx return nil, eth.ErrAccountNil } - log.Error("TODO") - return nil, errTODO + nextBlock := c.nextBlock() + r := nextBlock.Rollup + r.Vars.ForgeL1L2BatchTimeout = newForgeL1Timeout + r.Events.UpdateForgeL1L2BatchTimeout = append(r.Events.UpdateForgeL1L2BatchTimeout, + eth.RollupEventUpdateForgeL1L2BatchTimeout{NewForgeL1L2BatchTimeout: newForgeL1Timeout}) + + return r.addTransaction(newTransaction("updateForgeL1L2BatchTimeout", newForgeL1Timeout)), nil } // RollupUpdateFeeAddToken is the interface to call the smart contract function @@ -990,8 +1039,13 @@ func (c *Client) AuctionSetOpenAuctionSlots(newOpenAuctionSlots uint16) (tx *typ return nil, eth.ErrAccountNil } - log.Error("TODO") - return nil, errTODO + nextBlock := c.nextBlock() + a := nextBlock.Auction + a.Vars.OpenAuctionSlots = newOpenAuctionSlots + a.Events.NewOpenAuctionSlots = append(a.Events.NewOpenAuctionSlots, + eth.AuctionEventNewOpenAuctionSlots{NewOpenAuctionSlots: newOpenAuctionSlots}) + + return a.addTransaction(newTransaction("setOpenAuctionSlots", newOpenAuctionSlots)), nil } // AuctionGetOpenAuctionSlots is the interface to call the smart contract function @@ -1507,8 +1561,13 @@ func (c *Client) WDelayerChangeWithdrawalDelay(newWithdrawalDelay uint64) (tx *t return nil, eth.ErrAccountNil } - log.Error("TODO") - return nil, errTODO + nextBlock := c.nextBlock() + w := nextBlock.WDelayer + w.Vars.WithdrawalDelay = newWithdrawalDelay + w.Events.NewWithdrawalDelay = append(w.Events.NewWithdrawalDelay, + eth.WDelayerEventNewWithdrawalDelay{WithdrawalDelay: newWithdrawalDelay}) + + return w.addTransaction(newTransaction("changeWithdrawalDelay", newWithdrawalDelay)), nil } // WDelayerDepositInfo is the interface to call the smart contract function diff --git a/test/til/txs.go b/test/til/txs.go index c058424..8c74255 100644 --- a/test/til/txs.go +++ b/test/til/txs.go @@ -663,9 +663,33 @@ type ConfigExtra struct { CoordUser string } +// FillBlocksL1UserTxsBatchNum fills the BatchNum of forged L1UserTxs: +// - blocks[].Rollup.L1UserTxs[].BatchNum +func (tc *Context) FillBlocksL1UserTxsBatchNum(blocks []common.BlockData) { + for i := range blocks { + block := &blocks[i] + for j := range block.Rollup.Batches { + batch := &block.Rollup.Batches[j] + if batch.L1Batch { + // Set BatchNum for forged L1UserTxs to til blocks + bn := batch.Batch.BatchNum + for k := range blocks { + block := &blocks[k] + for l := range block.Rollup.L1UserTxs { + tx := &block.Rollup.L1UserTxs[l] + if *tx.ToForgeL1TxsNum == tc.extra.openToForge { + tx.BatchNum = &bn + } + } + } + tc.extra.openToForge++ + } + } + } +} + // FillBlocksExtra fills extra fields not generated by til in each block, so // that the blockData is closer to what the HistoryDB stores. The filled fields are: -// - blocks[].Rollup.L1UserTxs[].BatchNum // - blocks[].Rollup.Batch.EthBlockNum // - blocks[].Rollup.Batch.ForgerAddr // - blocks[].Rollup.Batch.ForgeL1TxsNum @@ -686,21 +710,6 @@ func (tc *Context) FillBlocksExtra(blocks []common.BlockData, cfg *ConfigExtra) block := &blocks[i] for j := range block.Rollup.Batches { batch := &block.Rollup.Batches[j] - if batch.L1Batch { - // Set BatchNum for forged L1UserTxs to til blocks - bn := batch.Batch.BatchNum - for k := range blocks { - block := &blocks[k] - for l := range block.Rollup.L1UserTxs { - tx := &block.Rollup.L1UserTxs[l] - if *tx.ToForgeL1TxsNum == tc.extra.openToForge { - tx.BatchNum = &bn - } - } - } - tc.extra.openToForge++ - } - batch.Batch.EthBlockNum = block.Block.EthBlockNum // til doesn't fill the batch forger addr batch.Batch.ForgerAddr = cfg.BootCoordAddr @@ -802,6 +811,7 @@ func (tc *Context) FillBlocksExtra(blocks []common.BlockData, cfg *ConfigExtra) batch.ExitTree = append(batch.ExitTree, common.ExitInfo{ + BatchNum: batch.Batch.BatchNum, AccountIdx: tx.FromIdx, Balance: tx.Amount, }) @@ -812,6 +822,7 @@ func (tc *Context) FillBlocksExtra(blocks []common.BlockData, cfg *ConfigExtra) tx := &batch.L2Txs[k] if tx.Type == common.TxTypeExit { batch.ExitTree = append(batch.ExitTree, common.ExitInfo{ + BatchNum: batch.Batch.BatchNum, AccountIdx: tx.FromIdx, Balance: tx.Amount, })