Fix exit table, set delayed_withdrawn in exits

- In exit table, `instant_withdrawn`, `delayed_withdraw_request`, and
  `delayed_withdrawn` were referencing batch_num.  But these actions happen
  outside a batch, so they should reference a block_num.
- Process delayed withdrawns:
    - In Synchronizer, first match a Rollup delayed withdrawn request, with the
      WDelayer deposit (via TxHash), and store the owner and token associated
      with the delayed withdrawn.
    - In HistoryDB: store the owner and token of a delayed withdrawal request
      in the exit_tree, and set delayed_withdrawn when the withdraw is done in
      the WDelayer.
- Update dependency of sqlx to master
    - Last release of sqlx is from 2018 October, and it doesn't support
      `NamedQuery` with a slice of structs, which is used in this commit.

db/historydb/historydb.go

@@ -503,48 +503,72 @@ func (hdb *HistoryDB) addExitTree(d meddler.DB, exitTree []common.ExitInfo) erro
 	)
 }
 
-type exitID struct {
-	batchNum int64
-	idx      int64
-}
-
-func (hdb *HistoryDB) updateExitTree(d meddler.DB, blockNum int64,
-	instantWithdrawn []exitID, delayedWithdrawRequest []exitID) error {
-	// helperQueryExitIDTuples is a helper function to build the query with
-	// an array of tuples in the arguments side built from []exitID
-	helperQueryExitIDTuples := func(queryTmpl string, blockNum int64, exits []exitID) (string, []interface{}) {
-		args := make([]interface{}, len(exits)*2+1)
-		holder := ""
-		args[0] = blockNum
-		for i, v := range exits {
-			args[1+i*2+0] = v.batchNum
-			args[1+i*2+1] = v.idx
-			holder += "(?, ?),"
-		}
-		query := fmt.Sprintf(queryTmpl, holder[:len(holder)-1])
-		return hdb.db.Rebind(query), args
+func (hdb *HistoryDB) updateExitTree(d sqlx.Ext, blockNum int64,
+	rollupWithdrawals []common.WithdrawInfo, wDelayerWithdrawals []common.WDelayerTransfer) error {
+	type withdrawal struct {
+		BatchNum               int64              `db:"batch_num"`
+		AccountIdx             int64              `db:"account_idx"`
+		InstantWithdrawn       *int64             `db:"instant_withdrawn"`
+		DelayedWithdrawRequest *int64             `db:"delayed_withdraw_request"`
+		DelayedWithdrawn       *int64             `db:"delayed_withdrawn"`
+		Owner                  *ethCommon.Address `db:"owner"`
+		Token                  *ethCommon.Address `db:"token"`
 	}
-
-	if len(instantWithdrawn) > 0 {
-		query, args := helperQueryExitIDTuples(
-			`UPDATE exit_tree SET instant_withdrawn = ? WHERE (batch_num, account_idx) IN (%s);`,
-			blockNum,
-			instantWithdrawn,
-		)
-		if _, err := hdb.db.DB.Exec(query, args...); err != nil {
-			return err
-		}
-	}
-	if len(delayedWithdrawRequest) > 0 {
-		query, args := helperQueryExitIDTuples(
-			`UPDATE exit_tree SET delayed_withdraw_request = ? WHERE (batch_num, account_idx) IN (%s);`,
-			blockNum,
-			delayedWithdrawRequest,
-		)
-		if _, err := hdb.db.DB.Exec(query, args...); err != nil {
+	withdrawals := make([]withdrawal, len(rollupWithdrawals)+len(wDelayerWithdrawals))
+	for i := range rollupWithdrawals {
+		info := &rollupWithdrawals[i]
+		withdrawals[i] = withdrawal{
+			BatchNum:   int64(info.NumExitRoot),
+			AccountIdx: int64(info.Idx),
+		}
+		if info.InstantWithdraw {
+			withdrawals[i].InstantWithdrawn = &blockNum
+		} else {
+			withdrawals[i].DelayedWithdrawRequest = &blockNum
+			withdrawals[i].Owner = &info.Owner
+			withdrawals[i].Token = &info.Token
+		}
+	}
+	for i := range wDelayerWithdrawals {
+		info := &wDelayerWithdrawals[i]
+		withdrawals[len(rollupWithdrawals)+i] = withdrawal{
+			DelayedWithdrawn: &blockNum,
+			Owner:            &info.Owner,
+			Token:            &info.Token,
+		}
+	}
+	// In VALUES we set an initial row of NULLs to set the types of each
+	// variable passed as argument
+	query := `
+		UPDATE exit_tree e SET
+			instant_withdrawn = d.instant_withdrawn,
+			delayed_withdraw_request = CASE
+				WHEN e.delayed_withdraw_request IS NOT NULL THEN e.delayed_withdraw_request
+				ELSE d.delayed_withdraw_request
+			END,
+			delayed_withdrawn = d.delayed_withdrawn,
+			owner = d.owner,
+			token = d.token
+		FROM (VALUES
+			(NULL::::BIGINT, NULL::::BIGINT, NULL::::BIGINT, NULL::::BIGINT, NULL::::BIGINT, NULL::::BYTEA, NULL::::BYTEA),
+			(:batch_num,
+			 :account_idx,
+			 :instant_withdrawn,
+			 :delayed_withdraw_request,
+			 :delayed_withdrawn,
+			 :owner,
+			 :token)
+		) as d (batch_num, account_idx, instant_withdrawn, delayed_withdraw_request, delayed_withdrawn, owner, token)
+		WHERE
+			(d.batch_num IS NOT NULL AND e.batch_num = d.batch_num AND e.account_idx = d.account_idx) OR
+			(d.delayed_withdrawn IS NOT NULL AND e.delayed_withdrawn IS NULL AND e.owner = d.owner AND e.token = d.token)
+		`
+	if len(withdrawals) > 0 {
+		if _, err := sqlx.NamedQuery(d, query, withdrawals); err != nil {
 			return err
 		}
 	}
+
 	return nil
 }
 
@@ -1210,7 +1234,7 @@ func (hdb *HistoryDB) setWDelayerVars(d meddler.DB, wDelayer *common.WDelayerVar
 // exist in the smart contracts.
 func (hdb *HistoryDB) SetInitialSCVars(rollup *common.RollupVariables,
 	auction *common.AuctionVariables, wDelayer *common.WDelayerVariables) error {
-	txn, err := hdb.db.Begin()
+	txn, err := hdb.db.Beginx()
 	if err != nil {
 		return err
 	}
@@ -1242,7 +1266,7 @@ func (hdb *HistoryDB) SetInitialSCVars(rollup *common.RollupVariables,
 // the pagination system of the API/DB depends on this.  Within blocks, all
 // items should also be in the correct order (Accounts, Tokens, Txs, etc.)
 func (hdb *HistoryDB) AddBlockSCData(blockData *common.BlockData) (err error) {
-	txn, err := hdb.db.Begin()
+	txn, err := hdb.db.Beginx()
 	if err != nil {
 		return err
 	}
@@ -1340,28 +1364,11 @@ func (hdb *HistoryDB) AddBlockSCData(blockData *common.BlockData) (err error) {
 		}
 	}
 
-	if len(blockData.Rollup.Withdrawals) > 0 {
-		instantWithdrawn := []exitID{}
-		delayedWithdrawRequest := []exitID{}
-		for _, withdraw := range blockData.Rollup.Withdrawals {
-			exitID := exitID{
-				batchNum: int64(withdraw.NumExitRoot),
-				idx:      int64(withdraw.Idx),
-			}
-			if withdraw.InstantWithdraw {
-				instantWithdrawn = append(instantWithdrawn, exitID)
-			} else {
-				delayedWithdrawRequest = append(delayedWithdrawRequest, exitID)
-			}
-		}
-		if err := hdb.updateExitTree(txn, blockData.Block.EthBlockNum,
-			instantWithdrawn, delayedWithdrawRequest); err != nil {
-			return err
-		}
+	if err := hdb.updateExitTree(txn, blockData.Block.EthBlockNum,
+		blockData.Rollup.Withdrawals, blockData.WDelayer.Withdrawals); err != nil {
+		return err
 	}
 
-	// TODO: Process WDelayer withdrawals
-
 	return txn.Commit()
 }
 

db/historydb/historydb_test.go

@@ -481,7 +481,7 @@ func TestExitTree(t *testing.T) {
 	const nAccounts = 3
 	accs := test.GenAccounts(nAccounts, 0, tokens, nil, nil, batches)
 	assert.NoError(t, historyDB.AddAccounts(accs))
-	exitTree := test.GenExitTree(nBatches, batches, accs)
+	exitTree := test.GenExitTree(nBatches, batches, accs, blocks)
 	err = historyDB.AddExitTree(exitTree)
 	assert.NoError(t, err)
 }
@@ -509,7 +509,6 @@ func TestGetL1UserTxs(t *testing.T) {
 	// Sanity check
 	require.Equal(t, 1, len(blocks))
 	require.Equal(t, 5, len(blocks[0].Rollup.L1UserTxs))
-	// fmt.Printf("DBG Blocks: %+v\n", blocks)
 
 	toForgeL1TxsNum := int64(1)
 
@@ -597,14 +596,15 @@ func TestUpdateExitTree(t *testing.T) {
 		ForceExit(1) A: 100
 		ForceExit(1) B: 80
 
-		Exit(1) C: 50 (200)
-		Exit(1) D: 30 (200)
+		Exit(1) C: 50 (172)
+		Exit(1) D: 30 (172)
 
 		> batchL1 // forge L1UserTxs{nil}, freeze defined L1UserTxs{3}
 		> batchL1 // forge L1UserTxs{3}, freeze defined L1UserTxs{nil}
 		> block // blockNum=3
 
 		> block // blockNum=4 (empty block)
+		> block // blockNum=5 (empty block)
 	`
 
 	tc := til.NewContext(common.RollupConstMaxL1UserTx)
@@ -617,39 +617,33 @@ func TestUpdateExitTree(t *testing.T) {
 	err = tc.FillBlocksExtra(blocks, &tilCfgExtra)
 	assert.Nil(t, err)
 
-	// Add all blocks except for the last one
-	for i := range blocks[:len(blocks)-1] {
+	// Add all blocks except for the last two
+	for i := range blocks[:len(blocks)-2] {
 		err = historyDB.AddBlockSCData(&blocks[i])
 		require.Nil(t, err)
 	}
 
-	// Add withdraws to the last block, and insert block into the DB
-	block := &blocks[len(blocks)-1]
+	// Add withdraws to the second-to-last block, and insert block into the DB
+	block := &blocks[len(blocks)-2]
 	require.Equal(t, int64(4), block.Block.EthBlockNum)
+	tokenAddr := blocks[0].Rollup.AddedTokens[0].EthAddr
+	// block.WDelayer.Deposits = append(block.WDelayer.Deposits,
+	// 	common.WDelayerTransfer{Owner: tc.UsersByIdx[257].Addr, Token: tokenAddr, Amount: big.NewInt(80)}, // 257
+	// 	common.WDelayerTransfer{Owner: tc.UsersByIdx[259].Addr, Token: tokenAddr, Amount: big.NewInt(15)}, // 259
+	// )
 	block.Rollup.Withdrawals = append(block.Rollup.Withdrawals,
 		common.WithdrawInfo{Idx: 256, NumExitRoot: 4, InstantWithdraw: true},
-		common.WithdrawInfo{Idx: 257, NumExitRoot: 4, InstantWithdraw: false},
+		common.WithdrawInfo{Idx: 257, NumExitRoot: 4, InstantWithdraw: false,
+			Owner: tc.UsersByIdx[257].Addr, Token: tokenAddr},
 		common.WithdrawInfo{Idx: 258, NumExitRoot: 3, InstantWithdraw: true},
-		common.WithdrawInfo{Idx: 259, NumExitRoot: 3, InstantWithdraw: false},
+		common.WithdrawInfo{Idx: 259, NumExitRoot: 3, InstantWithdraw: false,
+			Owner: tc.UsersByIdx[259].Addr, Token: tokenAddr},
 	)
 	err = historyDB.addBlock(historyDB.db, &block.Block)
 	require.Nil(t, err)
 
-	// update exit trees in DB
-	instantWithdrawn := []exitID{}
-	delayedWithdrawRequest := []exitID{}
-	for _, withdraw := range block.Rollup.Withdrawals {
-		exitID := exitID{
-			batchNum: int64(withdraw.NumExitRoot),
-			idx:      int64(withdraw.Idx),
-		}
-		if withdraw.InstantWithdraw {
-			instantWithdrawn = append(instantWithdrawn, exitID)
-		} else {
-			delayedWithdrawRequest = append(delayedWithdrawRequest, exitID)
-		}
-	}
-	err = historyDB.updateExitTree(historyDB.db, block.Block.EthBlockNum, instantWithdrawn, delayedWithdrawRequest)
+	err = historyDB.updateExitTree(historyDB.db, block.Block.EthBlockNum,
+		block.Rollup.Withdrawals, block.WDelayer.Withdrawals)
 	require.Nil(t, err)
 
 	// Check that exits in DB match with the expected values
@@ -668,6 +662,30 @@ func TestUpdateExitTree(t *testing.T) {
 			assert.Equal(t, &block.Block.EthBlockNum, dbExitsByIdx[withdraw.Idx].DelayedWithdrawRequest)
 		}
 	}
+
+	// Add delayed withdraw to the last block, and insert block into the DB
+	block = &blocks[len(blocks)-1]
+	require.Equal(t, int64(5), block.Block.EthBlockNum)
+	block.WDelayer.Withdrawals = append(block.WDelayer.Withdrawals,
+		common.WDelayerTransfer{
+			Owner:  tc.UsersByIdx[257].Addr,
+			Token:  tokenAddr,
+			Amount: big.NewInt(80),
+		})
+	err = historyDB.addBlock(historyDB.db, &block.Block)
+	require.Nil(t, err)
+
+	err = historyDB.updateExitTree(historyDB.db, block.Block.EthBlockNum,
+		block.Rollup.Withdrawals, block.WDelayer.Withdrawals)
+	require.Nil(t, err)
+
+	// Check that delayed withdrawn has been set
+	dbExits, err = historyDB.GetAllExits()
+	require.Nil(t, err)
+	for _, dbExit := range dbExits {
+		dbExitsByIdx[dbExit.AccountIdx] = dbExit
+	}
+	require.Equal(t, &block.Block.EthBlockNum, dbExitsByIdx[257].DelayedWithdrawn)
 }
 
 // setTestBlocks WARNING: this will delete the blocks and recreate them