arnaucube
arnaucube/hermez-nodePublic Self-hosted repository
Merge pull request #278 from hermeznetwork/feature/integration14

Test synchronizer reorg
a_bennassar arnaubennassar5@gmail.com
Mon, 16 Nov 2020 11:46:38 +0100
commit

7294d766cdbab04f9fdd1363f403eae77af7d309

parent

adedd77956e6ba1f952e52b463a04f3ff5fcc4a9

M db/historydb/historydb.go → db/historydb/historydb.go

@@ -729,8 +729,8 @@ accounts[:],

) } -// GetAccounts returns a list of accounts from the DB -func (hdb *HistoryDB) GetAccounts() ([]common.Account, error) { +// GetAllAccounts returns a list of accounts from the DB +func (hdb *HistoryDB) GetAllAccounts() ([]common.Account, error) { var accs []*common.Account err := meddler.QueryAll( hdb.db, &accs,
M db/historydb/historydb_test.go → db/historydb/historydb_test.go

@@ -293,7 +293,7 @@ accs := test.GenAccounts(nAccounts, 0, tokens, nil, nil, batches)

err = historyDB.AddAccounts(accs) assert.NoError(t, err) // Fetch accounts - fetchedAccs, err := historyDB.GetAccounts() + fetchedAccs, err := historyDB.GetAllAccounts() assert.NoError(t, err) // Compare fetched accounts vs generated accounts for i, acc := range fetchedAccs {
M db/statedb/statedb.go → db/statedb/statedb.go

@@ -249,7 +249,6 @@ // those checkpoints will remain in the storage, and eventually will be

// deleted when MakeCheckpoint overwrites them. `closeCurrent` will close the // currently opened db before doing the reset. func (s *StateDB) reset(batchNum common.BatchNum, closeCurrent bool) error { - checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum)) currentPath := s.path + PathCurrent if closeCurrent {

@@ -271,9 +270,19 @@ }

s.db = sto s.idx = 255 s.currentBatch = batchNum + + if s.mt != nil { + // open the MT for the current s.db + mt, err := merkletree.NewMerkleTree(s.db.WithPrefix(PrefixKeyMT), s.mt.MaxLevels()) + if err != nil { + return err + } + s.mt = mt + } return nil } + checkpointPath := s.path + PathBatchNum + strconv.Itoa(int(batchNum)) // copy 'BatchNumX' to 'current' err = pebbleMakeCheckpoint(checkpointPath, currentPath) if err != nil {
M synchronizer/synchronizer.go → synchronizer/synchronizer.go

@@ -289,11 +289,12 @@ batchNum, err := s.historyDB.GetLastBatchNum()

if err != nil && err != sql.ErrNoRows { return 0, err } - if batchNum != 0 { - err = s.stateDB.Reset(batchNum) - if err != nil { - return 0, err - } + if err == sql.ErrNoRows { + batchNum = 0 + } + err = s.stateDB.Reset(batchNum) + if err != nil { + return 0, err } return blockNum, nil
M synchronizer/synchronizer_test.go → synchronizer/synchronizer_test.go

@@ -7,6 +7,7 @@ "fmt"

"io/ioutil" "math/big" "os" + "sort" "testing" ethCommon "github.com/ethereum/go-ethereum/common"

@@ -32,6 +33,10 @@ func (t *timer) Time() int64 {

currentTime := t.time t.time++ return currentTime +} + +func accountsCmp(accounts []common.Account) func(i, j int) bool { + return func(i, j int) bool { return accounts[i].Idx < accounts[j].Idx } } // Check Sync output and HistoryDB state against expected values generated by

@@ -194,6 +199,80 @@ dbExit.MerkleProof = exit.MerkleProof

assert.Equal(t, &exit, dbExit) //nolint:gosec } } + + // Compare accounts from HistoryDB with StateDB (they should match) + dbAccounts, err := s.historyDB.GetAllAccounts() + require.Nil(t, err) + sdbAccounts, err := s.stateDB.GetAccounts() + require.Nil(t, err) + assertEqualAccountsHistoryDBStateDB(t, dbAccounts, sdbAccounts) +} + +func assertEqualAccountsHistoryDBStateDB(t *testing.T, hdbAccs, sdbAccs []common.Account) { + assert.Equal(t, len(hdbAccs), len(sdbAccs)) + sort.SliceStable(hdbAccs, accountsCmp(hdbAccs)) + sort.SliceStable(sdbAccs, accountsCmp(sdbAccs)) + for i := range hdbAccs { + hdbAcc := hdbAccs[i] + sdbAcc := sdbAccs[i] + assert.Equal(t, hdbAcc.Idx, sdbAcc.Idx) + assert.Equal(t, hdbAcc.TokenID, sdbAcc.TokenID) + assert.Equal(t, hdbAcc.EthAddr, sdbAcc.EthAddr) + assert.Equal(t, hdbAcc.PublicKey, sdbAcc.PublicKey) + } +} + +// ethAddTokens adds the tokens from the blocks to the blockchain +func ethAddTokens(blocks []common.BlockData, client *test.Client) { + for _, block := range blocks { + for _, token := range block.Rollup.AddedTokens { + consts := eth.ERC20Consts{ + Name: fmt.Sprintf("Token %d", token.TokenID), + Symbol: fmt.Sprintf("TK%d", token.TokenID), + Decimals: 18, + } + tokenConsts[token.TokenID] = consts + client.CtlAddERC20(token.EthAddr, consts) + } + } +} + +// ethAddBlocks adds block data to the smart contracts +func ethAddBlocks(t *testing.T, blocks []common.BlockData, + client *test.Client, clientSetup *test.ClientSetup) { + for _, block := range blocks { + for _, token := range block.Rollup.AddedTokens { + _, err := client.RollupAddTokenSimple(token.EthAddr, clientSetup.RollupVariables.FeeAddToken) + require.Nil(t, err) + } + for _, tx := range block.Rollup.L1UserTxs { + client.CtlSetAddr(tx.FromEthAddr) + _, err := client.RollupL1UserTxERC20ETH(tx.FromBJJ, int64(tx.FromIdx), tx.LoadAmount, tx.Amount, + uint32(tx.TokenID), int64(tx.ToIdx)) + require.Nil(t, err) + } + client.CtlSetAddr(clientSetup.AuctionVariables.BootCoordinator) + for _, batch := range block.Rollup.Batches { + _, err := client.RollupForgeBatch(&eth.RollupForgeBatchArgs{ + NewLastIdx: batch.Batch.LastIdx, + NewStRoot: batch.Batch.StateRoot, + NewExitRoot: batch.Batch.ExitRoot, + L1CoordinatorTxs: batch.L1CoordinatorTxs, + L1CoordinatorTxsAuths: [][]byte{}, // Intentionally empty + L2TxsData: batch.L2Txs, + FeeIdxCoordinator: batch.Batch.FeeIdxsCoordinator, + // Circuit selector + VerifierIdx: 0, // Intentionally empty + L1Batch: batch.L1Batch, + ProofA: [2]*big.Int{}, // Intentionally empty + ProofB: [2][2]*big.Int{}, // Intentionally empty + ProofC: [2]*big.Int{}, // Intentionally empty + }) + require.Nil(t, err) + } + // Mine block and sync + client.CtlMineBlock() + } } func TestSync(t *testing.T) {

@@ -322,62 +401,14 @@ require.Equal(t, 2, len(blocks[i].Rollup.Batches))

require.Equal(t, 3, len(blocks[i].Rollup.Batches[0].L2Txs)) // Generate extra required data - for _, block := range blocks { - for _, token := range block.Rollup.AddedTokens { - consts := eth.ERC20Consts{ - Name: fmt.Sprintf("Token %d", token.TokenID), - Symbol: fmt.Sprintf("TK%d", token.TokenID), - Decimals: 18, - } - tokenConsts[token.TokenID] = consts - client.CtlAddERC20(token.EthAddr, consts) - } - } + ethAddTokens(blocks, client) err = tc.FillBlocksExtra(blocks, &tilCfgExtra) assert.Nil(t, err) tc.FillBlocksL1UserTxsBatchNum(blocks) // Add block data to the smart contracts - for _, block := range blocks { - for _, token := range block.Rollup.AddedTokens { - _, err := client.RollupAddTokenSimple(token.EthAddr, clientSetup.RollupVariables.FeeAddToken) - require.Nil(t, err) - } - for _, tx := range block.Rollup.L1UserTxs { - client.CtlSetAddr(tx.FromEthAddr) - _, err := client.RollupL1UserTxERC20ETH(tx.FromBJJ, int64(tx.FromIdx), tx.LoadAmount, tx.Amount, - uint32(tx.TokenID), int64(tx.ToIdx)) - require.Nil(t, err) - } - client.CtlSetAddr(bootCoordAddr) - // feeIdxCoordinator := []common.Idx{} - // if block.Block.EthBlockNum > 2 { - // // After blockNum=2 we have some accounts, use them as - // // coordinator owned to receive fees. - // feeIdxCoordinator = []common.Idx{common.Idx(256), common.Idx(259)} - // } - for _, batch := range block.Rollup.Batches { - _, err := client.RollupForgeBatch(&eth.RollupForgeBatchArgs{ - NewLastIdx: batch.Batch.LastIdx, - NewStRoot: batch.Batch.StateRoot, - NewExitRoot: batch.Batch.ExitRoot, - L1CoordinatorTxs: batch.L1CoordinatorTxs, - L1CoordinatorTxsAuths: [][]byte{}, // Intentionally empty - L2TxsData: batch.L2Txs, - FeeIdxCoordinator: batch.Batch.FeeIdxsCoordinator, - // Circuit selector - VerifierIdx: 0, // Intentionally empty - L1Batch: batch.L1Batch, - ProofA: [2]*big.Int{}, // Intentionally empty - ProofB: [2][2]*big.Int{}, // Intentionally empty - ProofC: [2]*big.Int{}, // Intentionally empty - }) - require.Nil(t, err) - } - // Mine block and sync - client.CtlMineBlock() - } + ethAddBlocks(t, blocks, client, clientSetup) // // Sync to synchronize the current state from the test smart contracts,

@@ -467,25 +498,94 @@ assert.Equal(t, rollupVars, dbRollupVars)

assert.Equal(t, auctionVars, dbAuctionVars) assert.Equal(t, wDelayerVars, dbWDelayerVars) - // TODO: Reorg will be properly tested once we have the mock ethClient implemented - /* - // Force a Reorg - lastSavedBlock, err := historyDB.GetLastBlock() - require.Nil(t, err) + // + // Reorg test + // + + // Redo blocks 2-5 (as a reorg) only leaving: + // - 2 create account transactions + // - 2 add tokens + // We add a 6th block so that the synchronizer can detect the reorg + set2 := ` + Type: Blockchain + + AddToken(1) + AddToken(2) + + CreateAccountDeposit(1) C: 2000 // Idx=256+1=257 + + CreateAccountCoordinator(1) A // Idx=256+0=256 + + > batchL1 // forge L1UserTxs{nil}, freeze defined L1UserTxs{1} + > batchL1 // forge defined L1UserTxs{1}, freeze L1UserTxs{nil} + > block // blockNum=2 + > block // blockNum=3 + > block // blockNum=4 + > block // blockNum=5 + > block // blockNum=6 + ` + tc = til.NewContext(common.RollupConstMaxL1UserTx) + tilCfgExtra = til.ConfigExtra{ + BootCoordAddr: bootCoordAddr, + CoordUser: "A", + } + blocks, err = tc.GenerateBlocks(set2) + require.Nil(t, err) + + for i := 0; i < 4; i++ { + client.CtlRollback() + } + blockNum := client.CtlCurrentBlock() + require.Equal(t, int64(1), blockNum) + + // Generate extra required data + ethAddTokens(blocks, client) - lastSavedBlock.EthBlockNum++ - err = historyDB.AddBlock(lastSavedBlock) - require.Nil(t, err) + err = tc.FillBlocksExtra(blocks, &tilCfgExtra) + assert.Nil(t, err) + tc.FillBlocksL1UserTxsBatchNum(blocks) - lastSavedBlock.EthBlockNum++ - err = historyDB.AddBlock(lastSavedBlock) - require.Nil(t, err) + // Add block data to the smart contracts + ethAddBlocks(t, blocks, client, clientSetup) - log.Debugf("Wait for the blockchain to generate some blocks...") - time.Sleep(40 * time.Second) + // First sync detects the reorg and discards 4 blocks + syncBlock, discards, err = s.Sync2(ctx, nil) + require.Nil(t, err) + expetedDiscards := int64(4) + require.Equal(t, &expetedDiscards, discards) + require.Nil(t, syncBlock) + // At this point, the DB only has data up to block 1 + dbBlock, err := s.historyDB.GetLastBlock() + require.Nil(t, err) + assert.Equal(t, int64(1), dbBlock.EthBlockNum) - err = s.Sync() + // Accounts in HistoryDB and StateDB must be empty + dbAccounts, err := s.historyDB.GetAllAccounts() + require.Nil(t, err) + sdbAccounts, err := s.stateDB.GetAccounts() + require.Nil(t, err) + assert.Equal(t, 0, len(dbAccounts)) + assertEqualAccountsHistoryDBStateDB(t, dbAccounts, sdbAccounts) + + // Sync blocks 2-6 + for i := 0; i < 5; i++ { + syncBlock, discards, err = s.Sync2(ctx, nil) require.Nil(t, err) - */ + require.Nil(t, discards) + require.NotNil(t, syncBlock) + assert.Equal(t, int64(2+i), syncBlock.Block.EthBlockNum) + } + + dbBlock, err = s.historyDB.GetLastBlock() + require.Nil(t, err) + assert.Equal(t, int64(6), dbBlock.EthBlockNum) + + // Accounts in HistoryDB and StateDB is only 2 entries + dbAccounts, err = s.historyDB.GetAllAccounts() + require.Nil(t, err) + sdbAccounts, err = s.stateDB.GetAccounts() + require.Nil(t, err) + assert.Equal(t, 2, len(dbAccounts)) + assertEqualAccountsHistoryDBStateDB(t, dbAccounts, sdbAccounts) }
M test/ethclient.go → test/ethclient.go

@@ -67,7 +67,6 @@ }

func (r *RollupBlock) addTransaction(tx *types.Transaction) *types.Transaction { txHash := tx.Hash() - fmt.Printf("DBG txHash %v\n", txHash.Hex()) r.Txs[txHash] = tx return tx }

@@ -552,6 +551,13 @@ //

// Ethereum // +// CtlCurrentBlock returns the current blockNum without checks +func (c *Client) CtlCurrentBlock() int64 { + c.rw.RLock() + defer c.rw.RUnlock() + return c.blockNum +} + // EthCurrentBlock returns the current blockNum func (c *Client) EthCurrentBlock() (int64, error) { c.rw.RLock()

@@ -822,7 +828,6 @@ data, err := json.Marshal(transactionData{name, value})

if err != nil { panic(err) } - fmt.Printf("DBG dataJSON: %v\n", string(data)) return types.NewTransaction(0, ethCommon.Address{}, nil, 0, nil, data) }