1package coordinator
  2
  3import (
  4	"context"
  5	"fmt"
  6	"io/ioutil"
  7	"math/big"
  8	"os"
  9	"sync"
 10	"testing"
 11	"time"
 12
 13	ethCommon "github.com/ethereum/go-ethereum/common"
 14	"github.com/hermeznetwork/hermez-node/batchbuilder"
 15	"github.com/hermeznetwork/hermez-node/common"
 16	dbUtils "github.com/hermeznetwork/hermez-node/db"
 17	"github.com/hermeznetwork/hermez-node/db/historydb"
 18	"github.com/hermeznetwork/hermez-node/db/l2db"
 19	"github.com/hermeznetwork/hermez-node/db/statedb"
 20	"github.com/hermeznetwork/hermez-node/eth"
 21	"github.com/hermeznetwork/hermez-node/log"
 22	"github.com/hermeznetwork/hermez-node/prover"
 23	"github.com/hermeznetwork/hermez-node/synchronizer"
 24	"github.com/hermeznetwork/hermez-node/test"
 25	"github.com/hermeznetwork/hermez-node/txprocessor"
 26	"github.com/hermeznetwork/hermez-node/txselector"
 27	"github.com/hermeznetwork/tracerr"
 28	"github.com/iden3/go-iden3-crypto/babyjub"
 29	"github.com/iden3/go-merkletree/db/pebble"
 30	"github.com/stretchr/testify/assert"
 31	"github.com/stretchr/testify/require"
 32)
 33
 34var deleteme = []string{}
 35
 36func pebbleMakeCheckpoint(source, dest string) error {
 37	// Remove dest folder (if it exists) before doing the checkpoint
 38	if _, err := os.Stat(dest); !os.IsNotExist(err) {
 39		err := os.RemoveAll(dest)
 40		if err != nil {
 41			return tracerr.Wrap(err)
 42		}
 43	} else if err != nil && !os.IsNotExist(err) {
 44		return tracerr.Wrap(err)
 45	}
 46
 47	sto, err := pebble.NewPebbleStorage(source, false)
 48	if err != nil {
 49		return tracerr.Wrap(err)
 50	}
 51	defer func() {
 52		errClose := sto.Pebble().Close()
 53		if errClose != nil {
 54			log.Errorw("Pebble.Close", "err", errClose)
 55		}
 56	}()
 57
 58	// execute Checkpoint
 59	err = sto.Pebble().Checkpoint(dest)
 60	if err != nil {
 61		return tracerr.Wrap(err)
 62	}
 63
 64	return nil
 65}
 66
 67func TestMain(m *testing.M) {
 68	exitVal := m.Run()
 69	for _, dir := range deleteme {
 70		if err := os.RemoveAll(dir); err != nil {
 71			panic(err)
 72		}
 73	}
 74	os.Exit(exitVal)
 75}
 76
 77var syncDBPath string
 78var txSelDBPath string
 79var batchBuilderDBPath string
 80
 81type modules struct {
 82	historyDB    *historydb.HistoryDB
 83	l2DB         *l2db.L2DB
 84	txSelector   *txselector.TxSelector
 85	batchBuilder *batchbuilder.BatchBuilder
 86	stateDB      *statedb.StateDB
 87}
 88
 89var maxL1Txs uint64 = 256
 90var maxTxs uint64 = 376
 91var nLevels uint32 = 32   //nolint:deadcode,unused
 92var maxFeeTxs uint32 = 64 //nolint:deadcode,varcheck
 93var chainID uint16 = 0
 94
 95func newTestModules(t *testing.T) modules {
 96	var err error
 97	syncDBPath, err = ioutil.TempDir("", "tmpSyncDB")
 98	require.NoError(t, err)
 99	deleteme = append(deleteme, syncDBPath)
100	syncStateDB, err := statedb.NewStateDB(statedb.Config{Path: syncDBPath, Keep: 128,
101		Type: statedb.TypeSynchronizer, NLevels: 48})
102	assert.NoError(t, err)
103
104	pass := os.Getenv("POSTGRES_PASS")
105	db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez")
106	require.NoError(t, err)
107	test.WipeDB(db)
108	l2DB := l2db.NewL2DB(db, db, 10, 100, 0.0, 24*time.Hour, nil)
109	historyDB := historydb.NewHistoryDB(db, db, nil)
110
111	txSelDBPath, err = ioutil.TempDir("", "tmpTxSelDB")
112	require.NoError(t, err)
113	deleteme = append(deleteme, txSelDBPath)
114
115	var bjj babyjub.PublicKeyComp
116	err = bjj.UnmarshalText([]byte("c433f7a696b7aa3a5224efb3993baf0ccd9e92eecee0c29a3f6c8208a9e81d9e"))
117	require.NoError(t, err)
118	coordAccount := &txselector.CoordAccount{ // TODO TMP
119		Addr:                ethCommon.HexToAddress("0xc58d29fA6e86E4FAe04DDcEd660d45BCf3Cb2370"),
120		BJJ:                 bjj,
121		AccountCreationAuth: nil,
122	}
123	txSelector, err := txselector.NewTxSelector(coordAccount, txSelDBPath, syncStateDB, l2DB)
124	assert.NoError(t, err)
125
126	batchBuilderDBPath, err = ioutil.TempDir("", "tmpBatchBuilderDB")
127	require.NoError(t, err)
128	deleteme = append(deleteme, batchBuilderDBPath)
129	batchBuilder, err := batchbuilder.NewBatchBuilder(batchBuilderDBPath, syncStateDB, 0,
130		uint64(nLevels))
131	assert.NoError(t, err)
132
133	return modules{
134		historyDB:    historyDB,
135		l2DB:         l2DB,
136		txSelector:   txSelector,
137		batchBuilder: batchBuilder,
138		stateDB:      syncStateDB,
139	}
140}
141
142type timer struct {
143	time int64
144}
145
146func (t *timer) Time() int64 {
147	currentTime := t.time
148	t.time++
149	return currentTime
150}
151
152var bidder = ethCommon.HexToAddress("0x6b175474e89094c44da98b954eedeac495271d0f")
153var forger = ethCommon.HexToAddress("0xc344E203a046Da13b0B4467EB7B3629D0C99F6E6")
154
155func newTestCoordinator(t *testing.T, forgerAddr ethCommon.Address, ethClient *test.Client,
156	ethClientSetup *test.ClientSetup, modules modules) *Coordinator {
157	debugBatchPath, err := ioutil.TempDir("", "tmpDebugBatch")
158	require.NoError(t, err)
159	deleteme = append(deleteme, debugBatchPath)
160
161	conf := Config{
162		ForgerAddress:           forgerAddr,
163		ConfirmBlocks:           5,
164		L1BatchTimeoutPerc:      0.5,
165		EthClientAttempts:       5,
166		SyncRetryInterval:       400 * time.Microsecond,
167		EthClientAttemptsDelay:  100 * time.Millisecond,
168		TxManagerCheckInterval:  300 * time.Millisecond,
169		DebugBatchPath:          debugBatchPath,
170		MustForgeAtSlotDeadline: true,
171		Purger: PurgerCfg{
172			PurgeBatchDelay:      10,
173			PurgeBlockDelay:      10,
174			InvalidateBatchDelay: 4,
175			InvalidateBlockDelay: 4,
176		},
177		TxProcessorConfig: txprocessor.Config{
178			NLevels:  nLevels,
179			MaxFeeTx: maxFeeTxs,
180			MaxTx:    uint32(maxTxs),
181			MaxL1Tx:  uint32(maxL1Txs),
182			ChainID:  chainID,
183		},
184		VerifierIdx: 0,
185	}
186
187	serverProofs := []prover.Client{
188		&prover.MockClient{Delay: 300 * time.Millisecond},
189		&prover.MockClient{Delay: 400 * time.Millisecond},
190	}
191
192	scConsts := &synchronizer.SCConsts{
193		Rollup:   *ethClientSetup.RollupConstants,
194		Auction:  *ethClientSetup.AuctionConstants,
195		WDelayer: *ethClientSetup.WDelayerConstants,
196	}
197	initSCVars := &synchronizer.SCVariables{
198		Rollup:   *ethClientSetup.RollupVariables,
199		Auction:  *ethClientSetup.AuctionVariables,
200		WDelayer: *ethClientSetup.WDelayerVariables,
201	}
202	coord, err := NewCoordinator(conf, modules.historyDB, modules.l2DB, modules.txSelector,
203		modules.batchBuilder, serverProofs, ethClient, scConsts, initSCVars)
204	require.NoError(t, err)
205	return coord
206}
207
208func newTestSynchronizer(t *testing.T, ethClient *test.Client, ethClientSetup *test.ClientSetup,
209	modules modules) *synchronizer.Synchronizer {
210	sync, err := synchronizer.NewSynchronizer(ethClient, modules.historyDB, modules.stateDB,
211		synchronizer.Config{
212			StatsRefreshPeriod: 0 * time.Second,
213		})
214	require.NoError(t, err)
215	return sync
216}
217
218// TestCoordinatorFlow is a test where the coordinator is stared (which means
219// that goroutines are spawned), and ethereum blocks are mined via the
220// test.Client to simulate starting and stopping forging times.  This test
221// works without a synchronizer, and no l2txs are inserted in the pool, so all
222// the batches are forged empty.  The purpose of this test is to manually
223// observe via the logs that nothing crashes and that the coordinator starts
224// and stops forging at the right blocks.
225func TestCoordinatorFlow(t *testing.T) {
226	if os.Getenv("TEST_COORD_FLOW") == "" {
227		return
228	}
229	ethClientSetup := test.NewClientSetupExample()
230	ethClientSetup.ChainID = big.NewInt(int64(chainID))
231	var timer timer
232	ethClient := test.NewClient(true, &timer, &bidder, ethClientSetup)
233	modules := newTestModules(t)
234	coord := newTestCoordinator(t, forger, ethClient, ethClientSetup, modules)
235
236	// Bid for slot 2 and 4
237	_, err := ethClient.AuctionSetCoordinator(forger, "https://foo.bar")
238	require.NoError(t, err)
239	bid, ok := new(big.Int).SetString("11000000000000000000", 10)
240	if !ok {
241		panic("bad bid")
242	}
243	_, err = ethClient.AuctionBidSimple(3, bid)
244	require.NoError(t, err)
245	_, err = ethClient.AuctionBidSimple(5, bid)
246	require.NoError(t, err)
247
248	coord.Start()
249	ctx := context.Background()
250	time.Sleep(1 * time.Second)
251
252	waitForSlot := func(slot int64) {
253		for {
254			blockNum, err := ethClient.EthLastBlock()
255			require.NoError(t, err)
256			nextBlockSlot, err := ethClient.AuctionGetSlotNumber(blockNum + 1)
257			require.NoError(t, err)
258			if nextBlockSlot == slot {
259				break
260			}
261			ethClient.CtlMineBlock()
262			time.Sleep(100 * time.Millisecond)
263			var stats synchronizer.Stats
264			stats.Eth.LastBlock = *ethClient.CtlLastBlock()
265			stats.Sync.LastBlock = stats.Eth.LastBlock
266			stats.Eth.LastBatchNum = ethClient.CtlLastForgedBatch()
267			stats.Sync.LastBatch.BatchNum = common.BatchNum(stats.Eth.LastBatchNum)
268			canForge, err := ethClient.AuctionCanForge(forger, blockNum+1)
269			require.NoError(t, err)
270			var slot common.Slot
271			slotNum := ethClientSetup.AuctionConstants.SlotNum(blockNum + 1)
272			slot.StartBlock = ethClientSetup.AuctionConstants.GenesisBlockNum +
273				(slotNum)*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
274			slot.EndBlock = ethClientSetup.AuctionConstants.GenesisBlockNum +
275				(slotNum+1)*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
276			if canForge {
277				slot.Forger = forger
278			}
279			stats.Sync.Auction.CurrentSlot = slot
280
281			// Copy stateDB to synchronizer if there was a new batch
282			source := fmt.Sprintf("%v/BatchNum%v", batchBuilderDBPath, stats.Sync.LastBatch)
283			dest := fmt.Sprintf("%v/BatchNum%v", syncDBPath, stats.Sync.LastBatch)
284			if stats.Sync.LastBatch.BatchNum != 0 {
285				if _, err := os.Stat(dest); os.IsNotExist(err) {
286					log.Infow("Making pebble checkpoint for sync",
287						"source", source, "dest", dest)
288					err = pebbleMakeCheckpoint(source, dest)
289					require.NoError(t, err)
290				}
291			}
292			coord.SendMsg(ctx, MsgSyncBlock{
293				Stats: stats,
294			})
295		}
296	}
297
298	// NOTE: With the current test, the coordinator will enter in forge
299	// time before the bidded slot because no one else is forging in the
300	// other slots before the slot deadline.
301	// simulate forgeSequence time
302	waitForSlot(2)
303	log.Info("~~~ simulate entering in forge time")
304	time.Sleep(1 * time.Second)
305
306	// simulate going out from forgeSequence
307	waitForSlot(3)
308	log.Info("~~~ simulate going out from forge time")
309	time.Sleep(1 * time.Second)
310
311	// simulate entering forgeSequence time again
312	waitForSlot(4)
313	log.Info("~~~ simulate entering in forge time again")
314	time.Sleep(2 * time.Second)
315
316	// simulate stopping forgerLoop by channel
317	log.Info("~~~ simulate stopping forgerLoop by closing coordinator stopch")
318	coord.Stop()
319	time.Sleep(1 * time.Second)
320}
321
322func TestCoordinatorStartStop(t *testing.T) {
323	ethClientSetup := test.NewClientSetupExample()
324	ethClientSetup.ChainID = big.NewInt(int64(chainID))
325	var timer timer
326	ethClient := test.NewClient(true, &timer, &bidder, ethClientSetup)
327	modules := newTestModules(t)
328	coord := newTestCoordinator(t, forger, ethClient, ethClientSetup, modules)
329	coord.Start()
330	coord.Stop()
331}
332
333func TestCoordCanForge(t *testing.T) {
334	ethClientSetup := test.NewClientSetupExample()
335	ethClientSetup.ChainID = big.NewInt(int64(chainID))
336	bootForger := ethClientSetup.AuctionVariables.BootCoordinator
337
338	var timer timer
339	ethClient := test.NewClient(true, &timer, &bidder, ethClientSetup)
340	modules := newTestModules(t)
341	coord := newTestCoordinator(t, forger, ethClient, ethClientSetup, modules)
342	_, err := ethClient.AuctionSetCoordinator(forger, "https://foo.bar")
343	require.NoError(t, err)
344	bid, ok := new(big.Int).SetString("12000000000000000000", 10)
345	if !ok {
346		panic("bad bid")
347	}
348	_, err = ethClient.AuctionBidSimple(3, bid)
349	require.NoError(t, err)
350
351	modules2 := newTestModules(t)
352	bootCoord := newTestCoordinator(t, bootForger, ethClient, ethClientSetup, modules2)
353
354	assert.Equal(t, forger, coord.cfg.ForgerAddress)
355	assert.Equal(t, bootForger, bootCoord.cfg.ForgerAddress)
356	ethBootCoord, err := ethClient.AuctionGetBootCoordinator()
357	require.NoError(t, err)
358	assert.Equal(t, &bootForger, ethBootCoord)
359
360	var stats synchronizer.Stats
361
362	slots := [4]common.Slot{}
363	for i := 0; i < 4; i++ {
364		slots[i].StartBlock = ethClientSetup.AuctionConstants.GenesisBlockNum +
365			int64(i)*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
366		slots[i].EndBlock = ethClientSetup.AuctionConstants.GenesisBlockNum +
367			int64(i+1)*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
368		// Only slot 3 has Coordinator winner, the rest are BootCoordinator
369		if i == 3 {
370			slots[i].Forger = forger
371		} else {
372			slots[i].Forger = bootForger
373		}
374	}
375
376	// Slot 0.  No bid, so the winner is the boot coordinator
377	stats.Eth.LastBlock.Num = ethClientSetup.AuctionConstants.GenesisBlockNum
378	stats.Sync.LastBlock = stats.Eth.LastBlock
379	stats.Sync.Auction.CurrentSlot = slots[0]
380	coord.stats = stats
381	bootCoord.stats = stats
382	assert.Equal(t, false, coord.canForge())
383	assert.Equal(t, true, bootCoord.canForge())
384
385	// Slot 0.  No bid, and we reach the deadline, so anyone can forge
386	stats.Eth.LastBlock.Num = ethClientSetup.AuctionConstants.GenesisBlockNum +
387		int64(ethClientSetup.AuctionVariables.SlotDeadline)
388	stats.Sync.LastBlock = stats.Eth.LastBlock
389	stats.Sync.Auction.CurrentSlot = slots[0]
390	coord.stats = stats
391	bootCoord.stats = stats
392	assert.Equal(t, true, coord.canForge())
393	assert.Equal(t, true, bootCoord.canForge())
394
395	// Anyone can forge but the node MustForgeAtSlotDeadline as set as false
396	coord.cfg.MustForgeAtSlotDeadline = false
397	assert.Equal(t, false, coord.canForge())
398
399	// Slot 3. coordinator bid, so the winner is the coordinator
400	stats.Eth.LastBlock.Num = ethClientSetup.AuctionConstants.GenesisBlockNum +
401		3*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
402	stats.Sync.LastBlock = stats.Eth.LastBlock
403	stats.Sync.Auction.CurrentSlot = slots[3]
404	coord.stats = stats
405	bootCoord.stats = stats
406	assert.Equal(t, true, coord.canForge())
407	assert.Equal(t, false, bootCoord.canForge())
408}
409
410func TestCoordHandleMsgSyncBlock(t *testing.T) {
411	ethClientSetup := test.NewClientSetupExample()
412	ethClientSetup.ChainID = big.NewInt(int64(chainID))
413	bootForger := ethClientSetup.AuctionVariables.BootCoordinator
414
415	var timer timer
416	ethClient := test.NewClient(true, &timer, &bidder, ethClientSetup)
417	modules := newTestModules(t)
418	coord := newTestCoordinator(t, forger, ethClient, ethClientSetup, modules)
419	_, err := ethClient.AuctionSetCoordinator(forger, "https://foo.bar")
420	require.NoError(t, err)
421	bid, ok := new(big.Int).SetString("11000000000000000000", 10)
422	if !ok {
423		panic("bad bid")
424	}
425	_, err = ethClient.AuctionBidSimple(3, bid)
426	require.NoError(t, err)
427
428	slots := [4]common.Slot{}
429	for i := 0; i < 4; i++ {
430		slots[i].StartBlock = ethClientSetup.AuctionConstants.GenesisBlockNum +
431			int64(i)*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
432		slots[i].EndBlock = ethClientSetup.AuctionConstants.GenesisBlockNum +
433			int64(i+1)*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
434		// Only slot 3 has Coordinator winner, the rest are BootCoordinator
435		if i == 3 {
436			slots[i].Forger = forger
437		} else {
438			slots[i].Forger = bootForger
439		}
440	}
441
442	var msg MsgSyncBlock
443	coord.stats = msg.Stats
444	ctx := context.Background()
445
446	// Slot 0.  No bid, so the winner is the boot coordinator
447	// pipelineStarted: false -> false
448	coord.stats.Eth.LastBlock.Num = ethClientSetup.AuctionConstants.GenesisBlockNum
449	coord.stats.Sync.LastBlock = coord.stats.Eth.LastBlock
450	coord.stats.Sync.Auction.CurrentSlot = slots[0]
451	assert.Equal(t, false, coord.canForge())
452	msg.Stats = coord.stats
453	require.NoError(t, coord.handleMsgSyncBlock(ctx, &msg))
454	assert.Nil(t, coord.pipeline)
455
456	// Slot 0.  No bid, and we reach the deadline, so anyone can forge
457	// pipelineStarted: false -> true
458	coord.stats.Eth.LastBlock.Num = ethClientSetup.AuctionConstants.GenesisBlockNum +
459		int64(ethClientSetup.AuctionVariables.SlotDeadline)
460	coord.stats.Sync.LastBlock = coord.stats.Eth.LastBlock
461	coord.stats.Sync.Auction.CurrentSlot = slots[0]
462	assert.Equal(t, true, coord.canForge())
463	msg.Stats = coord.stats
464	require.NoError(t, coord.handleMsgSyncBlock(ctx, &msg))
465	assert.NotNil(t, coord.pipeline)
466
467	// Slot 0.  No bid, and we reach the deadline, so anyone can forge
468	// pipelineStarted: true -> true
469	coord.stats.Eth.LastBlock.Num = ethClientSetup.AuctionConstants.GenesisBlockNum +
470		int64(ethClientSetup.AuctionVariables.SlotDeadline) + 1
471	coord.stats.Sync.LastBlock = coord.stats.Eth.LastBlock
472	coord.stats.Sync.Auction.CurrentSlot = slots[0]
473	assert.Equal(t, true, coord.canForge())
474	msg.Stats = coord.stats
475	require.NoError(t, coord.handleMsgSyncBlock(ctx, &msg))
476	assert.NotNil(t, coord.pipeline)
477
478	// Slot 1. No bid, so the winner is the boot coordinator
479	// pipelineStarted: true -> false
480	coord.stats.Eth.LastBlock.Num = ethClientSetup.AuctionConstants.GenesisBlockNum +
481		1*int64(ethClientSetup.AuctionConstants.BlocksPerSlot)
482	coord.stats.Sync.LastBlock = coord.stats.Eth.LastBlock
483	coord.stats.Sync.Auction.CurrentSlot = slots[1]
484	assert.Equal(t, false, coord.canForge())
485	msg.Stats = coord.stats
486	require.NoError(t, coord.handleMsgSyncBlock(ctx, &msg))
487	assert.Nil(t, coord.pipeline)
488}
489
490// ethAddTokens adds the tokens from the blocks to the blockchain
491func ethAddTokens(blocks []common.BlockData, client *test.Client) {
492	for _, block := range blocks {
493		for _, token := range block.Rollup.AddedTokens {
494			consts := eth.ERC20Consts{
495				Name:     fmt.Sprintf("Token %d", token.TokenID),
496				Symbol:   fmt.Sprintf("TK%d", token.TokenID),
497				Decimals: 18,
498			}
499			// tokenConsts[token.TokenID] = consts
500			client.CtlAddERC20(token.EthAddr, consts)
501		}
502	}
503}
504
505func TestCoordinatorStress(t *testing.T) {
506	if os.Getenv("TEST_COORD_STRESS") == "" {
507		return
508	}
509	log.Info("Begin Test Coord Stress")
510	ethClientSetup := test.NewClientSetupExample()
511	ethClientSetup.ChainID = big.NewInt(int64(chainID))
512	var timer timer
513	ethClient := test.NewClient(true, &timer, &bidder, ethClientSetup)
514	modules := newTestModules(t)
515	coord := newTestCoordinator(t, forger, ethClient, ethClientSetup, modules)
516	syn := newTestSynchronizer(t, ethClient, ethClientSetup, modules)
517
518	coord.Start()
519	ctx, cancel := context.WithCancel(context.Background())
520	var wg sync.WaitGroup
521
522	// Synchronizer loop
523	wg.Add(1)
524	go func() {
525		for {
526			blockData, _, err := syn.Sync(ctx, nil)
527			if ctx.Err() != nil {
528				wg.Done()
529				return
530			}
531			require.NoError(t, err)
532			if blockData != nil {
533				stats := syn.Stats()
534				coord.SendMsg(ctx, MsgSyncBlock{
535					Stats:   *stats,
536					Batches: blockData.Rollup.Batches,
537					Vars: synchronizer.SCVariablesPtr{
538						Rollup:   blockData.Rollup.Vars,
539						Auction:  blockData.Auction.Vars,
540						WDelayer: blockData.WDelayer.Vars,
541					},
542				})
543			} else {
544				time.Sleep(100 * time.Millisecond)
545			}
546		}
547	}()
548
549	// Blockchain mining loop
550	wg.Add(1)
551	go func() {
552		for {
553			select {
554			case <-ctx.Done():
555				wg.Done()
556				return
557			case <-time.After(1 * time.Second):
558				ethClient.CtlMineBlock()
559			}
560		}
561	}()
562
563	time.Sleep(600 * time.Second)
564
565	cancel()
566	wg.Wait()
567	coord.Stop()
568}
569
570// TODO: Test Reorg
571// TODO: Test TxMonitor
572// TODO: Test forgeBatch
573// TODO: Test waitServerProof
574// TODO: Test handleReorg