TxProcessor put ExitInfos computation inside TypeSynchronizer logic to avoid computing it for TypeBatchBuilder case

Fix timers resetting in coordinator loops

README.md

@@ -4,6 +4,10 @@ Go implementation of the Hermez node.
 
 ## Developing
 
+### Go version
+
+The `hermez-node` has been tested with go version 1.14
+
 ### Unit testing
 
 Running the unit tests requires a connection to a PostgreSQL database.  You can

batchbuilder/batchbuilder.go

@@ -54,7 +54,7 @@ func NewBatchBuilder(dbpath string, synchronizerStateDB *statedb.StateDB, batchN
 // copy of the rollup state from the Synchronizer at that `batchNum`, otherwise
 // it can just roll back the internal copy.
 func (bb *BatchBuilder) Reset(batchNum common.BatchNum, fromSynchronizer bool) error {
-	return bb.localStateDB.Reset(batchNum, fromSynchronizer)
+	return tracerr.Wrap(bb.localStateDB.Reset(batchNum, fromSynchronizer))
 }
 
 // BuildBatch takes the transactions and returns the common.ZKInputs of the next batch

cli/node/README.md

@@ -2,6 +2,10 @@
 
 This is the main cli for the node
 
+## Go version
+
+The `hermez-node` has been tested with go version 1.14
+
 ## Usage
 
 ```
@@ -65,29 +69,64 @@ when running the coordinator in sync mode
 - The node requires a PostgreSQL database.  The parameters of the server and
   database must be set in the `PostgreSQL` section.
 
+## Building
+
+*All commands assume you are at the `cli/node` directory.*
+
+Building the node requires using the packr utility to bundle the database
+migrations inside the resulting binary.  Install the packr utility with:
+```
+cd /tmp && go get -u github.com/gobuffalo/packr/v2/packr2 && cd -
+```
+
+Make sure your `$PATH` contains `$GOPATH/bin`, otherwise the packr utility will
+not be found.
+
+Now build the node executable:
+```
+cd ../../db && packr2 && cd -
+go build .
+cd ../../db && packr2 clean && cd -
+```
+
+The executable is `node`.
+
 ## Usage Examples
 
+The following commands assume you have built the node previously.  You can also
+run the following examples by replacing `./node` with `go run .` and executing
+them in the `cli/node` directory to build from source and run at the same time.
+
 Run the node in mode synchronizer:
 ```
-go run . --mode sync --cfg cfg.buidler.toml run
+./node --mode sync --cfg cfg.buidler.toml run
 ```
 
 Run the node in mode coordinator:
 ```
-go run . --mode coord --cfg cfg.buidler.toml run
+./node --mode coord --cfg cfg.buidler.toml run
 ```
 
 Import an ethereum private key into the keystore:
 ```
-go run . --mode coord --cfg cfg.buidler.toml importkey --privatekey  0x618b35096c477aab18b11a752be619f0023a539bb02dd6c813477a6211916cde
+./node --mode coord --cfg cfg.buidler.toml importkey --privatekey  0x618b35096c477aab18b11a752be619f0023a539bb02dd6c813477a6211916cde
 ```
 
 Generate a new BabyJubJub key pair:
 ```
-go run . --mode coord --cfg cfg.buidler.toml genbjj
+./node --mode coord --cfg cfg.buidler.toml genbjj
 ```
 
-Wipe the entier SQL database (this will destroy all synchronized and pool data):
+Wipe the entier SQL database (this will destroy all synchronized and pool
+data):
 ```
-go run . --mode coord --cfg cfg.buidler.toml wipesql
+./node --mode coord --cfg cfg.buidler.toml wipesql
+```
+
+Discard all synchronized blocks and associated state up to a given block
+number.  This command is useful in case the synchronizer reaches an invalid
+state and you want to roll back a few blocks and try again (maybe with some
+fixes in the code).
+```
+./node --mode coord --cfg cfg.buidler.toml discard --block 8061330
 ```

cli/node/cfg.buidler.toml

@@ -41,15 +41,20 @@ TokenHEZ = "0x5D94e3e7aeC542aB0F9129B9a7BAdeb5B3Ca0f77"
 TokenHEZName = "Hermez Network Token"
 
 [Coordinator]
-# ForgerAddress = "0x05c23b938a85ab26A36E6314a0D02080E9ca6BeD" # Non-Boot Coordinator
+ForgerAddress = "0x05c23b938a85ab26A36E6314a0D02080E9ca6BeD" # Non-Boot Coordinator
 # ForgerAddressPrivateKey = "0x30f5fddb34cd4166adb2c6003fa6b18f380fd2341376be42cf1c7937004ac7a3"
-ForgerAddress = "0xb4124ceb3451635dacedd11767f004d8a28c6ee7" # Boot Coordinator
+# ForgerAddress = "0xb4124ceb3451635dacedd11767f004d8a28c6ee7" # Boot Coordinator
 # ForgerAddressPrivateKey = "0xa8a54b2d8197bc0b19bb8a084031be71835580a01e70a45a13babd16c9bc1563"
 ConfirmBlocks = 10
-L1BatchTimeoutPerc = 0.999
+L1BatchTimeoutPerc = 0.6
+StartSlotBlocksDelay = 2
+ScheduleBatchBlocksAheadCheck = 3
+SendBatchBlocksMarginCheck = 1
 ProofServerPollInterval = "1s"
 ForgeRetryInterval = "500ms"
 SyncRetryInterval = "1s"
+ForgeDelay = "10s"
+ForgeNoTxsDelay = "0s"
 
 [Coordinator.FeeAccount]
 Address = "0x56232B1c5B10038125Bc7345664B4AFD745bcF8E"
@@ -80,13 +85,13 @@ MaxTx = 512
 NLevels = 32
 
 [Coordinator.EthClient]
-ReceiptTimeout      = "60s"
-ReceiptLoopInterval = "500ms"
 CheckLoopInterval = "500ms"
 Attempts = 4
 AttemptsDelay = "500ms"
-CallGasLimit = 300000
+TxResendTimeout = "2m"
-GasPriceDiv = 100
+NoReuseNonce = false
+MaxGasPrice = "5000000000"
+GasPriceIncPerc = 10
 
 [Coordinator.EthClient.Keystore]
 Path = "/tmp/iden3-test/hermez/ethkeystore"

cli/node/main.go

@@ -11,6 +11,8 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/hermeznetwork/hermez-node/config"
 	dbUtils "github.com/hermeznetwork/hermez-node/db"
+	"github.com/hermeznetwork/hermez-node/db/historydb"
+	"github.com/hermeznetwork/hermez-node/db/l2db"
 	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/hermez-node/node"
 	"github.com/hermeznetwork/tracerr"
@@ -23,6 +25,7 @@ const (
 	flagMode  = "mode"
 	flagSK    = "privatekey"
 	flagYes   = "yes"
+	flagBlock = "block"
 	modeSync  = "sync"
 	modeCoord = "coord"
 )
@@ -139,6 +142,47 @@ func cmdRun(c *cli.Context) error {
 	return nil
 }
 
+func cmdDiscard(c *cli.Context) error {
+	_cfg, err := parseCli(c)
+	if err != nil {
+		return tracerr.Wrap(fmt.Errorf("error parsing flags and config: %w", err))
+	}
+	cfg := _cfg.node
+	blockNum := c.Int64(flagBlock)
+	log.Infof("Discarding all blocks up to block %v...", blockNum)
+
+	db, err := dbUtils.InitSQLDB(
+		cfg.PostgreSQL.Port,
+		cfg.PostgreSQL.Host,
+		cfg.PostgreSQL.User,
+		cfg.PostgreSQL.Password,
+		cfg.PostgreSQL.Name,
+	)
+	if err != nil {
+		return tracerr.Wrap(fmt.Errorf("dbUtils.InitSQLDB: %w", err))
+	}
+	historyDB := historydb.NewHistoryDB(db, nil)
+	if err := historyDB.Reorg(blockNum); err != nil {
+		return tracerr.Wrap(fmt.Errorf("historyDB.Reorg: %w", err))
+	}
+	batchNum, err := historyDB.GetLastBatchNum()
+	if err != nil {
+		return tracerr.Wrap(fmt.Errorf("historyDB.GetLastBatchNum: %w", err))
+	}
+	l2DB := l2db.NewL2DB(
+		db,
+		cfg.Coordinator.L2DB.SafetyPeriod,
+		cfg.Coordinator.L2DB.MaxTxs,
+		cfg.Coordinator.L2DB.TTL.Duration,
+		nil,
+	)
+	if err := l2DB.Reorg(batchNum); err != nil {
+		return tracerr.Wrap(fmt.Errorf("l2DB.Reorg: %w", err))
+	}
+
+	return nil
+}
+
 // Config is the configuration of the hermez node execution
 type Config struct {
 	mode node.Mode
@@ -239,6 +283,18 @@ func main() {
 			Usage:   "Run the hermez-node in the indicated mode",
 			Action:  cmdRun,
 		},
+		{
+			Name:    "discard",
+			Aliases: []string{},
+			Usage:   "Discard blocks up to a specified block number",
+			Action:  cmdDiscard,
+			Flags: []cli.Flag{
+				&cli.Int64Flag{
+					Name:     flagBlock,
+					Usage:    "last block number to keep",
+					Required: false,
+				}},
+		},
 	}
 
 	err := app.Run(os.Args)

common/batch.go

@@ -27,6 +27,24 @@ type Batch struct {
 	TotalFeesUSD       *float64             `meddler:"total_fees_usd"`
 }
 
+// NewEmptyBatch creates a new empty batch
+func NewEmptyBatch() *Batch {
+	return &Batch{
+		BatchNum:           0,
+		EthBlockNum:        0,
+		ForgerAddr:         ethCommon.Address{},
+		CollectedFees:      make(map[TokenID]*big.Int),
+		FeeIdxsCoordinator: make([]Idx, 0),
+		StateRoot:          big.NewInt(0),
+		NumAccounts:        0,
+		LastIdx:            0,
+		ExitRoot:           big.NewInt(0),
+		ForgeL1TxsNum:      nil,
+		SlotNum:            0,
+		TotalFeesUSD:       nil,
+	}
+}
+
 // BatchNum identifies a batch
 type BatchNum int64
 

common/ethauction.go

@@ -33,7 +33,8 @@ func (c *AuctionConstants) SlotNum(blockNum int64) int64 {
 	if blockNum >= c.GenesisBlockNum {
 		return (blockNum - c.GenesisBlockNum) / int64(c.BlocksPerSlot)
 	}
-	return -1
+	// This result will be negative
+	return (blockNum - c.GenesisBlockNum) / int64(c.BlocksPerSlot)
 }
 
 // SlotBlocks returns the first and the last block numbers included in that slot

common/float16.go

@@ -1,132 +0,0 @@
-// Package common Float16 provides methods to work with Hermez custom half float
-// precision, 16 bits, codification internally called Float16 has been adopted
-// to encode large integers. This is done in order to save bits when L2
-// transactions are published.
-//nolint:gomnd
-package common
-
-import (
-	"encoding/binary"
-	"errors"
-	"math/big"
-
-	"github.com/hermeznetwork/tracerr"
-)
-
-var (
-	// ErrRoundingLoss is used when converted big.Int to Float16 causes rounding loss
-	ErrRoundingLoss = errors.New("input value causes rounding loss")
-)
-
-// Float16 represents a float in a 16 bit format
-type Float16 uint16
-
-// Bytes return a byte array of length 2 with the Float16 value encoded in BigEndian
-func (f16 Float16) Bytes() []byte {
-	var b [2]byte
-	binary.BigEndian.PutUint16(b[:], uint16(f16))
-	return b[:]
-}
-
-// Float16FromBytes returns a Float16 from a byte array of 2 bytes.
-func Float16FromBytes(b []byte) *Float16 {
-	// WARNING b[:2] for a b where len(b)<2 can break
-	f16 := Float16(binary.BigEndian.Uint16(b[:2]))
-	return &f16
-}
-
-// BigInt converts the Float16 to a *big.Int integer
-func (f16 *Float16) BigInt() *big.Int {
-	fl := int64(*f16)
-
-	m := big.NewInt(fl & 0x3FF)
-	e := big.NewInt(fl >> 11)
-	e5 := (fl >> 10) & 0x01
-
-	exp := big.NewInt(0).Exp(big.NewInt(10), e, nil)
-	res := m.Mul(m, exp)
-
-	if e5 != 0 && e.Cmp(big.NewInt(0)) != 0 {
-		res.Add(res, exp.Div(exp, big.NewInt(2)))
-	}
-	return res
-}
-
-// floorFix2Float converts a fix to a float, always rounding down
-func floorFix2Float(_f *big.Int) Float16 {
-	zero := big.NewInt(0)
-	ten := big.NewInt(10)
-	e := int64(0)
-
-	m := big.NewInt(0)
-	m.Set(_f)
-
-	if m.Cmp(zero) == 0 {
-		return 0
-	}
-
-	s := big.NewInt(0).Rsh(m, 10)
-
-	for s.Cmp(zero) != 0 {
-		m.Div(m, ten)
-		s.Rsh(m, 10)
-		e++
-	}
-
-	return Float16(m.Int64() | e<<11)
-}
-
-// NewFloat16 encodes a *big.Int integer as a Float16, returning error in
-// case of loss during the encoding.
-func NewFloat16(f *big.Int) (Float16, error) {
-	fl1 := floorFix2Float(f)
-	fi1 := fl1.BigInt()
-	fl2 := fl1 | 0x400
-	fi2 := fl2.BigInt()
-
-	m3 := (fl1 & 0x3FF) + 1
-	e3 := fl1 >> 11
-
-	if m3&0x400 == 0 {
-		m3 = 0x66
-		e3++
-	}
-
-	fl3 := m3 + e3<<11
-	fi3 := fl3.BigInt()
-
-	res := fl1
-
-	d := big.NewInt(0).Abs(fi1.Sub(fi1, f))
-	d2 := big.NewInt(0).Abs(fi2.Sub(fi2, f))
-
-	if d.Cmp(d2) == 1 {
-		res = fl2
-		d = d2
-	}
-
-	d3 := big.NewInt(0).Abs(fi3.Sub(fi3, f))
-
-	if d.Cmp(d3) == 1 {
-		res = fl3
-	}
-
-	// Do rounding check
-	if res.BigInt().Cmp(f) == 0 {
-		return res, nil
-	}
-	return res, tracerr.Wrap(ErrRoundingLoss)
-}
-
-// NewFloat16Floor encodes a big.Int integer as a Float16, rounding down in
-// case of loss during the encoding.
-func NewFloat16Floor(f *big.Int) Float16 {
-	fl1 := floorFix2Float(f)
-	fl2 := fl1 | 0x400
-	fi2 := fl2.BigInt()
-
-	if fi2.Cmp(f) < 1 {
-		return fl2
-	}
-	return fl1
-}

common/float16_test.go

@@ -1,132 +0,0 @@
-package common
-
-import (
-	"math/big"
-	"testing"
-
-	"github.com/hermeznetwork/tracerr"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestConversionsFloat16(t *testing.T) {
-	testVector := map[Float16]string{
-		0x307B: "123000000",
-		0x1DC6: "454500",
-		0xFFFF: "10235000000000000000000000000000000",
-		0x0000: "0",
-		0x0400: "0",
-		0x0001: "1",
-		0x0401: "1",
-		0x0800: "0",
-		0x0c00: "5",
-		0x0801: "10",
-		0x0c01: "15",
-	}
-
-	for test := range testVector {
-		fix := test.BigInt()
-
-		assert.Equal(t, fix.String(), testVector[test])
-
-		bi := big.NewInt(0)
-		bi.SetString(testVector[test], 10)
-
-		fl, err := NewFloat16(bi)
-		assert.NoError(t, err)
-
-		fx2 := fl.BigInt()
-		assert.Equal(t, fx2.String(), testVector[test])
-	}
-}
-
-func TestFloorFix2FloatFloat16(t *testing.T) {
-	testVector := map[string]Float16{
-		"87999990000000000": 0x776f,
-		"87950000000000001": 0x776f,
-		"87950000000000000": 0x776f,
-		"87949999999999999": 0x736f,
-	}
-
-	for test := range testVector {
-		bi := big.NewInt(0)
-		bi.SetString(test, 10)
-
-		testFloat := NewFloat16Floor(bi)
-
-		assert.Equal(t, testFloat, testVector[test])
-	}
-}
-
-func TestConversionLossesFloat16(t *testing.T) {
-	a := big.NewInt(1000)
-	b, err := NewFloat16(a)
-	assert.NoError(t, err)
-	c := b.BigInt()
-	assert.Equal(t, c, a)
-
-	a = big.NewInt(1024)
-	b, err = NewFloat16(a)
-	assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err))
-	c = b.BigInt()
-	assert.NotEqual(t, c, a)
-
-	a = big.NewInt(32767)
-	b, err = NewFloat16(a)
-	assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err))
-	c = b.BigInt()
-	assert.NotEqual(t, c, a)
-
-	a = big.NewInt(32768)
-	b, err = NewFloat16(a)
-	assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err))
-	c = b.BigInt()
-	assert.NotEqual(t, c, a)
-
-	a = big.NewInt(65536000)
-	b, err = NewFloat16(a)
-	assert.Equal(t, ErrRoundingLoss, tracerr.Unwrap(err))
-	c = b.BigInt()
-	assert.NotEqual(t, c, a)
-}
-
-func BenchmarkFloat16(b *testing.B) {
-	newBigInt := func(s string) *big.Int {
-		bigInt, ok := new(big.Int).SetString(s, 10)
-		if !ok {
-			panic("Bad big int")
-		}
-		return bigInt
-	}
-	type pair struct {
-		Float16 Float16
-		BigInt  *big.Int
-	}
-	testVector := []pair{
-		{0x307B, newBigInt("123000000")},
-		{0x1DC6, newBigInt("454500")},
-		{0xFFFF, newBigInt("10235000000000000000000000000000000")},
-		{0x0000, newBigInt("0")},
-		{0x0400, newBigInt("0")},
-		{0x0001, newBigInt("1")},
-		{0x0401, newBigInt("1")},
-		{0x0800, newBigInt("0")},
-		{0x0c00, newBigInt("5")},
-		{0x0801, newBigInt("10")},
-		{0x0c01, newBigInt("15")},
-	}
-	b.Run("floorFix2Float()", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			NewFloat16Floor(testVector[i%len(testVector)].BigInt)
-		}
-	})
-	b.Run("NewFloat16()", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			_, _ = NewFloat16(testVector[i%len(testVector)].BigInt)
-		}
-	})
-	b.Run("Float16.BigInt()", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			testVector[i%len(testVector)].Float16.BigInt()
-		}
-	})
-}

common/float40.go

@@ -6,7 +6,6 @@
 package common
 
 import (
-	"bytes"
 	"encoding/binary"
 	"errors"
 	"math/big"
@@ -24,8 +23,8 @@ const (
 )
 
 var (
-	// ErrFloat40Overflow is used when a given nonce overflows the maximum
+	// ErrFloat40Overflow is used when a given Float40 overflows the
-	// capacity of the Float40 (2**40-1)
+	// maximum capacity of the Float40 (2**40-1)
 	ErrFloat40Overflow = errors.New("Float40 overflow, max value: 2**40 -1")
 	// ErrFloat40E31 is used when the e > 31 when trying to convert a
 	// *big.Int to Float40
@@ -88,15 +87,14 @@ func NewFloat40(f *big.Int) (Float40, error) {
 	zero := big.NewInt(0)
 	ten := big.NewInt(10)
 	thres := big.NewInt(0x08_00_00_00_00)
-	for bytes.Equal(zero.Bytes(), new(big.Int).Mod(m, ten).Bytes()) &&
+	for new(big.Int).Mod(m, ten).Cmp(zero) == 0 && m.Cmp(thres) >= 0 {
-		!bytes.Equal(zero.Bytes(), new(big.Int).Div(m, thres).Bytes()) {
 		m = new(big.Int).Div(m, ten)
 		e = new(big.Int).Add(e, big.NewInt(1))
 	}
 	if e.Int64() > 31 {
 		return 0, ErrFloat40E31
 	}
-	if !bytes.Equal(zero.Bytes(), new(big.Int).Div(m, thres).Bytes()) {
+	if m.Cmp(thres) >= 0 {
 		return 0, ErrFloat40NotEnoughPrecission
 	}
 	r := new(big.Int).Add(m,

common/pooll2tx.go

@@ -36,7 +36,7 @@ type PoolL2Tx struct {
 	ToEthAddr ethCommon.Address     `meddler:"to_eth_addr,zeroisnull"`
 	ToBJJ     babyjub.PublicKeyComp `meddler:"to_bjj,zeroisnull"`
 	TokenID   TokenID               `meddler:"token_id"`
-	Amount    *big.Int              `meddler:"amount,bigint"` // TODO: change to float40
+	Amount    *big.Int              `meddler:"amount,bigint"`
 	Fee       FeeSelector           `meddler:"fee"`
 	Nonce     Nonce                 `meddler:"nonce"` // effective 40 bits used
 	State     PoolL2TxState         `meddler:"state"`
@@ -53,7 +53,7 @@ type PoolL2Tx struct {
 	RqToEthAddr       ethCommon.Address     `meddler:"rq_to_eth_addr,zeroisnull"`
 	RqToBJJ           babyjub.PublicKeyComp `meddler:"rq_to_bjj,zeroisnull"`
 	RqTokenID         TokenID               `meddler:"rq_token_id,zeroisnull"`
-	RqAmount          *big.Int              `meddler:"rq_amount,bigintnull"` // TODO: change to float40
+	RqAmount          *big.Int              `meddler:"rq_amount,bigintnull"`
 	RqFee             FeeSelector           `meddler:"rq_fee,zeroisnull"`
 	RqNonce           Nonce                 `meddler:"rq_nonce,zeroisnull"` // effective 48 bits used
 	AbsoluteFee       float64               `meddler:"fee_usd,zeroisnull"`
@@ -126,7 +126,7 @@ func (tx *PoolL2Tx) SetID() error {
 // [ 48 bits ] fromIdx // 6 bytes
 // [ 16 bits ] chainId // 2 bytes
 // [ 32 bits ] signatureConstant // 4 bytes
-// Total bits compressed data:  241 bits // 31 bytes in *big.Int representation
+// Total bits compressed data:  225 bits // 29 bytes in *big.Int representation
 func (tx *PoolL2Tx) TxCompressedData(chainID uint16) (*big.Int, error) {
 	var b [29]byte
 
@@ -179,7 +179,7 @@ func TxCompressedDataEmpty(chainID uint16) *big.Int {
 // [ 40 bits ] amountFloat40 // 5 bytes
 // [ 48 bits ] toIdx // 6 bytes
 // [ 48 bits ] fromIdx // 6 bytes
-// Total bits compressed data:  193 bits // 25 bytes in *big.Int representation
+// Total bits compressed data:  217 bits // 28 bytes in *big.Int representation
 func (tx *PoolL2Tx) TxCompressedDataV2() (*big.Int, error) {
 	if tx.Amount == nil {
 		tx.Amount = big.NewInt(0)
@@ -238,7 +238,7 @@ func (tx *PoolL2Tx) TxCompressedDataV2() (*big.Int, error) {
 // [ 40 bits ] rqAmountFloat40 // 5 bytes
 // [ 48 bits ] rqToIdx // 6 bytes
 // [ 48 bits ] rqFromIdx // 6 bytes
-// Total bits compressed data:  193 bits // 25 bytes in *big.Int representation
+// Total bits compressed data:  217 bits // 28 bytes in *big.Int representation
 func (tx *PoolL2Tx) RqTxCompressedDataV2() (*big.Int, error) {
 	if tx.RqAmount == nil {
 		tx.RqAmount = big.NewInt(0)

common/zk.go

@@ -103,7 +103,7 @@ type ZKInputs struct {
 	// ToEthAddr
 	ToEthAddr []*big.Int `json:"toEthAddr"` // ethCommon.Address, len: [maxTx]
 	// AmountF encoded as float40
-	AmountF []*big.Int `json:"amountF"`
+	AmountF []*big.Int `json:"amountF"` // uint40 len: [maxTx]
 
 	// OnChain determines if is L1 (1/true) or L2 (0/false)
 	OnChain []*big.Int `json:"onChain"` // bool, len: [maxTx]
@@ -479,7 +479,7 @@ func (z ZKInputs) ToHashGlobalData() ([]byte, error) {
 	copy(newExitRoot, z.Metadata.NewExitRootRaw.Bytes())
 	b = append(b, newExitRoot...)
 
-	// [MAX_L1_TX * (2 * MAX_NLEVELS + 480) bits] L1TxsData
+	// [MAX_L1_TX * (2 * MAX_NLEVELS + 528) bits] L1TxsData
 	l1TxDataLen := (2*z.Metadata.MaxLevels + 528)
 	l1TxsDataLen := (z.Metadata.MaxL1Tx * l1TxDataLen)
 	l1TxsData := make([]byte, l1TxsDataLen/8) //nolint:gomnd
@@ -497,7 +497,7 @@ func (z ZKInputs) ToHashGlobalData() ([]byte, error) {
 	}
 	b = append(b, l1TxsDataAvailability...)
 
-	// [MAX_TX*(2*NLevels + 24) bits] L2TxsData
+	// [MAX_TX*(2*NLevels + 48) bits] L2TxsData
 	var l2TxsData []byte
 	l2TxDataLen := 2*z.Metadata.NLevels + 48 //nolint:gomnd
 	l2TxsDataLen := (z.Metadata.MaxTx * l2TxDataLen)

config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"io/ioutil"
+	"math/big"
 	"time"
 
 	"github.com/BurntSushi/toml"
@@ -51,12 +52,42 @@ type Coordinator struct {
 	// L1BatchTimeoutPerc is the portion of the range before the L1Batch
 	// timeout that will trigger a schedule to forge an L1Batch
 	L1BatchTimeoutPerc float64 `validate:"required"`
+	// StartSlotBlocksDelay is the number of blocks of delay to wait before
+	// starting the pipeline when we reach a slot in which we can forge.
+	StartSlotBlocksDelay int64
+	// ScheduleBatchBlocksAheadCheck is the number of blocks ahead in which
+	// the forger address is checked to be allowed to forge (apart from
+	// checking the next block), used to decide when to stop scheduling new
+	// batches (by stopping the pipeline).
+	// For example, if we are at block 10 and ScheduleBatchBlocksAheadCheck
+	// is 5, eventhough at block 11 we canForge, the pipeline will be
+	// stopped if we can't forge at block 15.
+	// This value should be the expected number of blocks it takes between
+	// scheduling a batch and having it mined.
+	ScheduleBatchBlocksAheadCheck int64
+	// SendBatchBlocksMarginCheck is the number of margin blocks ahead in
+	// which the coordinator is also checked to be allowed to forge, apart
+	// from the next block; used to decide when to stop sending batches to
+	// the smart contract.
+	// For example, if we are at block 10 and SendBatchBlocksMarginCheck is
+	// 5, eventhough at block 11 we canForge, the batch will be discarded
+	// if we can't forge at block 15.
+	SendBatchBlocksMarginCheck int64
 	// ProofServerPollInterval is the waiting interval between polling the
 	// ProofServer while waiting for a particular status
 	ProofServerPollInterval Duration `validate:"required"`
 	// ForgeRetryInterval is the waiting interval between calls forge a
 	// batch after an error
 	ForgeRetryInterval Duration `validate:"required"`
+	// ForgeDelay is the delay after which a batch is forged if the slot is
+	// already committed.  If set to 0s, the coordinator will continuously
+	// forge at the maximum rate.
+	ForgeDelay Duration `validate:"-"`
+	// ForgeNoTxsDelay is the delay after which a batch is forged even if
+	// there are no txs to forge if the slot is already committed.  If set
+	// to 0s, the coordinator will continuously forge even if the batches
+	// are empty.
+	ForgeNoTxsDelay Duration `validate:"-"`
 	// SyncRetryInterval is the waiting interval between calls to the main
 	// handler of a synced block after an error
 	SyncRetryInterval Duration `validate:"required"`
@@ -65,19 +96,33 @@ type Coordinator struct {
 		// SafetyPeriod is the number of batches after which
 		// non-pending L2Txs are deleted from the pool
 		SafetyPeriod common.BatchNum `validate:"required"`
-		// MaxTxs is the number of L2Txs that once reached triggers
+		// MaxTxs is the maximum number of pending L2Txs that can be
-		// deletion of old L2Txs
+		// stored in the pool.  Once this number of pending L2Txs is
+		// reached, inserts to the pool will be denied until some of
+		// the pending txs are forged.
 		MaxTxs uint32 `validate:"required"`
 		// TTL is the Time To Live for L2Txs in the pool.  Once MaxTxs
 		// L2Txs is reached, L2Txs older than TTL will be deleted.
 		TTL Duration `validate:"required"`
-		// PurgeBatchDelay is the delay between batches to purge outdated transactions
+		// PurgeBatchDelay is the delay between batches to purge
+		// outdated transactions.  Oudated L2Txs are those that have
+		// been forged or marked as invalid for longer than the
+		// SafetyPeriod and pending L2Txs that have been in the pool
+		// for longer than TTL once there are MaxTxs.
 		PurgeBatchDelay int64 `validate:"required"`
-		// InvalidateBatchDelay is the delay between batches to mark invalid transactions
+		// InvalidateBatchDelay is the delay between batches to mark
+		// invalid transactions due to nonce lower than the account
+		// nonce.
 		InvalidateBatchDelay int64 `validate:"required"`
-		// PurgeBlockDelay is the delay between blocks to purge outdated transactions
+		// PurgeBlockDelay is the delay between blocks to purge
+		// outdated transactions.  Oudated L2Txs are those that have
+		// been forged or marked as invalid for longer than the
+		// SafetyPeriod and pending L2Txs that have been in the pool
+		// for longer than TTL once there are MaxTxs.
 		PurgeBlockDelay int64 `validate:"required"`
-		// InvalidateBlockDelay is the delay between blocks to mark invalid transactions
+		// InvalidateBlockDelay is the delay between blocks to mark
+		// invalid transactions due to nonce lower than the account
+		// nonce.
 		InvalidateBlockDelay int64 `validate:"required"`
 	} `validate:"required"`
 	TxSelector struct {
@@ -97,12 +142,13 @@ type Coordinator struct {
 		NLevels int64 `validate:"required"`
 	} `validate:"required"`
 	EthClient struct {
-		// CallGasLimit is the default gas limit set for ethereum
+		// MaxGasPrice is the maximum gas price allowed for ethereum
-		// calls, except for methods  where a particular gas limit is
+		// transactions
-		// harcoded because it's known to be a big value
+		MaxGasPrice *big.Int `validate:"required"`
-		CallGasLimit uint64 `validate:"required"`
+		// GasPriceIncPerc is the percentage increase of gas price set
-		// GasPriceDiv is the gas price division
+		// in an ethereum transaction from the suggested gas price by
-		GasPriceDiv uint64 `validate:"required"`
+		// the ehtereum node
+		GasPriceIncPerc int64
 		// CheckLoopInterval is the waiting interval between receipt
 		// checks of ethereum transactions in the TxManager
 		CheckLoopInterval Duration `validate:"required"`
@@ -112,6 +158,13 @@ type Coordinator struct {
 		// AttemptsDelay is delay between attempts do do an eth client
 		// RPC call
 		AttemptsDelay Duration `validate:"required"`
+		// TxResendTimeout is the timeout after which a non-mined
+		// ethereum transaction will be resent (reusing the nonce) with
+		// a newly calculated gas price
+		TxResendTimeout Duration `validate:"required"`
+		// NoReuseNonce disables reusing nonces of pending transactions for
+		// new replacement transactions
+		NoReuseNonce bool
 		// Keystore is the ethereum keystore where private keys are kept
 		Keystore struct {
 			// Path to the keystore

coordinator/batch.go

@@ -47,6 +47,8 @@ type Debug struct {
 	MineBlockNum int64
 	// SendBlockNum is the blockNum when the batch was sent to ethereum
 	SendBlockNum int64
+	// ResendNum is the number of times the tx has been resent
+	ResendNum int
 	// LastScheduledL1BatchBlockNum is the blockNum when the last L1Batch
 	// was scheduled
 	LastScheduledL1BatchBlockNum int64
@@ -64,10 +66,17 @@ type Debug struct {
 	// StartToSendDelay is the delay between starting a batch and sending
 	// it to ethereum, in seconds
 	StartToSendDelay float64
+	// StartToMineDelay is the delay between starting a batch and having
+	// it mined in seconds
+	StartToMineDelay float64
+	// SendToMineDelay is the delay between sending a batch tx and having
+	// it mined in seconds
+	SendToMineDelay float64
 }
 
 // BatchInfo contans the Batch information
 type BatchInfo struct {
+	PipelineNum           int
 	BatchNum              common.BatchNum
 	ServerProof           prover.Client
 	ZKInputs              *common.ZKInputs
@@ -82,9 +91,16 @@ type BatchInfo struct {
 	CoordIdxs             []common.Idx
 	ForgeBatchArgs        *eth.RollupForgeBatchArgs
 	// FeesInfo
-	EthTx   *types.Transaction
+	EthTx    *types.Transaction
-	Receipt *types.Receipt
+	EthTxErr error
-	Debug   Debug
+	// SendTimestamp  the time of batch sent to ethereum
+	SendTimestamp time.Time
+	Receipt       *types.Receipt
+	// Fail is true if:
+	// - The receipt status is failed
+	// - A previous parent batch is failed
+	Fail  bool
+	Debug Debug
 }
 
 // DebugStore is a debug function to store the BatchInfo as a json text file in

coordinator/coordinator.go

@@ -3,8 +3,8 @@ package coordinator
 import (
 	"context"
 	"fmt"
+	"math/big"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -23,7 +23,9 @@ import (
 )
 
 var (
-	errLastL1BatchNotSynced = fmt.Errorf("last L1Batch not synced yet")
+	errLastL1BatchNotSynced  = fmt.Errorf("last L1Batch not synced yet")
+	errForgeNoTxsBeforeDelay = fmt.Errorf("no txs to forge and we haven't reached the forge no txs delay")
+	errForgeBeforeDelay      = fmt.Errorf("we haven't reached the forge delay")
 )
 
 const (
@@ -42,25 +44,73 @@ type Config struct {
 	// L1BatchTimeoutPerc is the portion of the range before the L1Batch
 	// timeout that will trigger a schedule to forge an L1Batch
 	L1BatchTimeoutPerc float64
+	// StartSlotBlocksDelay is the number of blocks of delay to wait before
+	// starting the pipeline when we reach a slot in which we can forge.
+	StartSlotBlocksDelay int64
+	// ScheduleBatchBlocksAheadCheck is the number of blocks ahead in which
+	// the forger address is checked to be allowed to forge (apart from
+	// checking the next block), used to decide when to stop scheduling new
+	// batches (by stopping the pipeline).
+	// For example, if we are at block 10 and ScheduleBatchBlocksAheadCheck
+	// is 5, eventhough at block 11 we canForge, the pipeline will be
+	// stopped if we can't forge at block 15.
+	// This value should be the expected number of blocks it takes between
+	// scheduling a batch and having it mined.
+	ScheduleBatchBlocksAheadCheck int64
+	// SendBatchBlocksMarginCheck is the number of margin blocks ahead in
+	// which the coordinator is also checked to be allowed to forge, apart
+	// from the next block; used to decide when to stop sending batches to
+	// the smart contract.
+	// For example, if we are at block 10 and SendBatchBlocksMarginCheck is
+	// 5, eventhough at block 11 we canForge, the batch will be discarded
+	// if we can't forge at block 15.
+	// This value should be the expected number of blocks it takes between
+	// sending a batch and having it mined.
+	SendBatchBlocksMarginCheck int64
 	// EthClientAttempts is the number of attempts to do an eth client RPC
 	// call before giving up
 	EthClientAttempts int
 	// ForgeRetryInterval is the waiting interval between calls forge a
 	// batch after an error
 	ForgeRetryInterval time.Duration
+	// ForgeDelay is the delay after which a batch is forged if the slot is
+	// already committed.  If set to 0s, the coordinator will continuously
+	// forge at the maximum rate.
+	ForgeDelay time.Duration
+	// ForgeNoTxsDelay is the delay after which a batch is forged even if
+	// there are no txs to forge if the slot is already committed.  If set
+	// to 0s, the coordinator will continuously forge even if the batches
+	// are empty.
+	ForgeNoTxsDelay time.Duration
 	// SyncRetryInterval is the waiting interval between calls to the main
 	// handler of a synced block after an error
 	SyncRetryInterval time.Duration
 	// EthClientAttemptsDelay is delay between attempts do do an eth client
 	// RPC call
 	EthClientAttemptsDelay time.Duration
+	// EthTxResendTimeout is the timeout after which a non-mined ethereum
+	// transaction will be resent (reusing the nonce) with a newly
+	// calculated gas price
+	EthTxResendTimeout time.Duration
+	// EthNoReuseNonce disables reusing nonces of pending transactions for
+	// new replacement transactions
+	EthNoReuseNonce bool
+	// MaxGasPrice is the maximum gas price allowed for ethereum
+	// transactions
+	MaxGasPrice *big.Int
+	// GasPriceIncPerc is the percentage increase of gas price set in an
+	// ethereum transaction from the suggested gas price by the ehtereum
+	// node
+	GasPriceIncPerc int64
 	// TxManagerCheckInterval is the waiting interval between receipt
 	// checks of ethereum transactions in the TxManager
 	TxManagerCheckInterval time.Duration
 	// DebugBatchPath if set, specifies the path where batchInfo is stored
 	// in JSON in every step/update of the pipeline
-	DebugBatchPath    string
+	DebugBatchPath string
-	Purger            PurgerCfg
+	Purger         PurgerCfg
+	// VerifierIdx is the index of the verifier contract registered in the
+	// smart contract
 	VerifierIdx       uint8
 	TxProcessorConfig txprocessor.Config
 }
@@ -74,15 +124,22 @@ func (c *Config) debugBatchStore(batchInfo *BatchInfo) {
 	}
 }
 
+type fromBatch struct {
+	BatchNum   common.BatchNum
+	ForgerAddr ethCommon.Address
+	StateRoot  *big.Int
+}
+
 // Coordinator implements the Coordinator type
 type Coordinator struct {
 	// State
-	pipelineBatchNum common.BatchNum // batchNum from which we started the pipeline
+	pipelineNum       int       // Pipeline sequential number.  The first pipeline is 1
-	provers          []prover.Client
+	pipelineFromBatch fromBatch // batch from which we started the pipeline
-	consts           synchronizer.SCConsts
+	provers           []prover.Client
-	vars             synchronizer.SCVariables
+	consts            synchronizer.SCConsts
-	stats            synchronizer.Stats
+	vars              synchronizer.SCVariables
-	started          bool
+	stats             synchronizer.Stats
+	started           bool
 
 	cfg Config
 
@@ -96,7 +153,8 @@ type Coordinator struct {
 	wg     sync.WaitGroup
 	cancel context.CancelFunc
 
-	pipeline *Pipeline
+	pipeline              *Pipeline
+	lastNonFailedBatchNum common.BatchNum
 
 	purger    *Purger
 	txManager *TxManager
@@ -139,10 +197,15 @@ func NewCoordinator(cfg Config,
 
 	ctx, cancel := context.WithCancel(context.Background())
 	c := Coordinator{
-		pipelineBatchNum: -1,
+		pipelineNum: 0,
-		provers:          serverProofs,
+		pipelineFromBatch: fromBatch{
-		consts:           *scConsts,
+			BatchNum:   0,
-		vars:             *initSCVars,
+			ForgerAddr: ethCommon.Address{},
+			StateRoot:  big.NewInt(0),
+		},
+		provers: serverProofs,
+		consts:  *scConsts,
+		vars:    *initSCVars,
 
 		cfg: cfg,
 
@@ -183,8 +246,9 @@ func (c *Coordinator) BatchBuilder() *batchbuilder.BatchBuilder {
 }
 
 func (c *Coordinator) newPipeline(ctx context.Context) (*Pipeline, error) {
-	return NewPipeline(ctx, c.cfg, c.historyDB, c.l2DB, c.txSelector,
+	c.pipelineNum++
-		c.batchBuilder, c.purger, c.txManager, c.provers, &c.consts)
+	return NewPipeline(ctx, c.cfg, c.pipelineNum, c.historyDB, c.l2DB, c.txSelector,
+		c.batchBuilder, c.purger, c, c.txManager, c.provers, &c.consts)
 }
 
 // MsgSyncBlock indicates an update to the Synchronizer stats
@@ -205,6 +269,9 @@ type MsgSyncReorg struct {
 // MsgStopPipeline indicates a signal to reset the pipeline
 type MsgStopPipeline struct {
 	Reason string
+	// FailedBatchNum indicates the first batchNum that failed in the
+	// pipeline.  If FailedBatchNum is 0, it should be ignored.
+	FailedBatchNum common.BatchNum
 }
 
 // SendMsg is a thread safe method to pass a message to the Coordinator
@@ -215,27 +282,36 @@ func (c *Coordinator) SendMsg(ctx context.Context, msg interface{}) {
 	}
 }
 
+func updateSCVars(vars *synchronizer.SCVariables, update synchronizer.SCVariablesPtr) {
+	if update.Rollup != nil {
+		vars.Rollup = *update.Rollup
+	}
+	if update.Auction != nil {
+		vars.Auction = *update.Auction
+	}
+	if update.WDelayer != nil {
+		vars.WDelayer = *update.WDelayer
+	}
+}
+
 func (c *Coordinator) syncSCVars(vars synchronizer.SCVariablesPtr) {
-	if vars.Rollup != nil {
+	updateSCVars(&c.vars, vars)
-		c.vars.Rollup = *vars.Rollup
-	}
-	if vars.Auction != nil {
-		c.vars.Auction = *vars.Auction
-	}
-	if vars.WDelayer != nil {
-		c.vars.WDelayer = *vars.WDelayer
-	}
 }
 
 func canForge(auctionConstants *common.AuctionConstants, auctionVars *common.AuctionVariables,
 	currentSlot *common.Slot, nextSlot *common.Slot, addr ethCommon.Address, blockNum int64) bool {
+	if blockNum < auctionConstants.GenesisBlockNum {
+		log.Infow("canForge: requested blockNum is < genesis", "blockNum", blockNum,
+			"genesis", auctionConstants.GenesisBlockNum)
+		return false
+	}
 	var slot *common.Slot
 	if currentSlot.StartBlock <= blockNum && blockNum <= currentSlot.EndBlock {
 		slot = currentSlot
 	} else if nextSlot.StartBlock <= blockNum && blockNum <= nextSlot.EndBlock {
 		slot = nextSlot
 	} else {
-		log.Warnw("Coordinator: requested blockNum for canForge is outside slot",
+		log.Warnw("canForge: requested blockNum is outside current and next slot",
 			"blockNum", blockNum, "currentSlot", currentSlot,
 			"nextSlot", nextSlot,
 		)
@@ -244,16 +320,23 @@ func canForge(auctionConstants *common.AuctionConstants, auctionVars *common.Auc
 	anyoneForge := false
 	if !slot.ForgerCommitment &&
 		auctionConstants.RelativeBlock(blockNum) >= int64(auctionVars.SlotDeadline) {
-		log.Debugw("Coordinator: anyone can forge in the current slot (slotDeadline passed)",
+		log.Debugw("canForge: anyone can forge in the current slot (slotDeadline passed)",
 			"block", blockNum)
 		anyoneForge = true
 	}
 	if slot.Forger == addr || anyoneForge {
 		return true
 	}
+	log.Debugw("canForge: can't forge", "slot.Forger", slot.Forger)
 	return false
 }
 
+func (c *Coordinator) canForgeAt(blockNum int64) bool {
+	return canForge(&c.consts.Auction, &c.vars.Auction,
+		&c.stats.Sync.Auction.CurrentSlot, &c.stats.Sync.Auction.NextSlot,
+		c.cfg.ForgerAddress, blockNum)
+}
+
 func (c *Coordinator) canForge() bool {
 	blockNum := c.stats.Eth.LastBlock.Num + 1
 	return canForge(&c.consts.Auction, &c.vars.Auction,
@@ -262,21 +345,39 @@ func (c *Coordinator) canForge() bool {
 }
 
 func (c *Coordinator) syncStats(ctx context.Context, stats *synchronizer.Stats) error {
-	canForge := c.canForge()
+	nextBlock := c.stats.Eth.LastBlock.Num + 1
+	canForge := c.canForgeAt(nextBlock)
+	if c.cfg.ScheduleBatchBlocksAheadCheck != 0 && canForge {
+		canForge = c.canForgeAt(nextBlock + c.cfg.ScheduleBatchBlocksAheadCheck)
+	}
 	if c.pipeline == nil {
-		if canForge {
+		relativeBlock := c.consts.Auction.RelativeBlock(nextBlock)
+		if canForge && relativeBlock < c.cfg.StartSlotBlocksDelay {
+			log.Debugf("Coordinator: delaying pipeline start due to "+
+				"relativeBlock (%v) < cfg.StartSlotBlocksDelay (%v)",
+				relativeBlock, c.cfg.StartSlotBlocksDelay)
+		} else if canForge {
 			log.Infow("Coordinator: forging state begin", "block",
-				stats.Eth.LastBlock.Num+1, "batch", stats.Sync.LastBatch)
+				stats.Eth.LastBlock.Num+1, "batch", stats.Sync.LastBatch.BatchNum)
-			batchNum := common.BatchNum(stats.Sync.LastBatch)
+			fromBatch := fromBatch{
+				BatchNum:   stats.Sync.LastBatch.BatchNum,
+				ForgerAddr: stats.Sync.LastBatch.ForgerAddr,
+				StateRoot:  stats.Sync.LastBatch.StateRoot,
+			}
+			if c.lastNonFailedBatchNum > fromBatch.BatchNum {
+				fromBatch.BatchNum = c.lastNonFailedBatchNum
+				fromBatch.ForgerAddr = c.cfg.ForgerAddress
+				fromBatch.StateRoot = big.NewInt(0)
+			}
 			var err error
 			if c.pipeline, err = c.newPipeline(ctx); err != nil {
 				return tracerr.Wrap(err)
 			}
-			if err := c.pipeline.Start(batchNum, stats, &c.vars); err != nil {
+			c.pipelineFromBatch = fromBatch
+			if err := c.pipeline.Start(fromBatch.BatchNum, stats, &c.vars); err != nil {
 				c.pipeline = nil
 				return tracerr.Wrap(err)
 			}
-			c.pipelineBatchNum = batchNum
 		}
 	} else {
 		if !canForge {
@@ -286,25 +387,12 @@ func (c *Coordinator) syncStats(ctx context.Context, stats *synchronizer.Stats)
 		}
 	}
 	if c.pipeline == nil {
-		// Mark invalid in Pool due to forged L2Txs
+		if _, err := c.purger.InvalidateMaybe(c.l2DB, c.txSelector.LocalAccountsDB(),
-		// for _, batch := range batches {
+			stats.Sync.LastBlock.Num, int64(stats.Sync.LastBatch.BatchNum)); err != nil {
-		// 	if err := c.l2DB.InvalidateOldNonces(
-		// 		idxsNonceFromL2Txs(batch.L2Txs), batch.Batch.BatchNum); err != nil {
-		// 		return err
-		// 	}
-		// }
-		if c.purger.CanInvalidate(stats.Sync.LastBlock.Num, stats.Sync.LastBatch) {
-			if err := c.txSelector.Reset(common.BatchNum(stats.Sync.LastBatch)); err != nil {
-				return tracerr.Wrap(err)
-			}
-		}
-		_, err := c.purger.InvalidateMaybe(c.l2DB, c.txSelector.LocalAccountsDB(),
-			stats.Sync.LastBlock.Num, stats.Sync.LastBatch)
-		if err != nil {
 			return tracerr.Wrap(err)
 		}
-		_, err = c.purger.PurgeMaybe(c.l2DB, stats.Sync.LastBlock.Num, stats.Sync.LastBatch)
+		if _, err := c.purger.PurgeMaybe(c.l2DB, stats.Sync.LastBlock.Num,
-		if err != nil {
+			int64(stats.Sync.LastBatch.BatchNum)); err != nil {
 			return tracerr.Wrap(err)
 		}
 	}
@@ -331,33 +419,43 @@ func (c *Coordinator) handleReorg(ctx context.Context, msg *MsgSyncReorg) error
 	if c.pipeline != nil {
 		c.pipeline.SetSyncStatsVars(ctx, &msg.Stats, &msg.Vars)
 	}
-	if common.BatchNum(c.stats.Sync.LastBatch) < c.pipelineBatchNum {
+	if c.stats.Sync.LastBatch.ForgerAddr != c.cfg.ForgerAddress &&
-		// There's been a reorg and the batch from which the pipeline
+		(c.stats.Sync.LastBatch.StateRoot == nil || c.pipelineFromBatch.StateRoot == nil ||
-		// was started was in a block that was discarded.  The batch
+			c.stats.Sync.LastBatch.StateRoot.Cmp(c.pipelineFromBatch.StateRoot) != 0) {
-		// may not be in the main chain, so we stop the pipeline as a
+		// There's been a reorg and the batch state root from which the
-		// precaution (it will be started again once the node is in
+		// pipeline was started has changed (probably because it was in
-		// sync).
+		// a block that was discarded), and it was sent by a different
-		log.Infow("Coordinator.handleReorg StopPipeline sync.LastBatch < c.pipelineBatchNum",
+		// coordinator than us.  That batch may never be in the main
-			"sync.LastBatch", c.stats.Sync.LastBatch,
+		// chain, so we stop the pipeline  (it will be started again
-			"c.pipelineBatchNum", c.pipelineBatchNum)
+		// once the node is in sync).
-		if err := c.handleStopPipeline(ctx, "reorg"); err != nil {
+		log.Infow("Coordinator.handleReorg StopPipeline sync.LastBatch.ForgerAddr != cfg.ForgerAddr "+
+			"& sync.LastBatch.StateRoot != pipelineFromBatch.StateRoot",
+			"sync.LastBatch.StateRoot", c.stats.Sync.LastBatch.StateRoot,
+			"pipelineFromBatch.StateRoot", c.pipelineFromBatch.StateRoot)
+		c.txManager.DiscardPipeline(ctx, c.pipelineNum)
+		if err := c.handleStopPipeline(ctx, "reorg", 0); err != nil {
 			return tracerr.Wrap(err)
 		}
 	}
 	return nil
 }
 
-func (c *Coordinator) handleStopPipeline(ctx context.Context, reason string) error {
+// handleStopPipeline handles stopping the pipeline.  If failedBatchNum is 0,
+// the next pipeline will start from the last state of the synchronizer,
+// otherwise, it will state from failedBatchNum-1.
+func (c *Coordinator) handleStopPipeline(ctx context.Context, reason string, failedBatchNum common.BatchNum) error {
+	batchNum := c.stats.Sync.LastBatch.BatchNum
+	if failedBatchNum != 0 {
+		batchNum = failedBatchNum - 1
+	}
 	if c.pipeline != nil {
 		c.pipeline.Stop(c.ctx)
 		c.pipeline = nil
 	}
-	if err := c.l2DB.Reorg(common.BatchNum(c.stats.Sync.LastBatch)); err != nil {
+	if err := c.l2DB.Reorg(batchNum); err != nil {
 		return tracerr.Wrap(err)
 	}
-	if strings.Contains(reason, common.AuctionErrMsgCannotForge) { //nolint:staticcheck
+	c.lastNonFailedBatchNum = batchNum
-		// TODO: Check that we are in a slot in which we can't forge
-	}
 	return nil
 }
 
@@ -373,7 +471,7 @@ func (c *Coordinator) handleMsg(ctx context.Context, msg interface{}) error {
 		}
 	case MsgStopPipeline:
 		log.Infow("Coordinator received MsgStopPipeline", "reason", msg.Reason)
-		if err := c.handleStopPipeline(ctx, msg.Reason); err != nil {
+		if err := c.handleStopPipeline(ctx, msg.Reason, msg.FailedBatchNum); err != nil {
 			return tracerr.Wrap(fmt.Errorf("Coordinator.handleStopPipeline: %w", err))
 		}
 	default:
@@ -396,7 +494,7 @@ func (c *Coordinator) Start() {
 
 	c.wg.Add(1)
 	go func() {
-		waitDuration := longWaitDuration
+		waitCh := time.After(longWaitDuration)
 		for {
 			select {
 			case <-c.ctx.Done():
@@ -408,23 +506,24 @@ func (c *Coordinator) Start() {
 					continue
 				} else if err != nil {
 					log.Errorw("Coordinator.handleMsg", "err", err)
-					waitDuration = c.cfg.SyncRetryInterval
+					waitCh = time.After(c.cfg.SyncRetryInterval)
 					continue
 				}
-				waitDuration = longWaitDuration
+				waitCh = time.After(longWaitDuration)
-			case <-time.After(waitDuration):
+			case <-waitCh:
 				if !c.stats.Synced() {
-					waitDuration = longWaitDuration
+					waitCh = time.After(longWaitDuration)
 					continue
 				}
 				if err := c.syncStats(c.ctx, &c.stats); c.ctx.Err() != nil {
+					waitCh = time.After(longWaitDuration)
 					continue
 				} else if err != nil {
 					log.Errorw("Coordinator.syncStats", "err", err)
-					waitDuration = c.cfg.SyncRetryInterval
+					waitCh = time.After(c.cfg.SyncRetryInterval)
 					continue
 				}
-				waitDuration = longWaitDuration
+				waitCh = time.After(longWaitDuration)
 			}
 		}
 	}()

coordinator/coordinator_test.go

@@ -261,8 +261,8 @@ func TestCoordinatorFlow(t *testing.T) {
 			var stats synchronizer.Stats
 			stats.Eth.LastBlock = *ethClient.CtlLastBlock()
 			stats.Sync.LastBlock = stats.Eth.LastBlock
-			stats.Eth.LastBatch = ethClient.CtlLastForgedBatch()
+			stats.Eth.LastBatchNum = ethClient.CtlLastForgedBatch()
-			stats.Sync.LastBatch = stats.Eth.LastBatch
+			stats.Sync.LastBatch.BatchNum = common.BatchNum(stats.Eth.LastBatchNum)
 			canForge, err := ethClient.AuctionCanForge(forger, blockNum+1)
 			require.NoError(t, err)
 			var slot common.Slot
@@ -279,7 +279,7 @@ func TestCoordinatorFlow(t *testing.T) {
 			// Copy stateDB to synchronizer if there was a new batch
 			source := fmt.Sprintf("%v/BatchNum%v", batchBuilderDBPath, stats.Sync.LastBatch)
 			dest := fmt.Sprintf("%v/BatchNum%v", syncDBPath, stats.Sync.LastBatch)
-			if stats.Sync.LastBatch != 0 {
+			if stats.Sync.LastBatch.BatchNum != 0 {
 				if _, err := os.Stat(dest); os.IsNotExist(err) {
 					log.Infow("Making pebble checkpoint for sync",
 						"source", source, "dest", dest)

coordinator/pipeline.go

@@ -2,6 +2,7 @@ package coordinator
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"math/big"
 	"sync"
@@ -24,19 +25,29 @@ type statsVars struct {
 	Vars  synchronizer.SCVariablesPtr
 }
 
+type state struct {
+	batchNum                     common.BatchNum
+	lastScheduledL1BatchBlockNum int64
+	lastForgeL1TxsNum            int64
+	lastSlotForged               int64
+}
+
 // Pipeline manages the forging of batches with parallel server proofs
 type Pipeline struct {
+	num    int
 	cfg    Config
 	consts synchronizer.SCConsts
 
 	// state
-	batchNum                     common.BatchNum
+	state         state
-	lastScheduledL1BatchBlockNum int64
+	started       bool
-	lastForgeL1TxsNum            int64
+	rw            sync.RWMutex
-	started                      bool
+	errAtBatchNum common.BatchNum
+	lastForgeTime time.Time
 
 	proversPool  *ProversPool
 	provers      []prover.Client
+	coord        *Coordinator
 	txManager    *TxManager
 	historyDB    *historydb.HistoryDB
 	l2DB         *l2db.L2DB
@@ -53,14 +64,28 @@ type Pipeline struct {
 	cancel context.CancelFunc
 }
 
+func (p *Pipeline) setErrAtBatchNum(batchNum common.BatchNum) {
+	p.rw.Lock()
+	defer p.rw.Unlock()
+	p.errAtBatchNum = batchNum
+}
+
+func (p *Pipeline) getErrAtBatchNum() common.BatchNum {
+	p.rw.RLock()
+	defer p.rw.RUnlock()
+	return p.errAtBatchNum
+}
+
 // NewPipeline creates a new Pipeline
 func NewPipeline(ctx context.Context,
 	cfg Config,
+	num int, // Pipeline sequential number
 	historyDB *historydb.HistoryDB,
 	l2DB *l2db.L2DB,
 	txSelector *txselector.TxSelector,
 	batchBuilder *batchbuilder.BatchBuilder,
 	purger *Purger,
+	coord *Coordinator,
 	txManager *TxManager,
 	provers []prover.Client,
 	scConsts *synchronizer.SCConsts,
@@ -79,6 +104,7 @@ func NewPipeline(ctx context.Context,
 		return nil, tracerr.Wrap(fmt.Errorf("no provers in the pool"))
 	}
 	return &Pipeline{
+		num:          num,
 		cfg:          cfg,
 		historyDB:    historyDB,
 		l2DB:         l2DB,
@@ -87,6 +113,7 @@ func NewPipeline(ctx context.Context,
 		provers:      provers,
 		proversPool:  proversPool,
 		purger:       purger,
+		coord:        coord,
 		txManager:    txManager,
 		consts:       *scConsts,
 		statsVarsCh:  make(chan statsVars, queueLen),
@@ -104,33 +131,68 @@ func (p *Pipeline) SetSyncStatsVars(ctx context.Context, stats *synchronizer.Sta
 // reset pipeline state
 func (p *Pipeline) reset(batchNum common.BatchNum,
 	stats *synchronizer.Stats, vars *synchronizer.SCVariables) error {
-	p.batchNum = batchNum
+	p.state = state{
-	p.lastForgeL1TxsNum = stats.Sync.LastForgeL1TxsNum
+		batchNum:                     batchNum,
+		lastForgeL1TxsNum:            stats.Sync.LastForgeL1TxsNum,
+		lastScheduledL1BatchBlockNum: 0,
+		lastSlotForged:               -1,
+	}
 	p.stats = *stats
 	p.vars = *vars
-	p.lastScheduledL1BatchBlockNum = 0
 
-	err := p.txSelector.Reset(p.batchNum)
+	// Reset the StateDB in TxSelector and BatchBuilder from the
+	// synchronizer only if the checkpoint we reset from either:
+	// a. Doesn't exist in the TxSelector/BatchBuilder
+	// b. The batch has already been synced by the synchronizer and has a
+	//    different MTRoot than the BatchBuilder
+	// Otherwise, reset from the local checkpoint.
+
+	// First attempt to reset from local checkpoint if such checkpoint exists
+	existsTxSelector, err := p.txSelector.LocalAccountsDB().CheckpointExists(p.state.batchNum)
 	if err != nil {
 		return tracerr.Wrap(err)
 	}
-	err = p.batchBuilder.Reset(p.batchNum, true)
+	fromSynchronizerTxSelector := !existsTxSelector
+	if err := p.txSelector.Reset(p.state.batchNum, fromSynchronizerTxSelector); err != nil {
+		return tracerr.Wrap(err)
+	}
+	existsBatchBuilder, err := p.batchBuilder.LocalStateDB().CheckpointExists(p.state.batchNum)
 	if err != nil {
 		return tracerr.Wrap(err)
 	}
+	fromSynchronizerBatchBuilder := !existsBatchBuilder
+	if err := p.batchBuilder.Reset(p.state.batchNum, fromSynchronizerBatchBuilder); err != nil {
+		return tracerr.Wrap(err)
+	}
+
+	// After reset, check that if the batch exists in the historyDB, the
+	// stateRoot matches with the local one, if not, force a reset from
+	// synchronizer
+	batch, err := p.historyDB.GetBatch(p.state.batchNum)
+	if tracerr.Unwrap(err) == sql.ErrNoRows {
+		// nothing to do
+	} else if err != nil {
+		return tracerr.Wrap(err)
+	} else {
+		localStateRoot := p.batchBuilder.LocalStateDB().MT.Root().BigInt()
+		if batch.StateRoot.Cmp(localStateRoot) != 0 {
+			log.Debugw("localStateRoot (%v) != historyDB stateRoot (%v).  "+
+				"Forcing reset from Synchronizer", localStateRoot, batch.StateRoot)
+			// StateRoot from synchronizer doesn't match StateRoot
+			// from batchBuilder, force a reset from synchronizer
+			if err := p.txSelector.Reset(p.state.batchNum, true); err != nil {
+				return tracerr.Wrap(err)
+			}
+			if err := p.batchBuilder.Reset(p.state.batchNum, true); err != nil {
+				return tracerr.Wrap(err)
+			}
+		}
+	}
 	return nil
 }
 
 func (p *Pipeline) syncSCVars(vars synchronizer.SCVariablesPtr) {
-	if vars.Rollup != nil {
+	updateSCVars(&p.vars, vars)
-		p.vars.Rollup = *vars.Rollup
-	}
-	if vars.Auction != nil {
-		p.vars.Auction = *vars.Auction
-	}
-	if vars.WDelayer != nil {
-		p.vars.WDelayer = *vars.WDelayer
-	}
 }
 
 // handleForgeBatch calls p.forgeBatch to forge the batch and get the zkInputs,
@@ -143,8 +205,11 @@ func (p *Pipeline) handleForgeBatch(ctx context.Context, batchNum common.BatchNu
 	} else if err != nil {
 		if tracerr.Unwrap(err) == errLastL1BatchNotSynced {
 			log.Warnw("forgeBatch: scheduled L1Batch too early", "err", err,
-				"lastForgeL1TxsNum", p.lastForgeL1TxsNum,
+				"lastForgeL1TxsNum", p.state.lastForgeL1TxsNum,
 				"syncLastForgeL1TxsNum", p.stats.Sync.LastForgeL1TxsNum)
+		} else if tracerr.Unwrap(err) == errForgeNoTxsBeforeDelay ||
+			tracerr.Unwrap(err) == errForgeBeforeDelay {
+			// no log
 		} else {
 			log.Errorw("forgeBatch", "err", err)
 		}
@@ -188,7 +253,7 @@ func (p *Pipeline) Start(batchNum common.BatchNum,
 
 	p.wg.Add(1)
 	go func() {
-		waitDuration := zeroDuration
+		waitCh := time.After(zeroDuration)
 		for {
 			select {
 			case <-p.ctx.Done():
@@ -198,20 +263,42 @@ func (p *Pipeline) Start(batchNum common.BatchNum,
 			case statsVars := <-p.statsVarsCh:
 				p.stats = statsVars.Stats
 				p.syncSCVars(statsVars.Vars)
-			case <-time.After(waitDuration):
+			case <-waitCh:
-				batchNum = p.batchNum + 1
+				// Once errAtBatchNum != 0, we stop forging
-				batchInfo, err := p.handleForgeBatch(p.ctx, batchNum)
+				// batches because there's been an error and we
-				if p.ctx.Err() != nil {
+				// wait for the pipeline to be stopped.
-					continue
+				if p.getErrAtBatchNum() != 0 {
-				} else if err != nil {
+					waitCh = time.After(p.cfg.ForgeRetryInterval)
-					waitDuration = p.cfg.SyncRetryInterval
 					continue
 				}
-				p.batchNum = batchNum
+				batchNum = p.state.batchNum + 1
+				batchInfo, err := p.handleForgeBatch(p.ctx, batchNum)
+				if p.ctx.Err() != nil {
+					waitCh = time.After(p.cfg.ForgeRetryInterval)
+					continue
+				} else if tracerr.Unwrap(err) == errLastL1BatchNotSynced ||
+					tracerr.Unwrap(err) == errForgeNoTxsBeforeDelay ||
+					tracerr.Unwrap(err) == errForgeBeforeDelay {
+					waitCh = time.After(p.cfg.ForgeRetryInterval)
+					continue
+				} else if err != nil {
+					p.setErrAtBatchNum(batchNum)
+					p.coord.SendMsg(p.ctx, MsgStopPipeline{
+						Reason: fmt.Sprintf(
+							"Pipeline.handleForgBatch: %v", err),
+						FailedBatchNum: batchNum,
+					})
+					waitCh = time.After(p.cfg.ForgeRetryInterval)
+					continue
+				}
+				p.lastForgeTime = time.Now()
+
+				p.state.batchNum = batchNum
 				select {
 				case batchChSentServerProof <- batchInfo:
 				case <-p.ctx.Done():
 				}
+				waitCh = time.After(zeroDuration)
 			}
 		}
 	}()
@@ -225,16 +312,28 @@ func (p *Pipeline) Start(batchNum common.BatchNum,
 				p.wg.Done()
 				return
 			case batchInfo := <-batchChSentServerProof:
+				// Once errAtBatchNum != 0, we stop forging
+				// batches because there's been an error and we
+				// wait for the pipeline to be stopped.
+				if p.getErrAtBatchNum() != 0 {
+					continue
+				}
 				err := p.waitServerProof(p.ctx, batchInfo)
-				// We are done with this serverProof, add it back to the pool
-				p.proversPool.Add(p.ctx, batchInfo.ServerProof)
-				batchInfo.ServerProof = nil
 				if p.ctx.Err() != nil {
 					continue
 				} else if err != nil {
 					log.Errorw("waitServerProof", "err", err)
+					p.setErrAtBatchNum(batchInfo.BatchNum)
+					p.coord.SendMsg(p.ctx, MsgStopPipeline{
+						Reason: fmt.Sprintf(
+							"Pipeline.waitServerProof: %v", err),
+						FailedBatchNum: batchInfo.BatchNum,
+					})
 					continue
 				}
+				// We are done with this serverProof, add it back to the pool
+				p.proversPool.Add(p.ctx, batchInfo.ServerProof)
+				// batchInfo.ServerProof = nil
 				p.txManager.AddBatch(p.ctx, batchInfo)
 			}
 		}
@@ -284,9 +383,10 @@ func (p *Pipeline) forgeBatch(batchNum common.BatchNum) (batchInfo *BatchInfo, e
 	if err != nil {
 		return nil, tracerr.Wrap(err)
 	}
-
+	// Structure to accumulate data and metadata of the batch
-	batchInfo = &BatchInfo{BatchNum: batchNum} // to accumulate metadata of the batch
+	now := time.Now()
-	batchInfo.Debug.StartTimestamp = time.Now()
+	batchInfo = &BatchInfo{PipelineNum: p.num, BatchNum: batchNum}
+	batchInfo.Debug.StartTimestamp = now
 	batchInfo.Debug.StartBlockNum = p.stats.Eth.LastBlock.Num + 1
 
 	selectionCfg := &txselector.SelectionConfig{
@@ -300,22 +400,26 @@ func (p *Pipeline) forgeBatch(batchNum common.BatchNum) (batchInfo *BatchInfo, e
 	var auths [][]byte
 	var coordIdxs []common.Idx
 
+	// Check if the slot is not yet fulfilled
+	slotCommitted := false
+	if p.stats.Sync.Auction.CurrentSlot.ForgerCommitment ||
+		p.stats.Sync.Auction.CurrentSlot.SlotNum == p.state.lastSlotForged {
+		slotCommitted = true
+	}
+
+	// If we haven't reached the ForgeDelay, skip forging the batch
+	if slotCommitted && now.Sub(p.lastForgeTime) < p.cfg.ForgeDelay {
+		return nil, errForgeBeforeDelay
+	}
+
 	// 1. Decide if we forge L2Tx or L1+L2Tx
 	if p.shouldL1L2Batch(batchInfo) {
 		batchInfo.L1Batch = true
-		defer func() {
+		if p.state.lastForgeL1TxsNum != p.stats.Sync.LastForgeL1TxsNum {
-			// If there's no error, update the parameters related
-			// to the last L1Batch forged
-			if err == nil {
-				p.lastScheduledL1BatchBlockNum = p.stats.Eth.LastBlock.Num + 1
-				p.lastForgeL1TxsNum++
-			}
-		}()
-		if p.lastForgeL1TxsNum != p.stats.Sync.LastForgeL1TxsNum {
 			return nil, tracerr.Wrap(errLastL1BatchNotSynced)
 		}
 		// 2a: L1+L2 txs
-		l1UserTxs, err := p.historyDB.GetUnforgedL1UserTxs(p.lastForgeL1TxsNum + 1)
+		l1UserTxs, err := p.historyDB.GetUnforgedL1UserTxs(p.state.lastForgeL1TxsNum + 1)
 		if err != nil {
 			return nil, tracerr.Wrap(err)
 		}
@@ -334,6 +438,43 @@ func (p *Pipeline) forgeBatch(batchNum common.BatchNum) (batchInfo *BatchInfo, e
 		l1UserTxsExtra = nil
 	}
 
+	// If there are no txs to forge, no l1UserTxs in the open queue to
+	// freeze, and we haven't reached the ForgeNoTxsDelay, skip forging the
+	// batch.
+	if slotCommitted && now.Sub(p.lastForgeTime) < p.cfg.ForgeNoTxsDelay {
+		noTxs := false
+		if len(l1UserTxsExtra) == 0 && len(l1CoordTxs) == 0 && len(poolL2Txs) == 0 {
+			if batchInfo.L1Batch {
+				// Query the L1UserTxs in the queue following
+				// the one we are trying to forge.
+				nextL1UserTxs, err := p.historyDB.GetUnforgedL1UserTxs(
+					p.state.lastForgeL1TxsNum + 1)
+				if err != nil {
+					return nil, tracerr.Wrap(err)
+				}
+				// If there are future L1UserTxs, we forge a
+				// batch to advance the queues and forge the
+				// L1UserTxs in the future.  Otherwise, skip.
+				if len(nextL1UserTxs) == 0 {
+					noTxs = true
+				}
+			} else {
+				noTxs = true
+			}
+		}
+		if noTxs {
+			if err := p.txSelector.Reset(batchInfo.BatchNum-1, false); err != nil {
+				return nil, tracerr.Wrap(err)
+			}
+			return nil, errForgeNoTxsBeforeDelay
+		}
+	}
+
+	if batchInfo.L1Batch {
+		p.state.lastScheduledL1BatchBlockNum = p.stats.Eth.LastBlock.Num + 1
+		p.state.lastForgeL1TxsNum++
+	}
+
 	// 3.  Save metadata from TxSelector output for BatchNum
 	batchInfo.L1UserTxsExtra = l1UserTxsExtra
 	batchInfo.L1CoordTxs = l1CoordTxs
@@ -378,6 +519,8 @@ func (p *Pipeline) forgeBatch(batchNum common.BatchNum) (batchInfo *BatchInfo, e
 	p.cfg.debugBatchStore(batchInfo)
 	log.Infow("Pipeline: batch forged internally", "batch", batchInfo.BatchNum)
 
+	p.state.lastSlotForged = p.stats.Sync.Auction.CurrentSlot.SlotNum
+
 	return batchInfo, nil
 }
 
@@ -399,12 +542,12 @@ func (p *Pipeline) waitServerProof(ctx context.Context, batchInfo *BatchInfo) er
 func (p *Pipeline) shouldL1L2Batch(batchInfo *BatchInfo) bool {
 	// Take the lastL1BatchBlockNum as the biggest between the last
 	// scheduled one, and the synchronized one.
-	lastL1BatchBlockNum := p.lastScheduledL1BatchBlockNum
+	lastL1BatchBlockNum := p.state.lastScheduledL1BatchBlockNum
 	if p.stats.Sync.LastL1BatchBlock > lastL1BatchBlockNum {
 		lastL1BatchBlockNum = p.stats.Sync.LastL1BatchBlock
 	}
 	// Set Debug information
-	batchInfo.Debug.LastScheduledL1BatchBlockNum = p.lastScheduledL1BatchBlockNum
+	batchInfo.Debug.LastScheduledL1BatchBlockNum = p.state.lastScheduledL1BatchBlockNum
 	batchInfo.Debug.LastL1BatchBlock = p.stats.Sync.LastL1BatchBlock
 	batchInfo.Debug.LastL1BatchBlockDelta = p.stats.Eth.LastBlock.Num + 1 - lastL1BatchBlockNum
 	batchInfo.Debug.L1BatchBlockScheduleDeadline =

coordinator/pipeline_test.go

@@ -25,6 +25,14 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func newBigInt(s string) *big.Int {
+	v, ok := new(big.Int).SetString(s, 10)
+	if !ok {
+		panic(fmt.Errorf("Can't set big.Int from %s", s))
+	}
+	return v
+}
+
 func TestPipelineShouldL1L2Batch(t *testing.T) {
 	ethClientSetup := test.NewClientSetupExample()
 	ethClientSetup.ChainID = big.NewInt(int64(chainID))
@@ -77,7 +85,7 @@ func TestPipelineShouldL1L2Batch(t *testing.T) {
 	//
 	// Scheduled L1Batch
 	//
-	pipeline.lastScheduledL1BatchBlockNum = startBlock
+	pipeline.state.lastScheduledL1BatchBlockNum = startBlock
 	stats.Sync.LastL1BatchBlock = startBlock - 10
 
 	// We are are one block before the timeout range * 0.5
@@ -128,6 +136,11 @@ func preloadSync(t *testing.T, ethClient *test.Client, sync *synchronizer.Synchr
 	blocks, err := tc.GenerateBlocksFromInstructions(set)
 	require.NoError(t, err)
 	require.NotNil(t, blocks)
+	// Set StateRoots for batches manually (til doesn't set it)
+	blocks[0].Rollup.Batches[0].Batch.StateRoot =
+		newBigInt("0")
+	blocks[0].Rollup.Batches[1].Batch.StateRoot =
+		newBigInt("10941365282189107056349764238909072001483688090878331371699519307087372995595")
 
 	ethAddTokens(blocks, ethClient)
 	err = ethClient.CtlAddBlocks(blocks)
@@ -172,7 +185,7 @@ func TestPipelineForgeBatchWithTxs(t *testing.T) {
 	// users with positive balances
 	tilCtx := preloadSync(t, ethClient, sync, modules.historyDB, modules.stateDB)
 	syncStats := sync.Stats()
-	batchNum := common.BatchNum(syncStats.Sync.LastBatch)
+	batchNum := syncStats.Sync.LastBatch.BatchNum
 	syncSCVars := sync.SCVars()
 
 	pipeline, err := coord.newPipeline(ctx)

coordinator/purger.go

@@ -13,13 +13,23 @@ import (
 
 // PurgerCfg is the purger configuration
 type PurgerCfg struct {
-	// PurgeBatchDelay is the delay between batches to purge outdated transactions
+	// PurgeBatchDelay is the delay between batches to purge outdated
+	// transactions.  Oudated L2Txs are those that have been forged or
+	// marked as invalid for longer than the SafetyPeriod and pending L2Txs
+	// that have been in the pool for longer than TTL once there are
+	// MaxTxs.
 	PurgeBatchDelay int64
-	// InvalidateBatchDelay is the delay between batches to mark invalid transactions
+	// InvalidateBatchDelay is the delay between batches to mark invalid
+	// transactions due to nonce lower than the account nonce.
 	InvalidateBatchDelay int64
-	// PurgeBlockDelay is the delay between blocks to purge outdated transactions
+	// PurgeBlockDelay is the delay between blocks to purge outdated
+	// transactions.  Oudated L2Txs are those that have been forged or
+	// marked as invalid for longer than the SafetyPeriod and pending L2Txs
+	// that have been in the pool for longer than TTL once there are
+	// MaxTxs.
 	PurgeBlockDelay int64
-	// InvalidateBlockDelay is the delay between blocks to mark invalid transactions
+	// InvalidateBlockDelay is the delay between blocks to mark invalid
+	// transactions due to nonce lower than the account nonce.
 	InvalidateBlockDelay int64
 }
 

coordinator/txmanager.go

@@ -2,6 +2,7 @@ package coordinator
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/big"
 	"time"
@@ -9,6 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db/l2db"
@@ -35,12 +37,20 @@ type TxManager struct {
 	vars        synchronizer.SCVariables
 	statsVarsCh chan statsVars
 
-	queue []*BatchInfo
+	discardPipelineCh chan int // int refers to the pipelineNum
+
+	minPipelineNum int
+	queue          Queue
 	// lastSuccessBatch stores the last BatchNum that who's forge call was confirmed
 	lastSuccessBatch common.BatchNum
-	lastPendingBatch common.BatchNum
+	// lastPendingBatch common.BatchNum
-	lastSuccessNonce uint64
+	// accNonce is the account nonce in the last mined block (due to mined txs)
-	lastPendingNonce uint64
+	accNonce uint64
+	// accNextNonce is the nonce that we should use to send the next tx.
+	// In some cases this will be a reused nonce of an already pending tx.
+	accNextNonce uint64
+
+	lastSentL1BatchBlockNum int64
 }
 
 // NewTxManager creates a new TxManager
@@ -54,26 +64,19 @@ func NewTxManager(ctx context.Context, cfg *Config, ethClient eth.ClientInterfac
 	if err != nil {
 		return nil, tracerr.Wrap(err)
 	}
-	lastSuccessNonce, err := ethClient.EthNonceAt(ctx, *address, nil)
+	accNonce, err := ethClient.EthNonceAt(ctx, *address, nil)
 	if err != nil {
 		return nil, err
 	}
-	lastPendingNonce, err := ethClient.EthPendingNonceAt(ctx, *address)
+	log.Infow("TxManager started", "nonce", accNonce)
-	if err != nil {
-		return nil, err
-	}
-	if lastSuccessNonce != lastPendingNonce {
-		return nil, tracerr.Wrap(fmt.Errorf("lastSuccessNonce (%v) != lastPendingNonce (%v)",
-			lastSuccessNonce, lastPendingNonce))
-	}
-	log.Infow("TxManager started", "nonce", lastSuccessNonce)
 	return &TxManager{
-		cfg:         *cfg,
+		cfg:               *cfg,
-		ethClient:   ethClient,
+		ethClient:         ethClient,
-		l2DB:        l2DB,
+		l2DB:              l2DB,
-		coord:       coord,
+		coord:             coord,
-		batchCh:     make(chan *BatchInfo, queueLen),
+		batchCh:           make(chan *BatchInfo, queueLen),
-		statsVarsCh: make(chan statsVars, queueLen),
+		statsVarsCh:       make(chan statsVars, queueLen),
+		discardPipelineCh: make(chan int, queueLen),
 		account: accounts.Account{
 			Address: *address,
 		},
@@ -82,8 +85,10 @@ func NewTxManager(ctx context.Context, cfg *Config, ethClient eth.ClientInterfac
 
 		vars: *initSCVars,
 
-		lastSuccessNonce: lastSuccessNonce,
+		minPipelineNum: 0,
-		lastPendingNonce: lastPendingNonce,
+		queue:          NewQueue(),
+		accNonce:       accNonce,
+		accNextNonce:   accNonce,
 	}, nil
 }
 
@@ -104,16 +109,17 @@ func (t *TxManager) SetSyncStatsVars(ctx context.Context, stats *synchronizer.St
 	}
 }
 
+// DiscardPipeline is a thread safe method to notify about a discarded pipeline
+// due to a reorg
+func (t *TxManager) DiscardPipeline(ctx context.Context, pipelineNum int) {
+	select {
+	case t.discardPipelineCh <- pipelineNum:
+	case <-ctx.Done():
+	}
+}
+
 func (t *TxManager) syncSCVars(vars synchronizer.SCVariablesPtr) {
-	if vars.Rollup != nil {
+	updateSCVars(&t.vars, vars)
-		t.vars.Rollup = *vars.Rollup
-	}
-	if vars.Auction != nil {
-		t.vars.Auction = *vars.Auction
-	}
-	if vars.WDelayer != nil {
-		t.vars.WDelayer = *vars.WDelayer
-	}
 }
 
 // NewAuth generates a new auth object for an ethereum transaction
@@ -122,10 +128,14 @@ func (t *TxManager) NewAuth(ctx context.Context) (*bind.TransactOpts, error) {
 	if err != nil {
 		return nil, tracerr.Wrap(err)
 	}
-	inc := new(big.Int).Set(gasPrice)
+	if t.cfg.GasPriceIncPerc != 0 {
-	const gasPriceDiv = 100
+		inc := new(big.Int).Set(gasPrice)
-	inc.Div(inc, new(big.Int).SetUint64(gasPriceDiv))
+		inc.Mul(inc, new(big.Int).SetInt64(t.cfg.GasPriceIncPerc))
-	gasPrice.Add(gasPrice, inc)
+		// nolint reason: to calculate percentages we use 100
+		inc.Div(inc, new(big.Int).SetUint64(100)) //nolint:gomnd
+		gasPrice.Add(gasPrice, inc)
+	}
+
 	// log.Debugw("TxManager: transaction metadata", "gasPrice", gasPrice)
 
 	auth, err := bind.NewKeyStoreTransactorWithChainID(t.ethClient.EthKeyStore(), t.account, t.chainID)
@@ -134,6 +144,13 @@ func (t *TxManager) NewAuth(ctx context.Context) (*bind.TransactOpts, error) {
 	}
 	auth.Value = big.NewInt(0) // in wei
 	// TODO: Calculate GasLimit based on the contents of the ForgeBatchArgs
+	// This requires a function that estimates the gas usage of the
+	// forgeBatch call based on the contents of the ForgeBatch args:
+	// - length of l2txs
+	// - length of l1Usertxs
+	// - length of l1CoordTxs with authorization signature
+	// - length of l1CoordTxs without authoriation signature
+	// - etc.
 	auth.GasLimit = 1000000
 	auth.GasPrice = gasPrice
 	auth.Nonce = nil
@@ -141,29 +158,75 @@ func (t *TxManager) NewAuth(ctx context.Context) (*bind.TransactOpts, error) {
 	return auth, nil
 }
 
-func (t *TxManager) sendRollupForgeBatch(ctx context.Context, batchInfo *BatchInfo) error {
+func (t *TxManager) shouldSendRollupForgeBatch(batchInfo *BatchInfo) error {
-	// TODO: Check if we can forge in the next blockNum, abort if we can't
+	nextBlock := t.stats.Eth.LastBlock.Num + 1
-	batchInfo.Debug.Status = StatusSent
+	if !t.canForgeAt(nextBlock) {
-	batchInfo.Debug.SendBlockNum = t.stats.Eth.LastBlock.Num + 1
+		return tracerr.Wrap(fmt.Errorf("can't forge in the next block: %v", nextBlock))
-	batchInfo.Debug.SendTimestamp = time.Now()
+	}
-	batchInfo.Debug.StartToSendDelay = batchInfo.Debug.SendTimestamp.Sub(
+	if t.mustL1L2Batch(nextBlock) && !batchInfo.L1Batch {
-		batchInfo.Debug.StartTimestamp).Seconds()
+		return tracerr.Wrap(fmt.Errorf("can't forge non-L1Batch in the next block: %v", nextBlock))
+	}
+	margin := t.cfg.SendBatchBlocksMarginCheck
+	if margin != 0 {
+		if !t.canForgeAt(nextBlock + margin) {
+			return tracerr.Wrap(fmt.Errorf("can't forge after %v blocks: %v",
+				margin, nextBlock))
+		}
+		if t.mustL1L2Batch(nextBlock+margin) && !batchInfo.L1Batch {
+			return tracerr.Wrap(fmt.Errorf("can't forge non-L1Batch after %v blocks: %v",
+				margin, nextBlock))
+		}
+	}
+	return nil
+}
+
+func addPerc(v *big.Int, p int64) *big.Int {
+	r := new(big.Int).Set(v)
+	r.Mul(r, big.NewInt(p))
+	// nolint reason: to calculate percentages we divide by 100
+	r.Div(r, big.NewInt(100)) //nolit:gomnd
+	return r.Add(v, r)
+}
+
+func (t *TxManager) sendRollupForgeBatch(ctx context.Context, batchInfo *BatchInfo, resend bool) error {
 	var ethTx *types.Transaction
 	var err error
 	auth, err := t.NewAuth(ctx)
 	if err != nil {
 		return tracerr.Wrap(err)
 	}
-	auth.Nonce = big.NewInt(int64(t.lastPendingNonce))
+	auth.Nonce = big.NewInt(int64(t.accNextNonce))
-	t.lastPendingNonce++
+	if resend {
+		auth.Nonce = big.NewInt(int64(batchInfo.EthTx.Nonce()))
+	}
 	for attempt := 0; attempt < t.cfg.EthClientAttempts; attempt++ {
+		if auth.GasPrice.Cmp(t.cfg.MaxGasPrice) > 0 {
+			return tracerr.Wrap(fmt.Errorf("calculated gasPrice (%v) > maxGasPrice (%v)",
+				auth.GasPrice, t.cfg.MaxGasPrice))
+		}
+		// RollupForgeBatch() calls ethclient.SendTransaction()
 		ethTx, err = t.ethClient.RollupForgeBatch(batchInfo.ForgeBatchArgs, auth)
-		if err != nil {
+		if errors.Is(err, core.ErrNonceTooLow) {
-			// if strings.Contains(err.Error(), common.AuctionErrMsgCannotForge) {
+			log.Warnw("TxManager ethClient.RollupForgeBatch incrementing nonce",
-			// 	log.Errorw("TxManager ethClient.RollupForgeBatch", "err", err,
+				"err", err, "nonce", auth.Nonce, "batchNum", batchInfo.BatchNum)
-			// 		"block", t.stats.Eth.LastBlock.Num+1)
+			auth.Nonce.Add(auth.Nonce, big.NewInt(1))
-			// 	return tracerr.Wrap(err)
+			attempt--
-			// }
+		} else if errors.Is(err, core.ErrNonceTooHigh) {
+			log.Warnw("TxManager ethClient.RollupForgeBatch decrementing nonce",
+				"err", err, "nonce", auth.Nonce, "batchNum", batchInfo.BatchNum)
+			auth.Nonce.Sub(auth.Nonce, big.NewInt(1))
+			attempt--
+		} else if errors.Is(err, core.ErrUnderpriced) {
+			log.Warnw("TxManager ethClient.RollupForgeBatch incrementing gasPrice",
+				"err", err, "gasPrice", auth.GasPrice, "batchNum", batchInfo.BatchNum)
+			auth.GasPrice = addPerc(auth.GasPrice, 10)
+			attempt--
+		} else if errors.Is(err, core.ErrReplaceUnderpriced) {
+			log.Warnw("TxManager ethClient.RollupForgeBatch incrementing gasPrice",
+				"err", err, "gasPrice", auth.GasPrice, "batchNum", batchInfo.BatchNum)
+			auth.GasPrice = addPerc(auth.GasPrice, 10)
+			attempt--
+		} else if err != nil {
 			log.Errorw("TxManager ethClient.RollupForgeBatch",
 				"attempt", attempt, "err", err, "block", t.stats.Eth.LastBlock.Num+1,
 				"batchNum", batchInfo.BatchNum)
@@ -179,10 +242,29 @@ func (t *TxManager) sendRollupForgeBatch(ctx context.Context, batchInfo *BatchIn
 	if err != nil {
 		return tracerr.Wrap(fmt.Errorf("reached max attempts for ethClient.RollupForgeBatch: %w", err))
 	}
+	if !resend {
+		t.accNextNonce = auth.Nonce.Uint64() + 1
+	}
 	batchInfo.EthTx = ethTx
-	log.Infow("TxManager ethClient.RollupForgeBatch", "batch", batchInfo.BatchNum, "tx", ethTx.Hash().Hex())
+	log.Infow("TxManager ethClient.RollupForgeBatch", "batch", batchInfo.BatchNum, "tx", ethTx.Hash())
+	now := time.Now()
+	batchInfo.SendTimestamp = now
+
+	if resend {
+		batchInfo.Debug.ResendNum++
+	}
+	batchInfo.Debug.Status = StatusSent
+	batchInfo.Debug.SendBlockNum = t.stats.Eth.LastBlock.Num + 1
+	batchInfo.Debug.SendTimestamp = batchInfo.SendTimestamp
+	batchInfo.Debug.StartToSendDelay = batchInfo.Debug.SendTimestamp.Sub(
+		batchInfo.Debug.StartTimestamp).Seconds()
 	t.cfg.debugBatchStore(batchInfo)
-	t.lastPendingBatch = batchInfo.BatchNum
+
+	if !resend {
+		if batchInfo.L1Batch {
+			t.lastSentL1BatchBlockNum = t.stats.Eth.LastBlock.Num + 1
+		}
+	}
 	if err := t.l2DB.DoneForging(common.TxIDsFromL2Txs(batchInfo.L2Txs), batchInfo.BatchNum); err != nil {
 		return tracerr.Wrap(err)
 	}
@@ -225,13 +307,20 @@ func (t *TxManager) checkEthTransactionReceipt(ctx context.Context, batchInfo *B
 func (t *TxManager) handleReceipt(ctx context.Context, batchInfo *BatchInfo) (*int64, error) {
 	receipt := batchInfo.Receipt
 	if receipt != nil {
+		if batchInfo.EthTx.Nonce()+1 > t.accNonce {
+			t.accNonce = batchInfo.EthTx.Nonce() + 1
+		}
 		if receipt.Status == types.ReceiptStatusFailed {
 			batchInfo.Debug.Status = StatusFailed
-			t.cfg.debugBatchStore(batchInfo)
 			_, err := t.ethClient.EthCall(ctx, batchInfo.EthTx, receipt.BlockNumber)
-			log.Warnw("TxManager receipt status is failed", "tx", receipt.TxHash.Hex(),
+			log.Warnw("TxManager receipt status is failed", "tx", receipt.TxHash,
 				"batch", batchInfo.BatchNum, "block", receipt.BlockNumber.Int64(),
 				"err", err)
+			batchInfo.EthTxErr = err
+			if batchInfo.BatchNum <= t.lastSuccessBatch {
+				t.lastSuccessBatch = batchInfo.BatchNum - 1
+			}
+			t.cfg.debugBatchStore(batchInfo)
 			return nil, tracerr.Wrap(fmt.Errorf(
 				"ethereum transaction receipt status is failed: %w", err))
 		} else if receipt.Status == types.ReceiptStatusSuccessful {
@@ -239,6 +328,17 @@ func (t *TxManager) handleReceipt(ctx context.Context, batchInfo *BatchInfo) (*i
 			batchInfo.Debug.MineBlockNum = receipt.BlockNumber.Int64()
 			batchInfo.Debug.StartToMineBlocksDelay = batchInfo.Debug.MineBlockNum -
 				batchInfo.Debug.StartBlockNum
+			if batchInfo.Debug.StartToMineDelay == 0 {
+				if block, err := t.ethClient.EthBlockByNumber(ctx,
+					receipt.BlockNumber.Int64()); err != nil {
+					log.Warnw("TxManager: ethClient.EthBlockByNumber", "err", err)
+				} else {
+					batchInfo.Debug.SendToMineDelay = block.Timestamp.Sub(
+						batchInfo.Debug.SendTimestamp).Seconds()
+					batchInfo.Debug.StartToMineDelay = block.Timestamp.Sub(
+						batchInfo.Debug.StartTimestamp).Seconds()
+				}
+			}
 			t.cfg.debugBatchStore(batchInfo)
 			if batchInfo.BatchNum > t.lastSuccessBatch {
 				t.lastSuccessBatch = batchInfo.BatchNum
@@ -250,10 +350,73 @@ func (t *TxManager) handleReceipt(ctx context.Context, batchInfo *BatchInfo) (*i
 	return nil, nil
 }
 
+// TODO:
+// - After sending a message: CancelPipeline, stop all consecutive pending Batches (transactions)
+
+// Queue of BatchInfos
+type Queue struct {
+	list []*BatchInfo
+	// nonceByBatchNum map[common.BatchNum]uint64
+	next int
+}
+
+// NewQueue returns a new queue
+func NewQueue() Queue {
+	return Queue{
+		list: make([]*BatchInfo, 0),
+		// nonceByBatchNum: make(map[common.BatchNum]uint64),
+		next: 0,
+	}
+}
+
+// Len is the length of the queue
+func (q *Queue) Len() int {
+	return len(q.list)
+}
+
+// At returns the BatchInfo at position (or nil if position is out of bounds)
+func (q *Queue) At(position int) *BatchInfo {
+	if position >= len(q.list) {
+		return nil
+	}
+	return q.list[position]
+}
+
+// Next returns the next BatchInfo (or nil if queue is empty)
+func (q *Queue) Next() (int, *BatchInfo) {
+	if len(q.list) == 0 {
+		return 0, nil
+	}
+	defer func() { q.next = (q.next + 1) % len(q.list) }()
+	return q.next, q.list[q.next]
+}
+
+// Remove removes the BatchInfo at position
+func (q *Queue) Remove(position int) {
+	// batchInfo := q.list[position]
+	// delete(q.nonceByBatchNum, batchInfo.BatchNum)
+	q.list = append(q.list[:position], q.list[position+1:]...)
+	if len(q.list) == 0 {
+		q.next = 0
+	} else {
+		q.next = position % len(q.list)
+	}
+}
+
+// Push adds a new BatchInfo
+func (q *Queue) Push(batchInfo *BatchInfo) {
+	q.list = append(q.list, batchInfo)
+	// q.nonceByBatchNum[batchInfo.BatchNum] = batchInfo.EthTx.Nonce()
+}
+
+// func (q *Queue) NonceByBatchNum(batchNum common.BatchNum) (uint64, bool) {
+// 	nonce, ok := q.nonceByBatchNum[batchNum]
+// 	return nonce, ok
+// }
+
 // Run the TxManager
 func (t *TxManager) Run(ctx context.Context) {
-	next := 0
+	waitCh := time.After(longWaitDuration)
-	waitDuration := longWaitDuration
 
 	var statsVars statsVars
 	select {
@@ -263,7 +426,7 @@ func (t *TxManager) Run(ctx context.Context) {
 	t.stats = statsVars.Stats
 	t.syncSCVars(statsVars.Vars)
 	log.Infow("TxManager: received initial statsVars",
-		"block", t.stats.Eth.LastBlock.Num, "batch", t.stats.Eth.LastBatch)
+		"block", t.stats.Eth.LastBlock.Num, "batch", t.stats.Eth.LastBatchNum)
 
 	for {
 		select {
@@ -273,8 +436,27 @@ func (t *TxManager) Run(ctx context.Context) {
 		case statsVars := <-t.statsVarsCh:
 			t.stats = statsVars.Stats
 			t.syncSCVars(statsVars.Vars)
+		case pipelineNum := <-t.discardPipelineCh:
+			t.minPipelineNum = pipelineNum + 1
+			if err := t.removeBadBatchInfos(ctx); ctx.Err() != nil {
+				continue
+			} else if err != nil {
+				log.Errorw("TxManager: removeBadBatchInfos", "err", err)
+				continue
+			}
 		case batchInfo := <-t.batchCh:
-			if err := t.sendRollupForgeBatch(ctx, batchInfo); ctx.Err() != nil {
+			if batchInfo.PipelineNum < t.minPipelineNum {
+				log.Warnw("TxManager: batchInfo received pipelineNum < minPipelineNum",
+					"num", batchInfo.PipelineNum, "minNum", t.minPipelineNum)
+			}
+			if err := t.shouldSendRollupForgeBatch(batchInfo); err != nil {
+				log.Warnw("TxManager: shouldSend", "err", err,
+					"batch", batchInfo.BatchNum)
+				t.coord.SendMsg(ctx, MsgStopPipeline{
+					Reason: fmt.Sprintf("forgeBatch shouldSend: %v", err)})
+				continue
+			}
+			if err := t.sendRollupForgeBatch(ctx, batchInfo, false); ctx.Err() != nil {
 				continue
 			} else if err != nil {
 				// If we reach here it's because our ethNode has
@@ -282,19 +464,20 @@ func (t *TxManager) Run(ctx context.Context) {
 				// ethereum.  This could be due to the ethNode
 				// failure, or an invalid transaction (that
 				// can't be mined)
-				t.coord.SendMsg(ctx, MsgStopPipeline{Reason: fmt.Sprintf("forgeBatch send: %v", err)})
+				log.Warnw("TxManager: forgeBatch send failed", "err", err,
+					"batch", batchInfo.BatchNum)
+				t.coord.SendMsg(ctx, MsgStopPipeline{
+					Reason: fmt.Sprintf("forgeBatch send: %v", err)})
 				continue
 			}
-			t.queue = append(t.queue, batchInfo)
+			t.queue.Push(batchInfo)
-			waitDuration = t.cfg.TxManagerCheckInterval
+			waitCh = time.After(t.cfg.TxManagerCheckInterval)
-		case <-time.After(waitDuration):
+		case <-waitCh:
-			if len(t.queue) == 0 {
+			queuePosition, batchInfo := t.queue.Next()
-				waitDuration = longWaitDuration
+			if batchInfo == nil {
+				waitCh = time.After(longWaitDuration)
 				continue
 			}
-			current := next
-			next = (current + 1) % len(t.queue)
-			batchInfo := t.queue[current]
 			if err := t.checkEthTransactionReceipt(ctx, batchInfo); ctx.Err() != nil {
 				continue
 			} else if err != nil { //nolint:staticcheck
@@ -304,7 +487,8 @@ func (t *TxManager) Run(ctx context.Context) {
 				// if it was not mined, mined and succesfull or
 				// mined and failed.  This could be due to the
 				// ethNode failure.
-				t.coord.SendMsg(ctx, MsgStopPipeline{Reason: fmt.Sprintf("forgeBatch receipt: %v", err)})
+				t.coord.SendMsg(ctx, MsgStopPipeline{
+					Reason: fmt.Sprintf("forgeBatch receipt: %v", err)})
 			}
 
 			confirm, err := t.handleReceipt(ctx, batchInfo)
@@ -312,32 +496,106 @@ func (t *TxManager) Run(ctx context.Context) {
 				continue
 			} else if err != nil { //nolint:staticcheck
 				// Transaction was rejected
-				t.queue = append(t.queue[:current], t.queue[current+1:]...)
+				if err := t.removeBadBatchInfos(ctx); ctx.Err() != nil {
-				if len(t.queue) == 0 {
+					continue
-					next = 0
+				} else if err != nil {
-				} else {
+					log.Errorw("TxManager: removeBadBatchInfos", "err", err)
-					next = current % len(t.queue)
+					continue
 				}
-				t.coord.SendMsg(ctx, MsgStopPipeline{Reason: fmt.Sprintf("forgeBatch reject: %v", err)})
+				t.coord.SendMsg(ctx, MsgStopPipeline{
+					Reason: fmt.Sprintf("forgeBatch reject: %v", err)})
+				continue
 			}
-			if confirm != nil && *confirm >= t.cfg.ConfirmBlocks {
+			now := time.Now()
-				log.Debugw("TxManager tx for RollupForgeBatch confirmed",
+			if !t.cfg.EthNoReuseNonce && confirm == nil &&
-					"batch", batchInfo.BatchNum)
+				now.Sub(batchInfo.SendTimestamp) > t.cfg.EthTxResendTimeout {
-				t.queue = append(t.queue[:current], t.queue[current+1:]...)
+				log.Infow("TxManager: forgeBatch tx not been mined timeout, resending",
-				if len(t.queue) == 0 {
+					"tx", batchInfo.EthTx.Hash(), "batch", batchInfo.BatchNum)
-					next = 0
+				if err := t.sendRollupForgeBatch(ctx, batchInfo, true); ctx.Err() != nil {
-				} else {
+					continue
-					next = current % len(t.queue)
+				} else if err != nil {
+					// If we reach here it's because our ethNode has
+					// been unable to send the transaction to
+					// ethereum.  This could be due to the ethNode
+					// failure, or an invalid transaction (that
+					// can't be mined)
+					log.Warnw("TxManager: forgeBatch resend failed", "err", err,
+						"batch", batchInfo.BatchNum)
+					t.coord.SendMsg(ctx, MsgStopPipeline{
+						Reason: fmt.Sprintf("forgeBatch resend: %v", err)})
+					continue
 				}
 			}
+
+			if confirm != nil && *confirm >= t.cfg.ConfirmBlocks {
+				log.Debugw("TxManager: forgeBatch tx confirmed",
+					"tx", batchInfo.EthTx.Hash(), "batch", batchInfo.BatchNum)
+				t.queue.Remove(queuePosition)
+			}
 		}
 	}
 }
 
-// nolint reason: this function will be used in the future
+func (t *TxManager) removeBadBatchInfos(ctx context.Context) error {
-//nolint:unused
+	next := 0
-func (t *TxManager) canForge(stats *synchronizer.Stats, blockNum int64) bool {
+	for {
+		batchInfo := t.queue.At(next)
+		if batchInfo == nil {
+			break
+		}
+		if err := t.checkEthTransactionReceipt(ctx, batchInfo); ctx.Err() != nil {
+			return nil
+		} else if err != nil {
+			// Our ethNode is giving an error different
+			// than "not found" when getting the receipt
+			// for the transaction, so we can't figure out
+			// if it was not mined, mined and succesfull or
+			// mined and failed.  This could be due to the
+			// ethNode failure.
+			next++
+			continue
+		}
+		confirm, err := t.handleReceipt(ctx, batchInfo)
+		if ctx.Err() != nil {
+			return nil
+		} else if err != nil {
+			// Transaction was rejected
+			if t.minPipelineNum <= batchInfo.PipelineNum {
+				t.minPipelineNum = batchInfo.PipelineNum + 1
+			}
+			t.queue.Remove(next)
+			continue
+		}
+		// If tx is pending but is from a cancelled pipeline, remove it
+		// from the queue
+		if confirm == nil {
+			if batchInfo.PipelineNum < t.minPipelineNum {
+				t.queue.Remove(next)
+				continue
+			}
+		}
+		next++
+	}
+	accNonce, err := t.ethClient.EthNonceAt(ctx, t.account.Address, nil)
+	if err != nil {
+		return err
+	}
+	if !t.cfg.EthNoReuseNonce {
+		t.accNextNonce = accNonce
+	}
+	return nil
+}
+
+func (t *TxManager) canForgeAt(blockNum int64) bool {
 	return canForge(&t.consts.Auction, &t.vars.Auction,
-		&stats.Sync.Auction.CurrentSlot, &stats.Sync.Auction.NextSlot,
+		&t.stats.Sync.Auction.CurrentSlot, &t.stats.Sync.Auction.NextSlot,
 		t.cfg.ForgerAddress, blockNum)
 }
+
+func (t *TxManager) mustL1L2Batch(blockNum int64) bool {
+	lastL1BatchBlockNum := t.lastSentL1BatchBlockNum
+	if t.stats.Sync.LastL1BatchBlock > lastL1BatchBlockNum {
+		lastL1BatchBlockNum = t.stats.Sync.LastL1BatchBlock
+	}
+	return blockNum-lastL1BatchBlockNum >= t.vars.Rollup.ForgeL1L2BatchTimeout-1
+}

coordinator/txmanager_test.go

@@ -0,0 +1,15 @@
+package coordinator
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAddPerc(t *testing.T) {
+	assert.Equal(t, "110", addPerc(big.NewInt(100), 10).String())
+	assert.Equal(t, "101", addPerc(big.NewInt(100), 1).String())
+	assert.Equal(t, "12", addPerc(big.NewInt(10), 20).String())
+	assert.Equal(t, "1500", addPerc(big.NewInt(1000), 50).String())
+}

db/historydb/historydb.go

@@ -164,6 +164,19 @@ func (hdb *HistoryDB) addBatches(d meddler.DB, batches []common.Batch) error {
 	return nil
 }
 
+// GetBatch returns the batch with the given batchNum
+func (hdb *HistoryDB) GetBatch(batchNum common.BatchNum) (*common.Batch, error) {
+	var batch common.Batch
+	err := meddler.QueryRow(
+		hdb.db, &batch, `SELECT batch.batch_num, batch.eth_block_num, batch.forger_addr,
+		batch.fees_collected, batch.fee_idxs_coordinator, batch.state_root,
+		batch.num_accounts, batch.last_idx, batch.exit_root, batch.forge_l1_txs_num,
+		batch.slot_num, batch.total_fees_usd FROM batch WHERE batch_num = $1;`,
+		batchNum,
+	)
+	return &batch, err
+}
+
 // GetAllBatches retrieve all batches from the DB
 func (hdb *HistoryDB) GetAllBatches() ([]common.Batch, error) {
 	var batches []*common.Batch
@@ -208,6 +221,18 @@ func (hdb *HistoryDB) GetLastBatchNum() (common.BatchNum, error) {
 	return batchNum, tracerr.Wrap(row.Scan(&batchNum))
 }
 
+// GetLastBatch returns the last forged batch
+func (hdb *HistoryDB) GetLastBatch() (*common.Batch, error) {
+	var batch common.Batch
+	err := meddler.QueryRow(
+		hdb.db, &batch, `SELECT batch.batch_num, batch.eth_block_num, batch.forger_addr,
+		batch.fees_collected, batch.fee_idxs_coordinator, batch.state_root,
+		batch.num_accounts, batch.last_idx, batch.exit_root, batch.forge_l1_txs_num,
+		batch.slot_num, batch.total_fees_usd FROM batch ORDER BY batch_num DESC LIMIT 1;`,
+	)
+	return &batch, err
+}
+
 // GetLastL1BatchBlockNum returns the blockNum of the latest forged l1Batch
 func (hdb *HistoryDB) GetLastL1BatchBlockNum() (int64, error) {
 	row := hdb.db.QueryRow(`SELECT eth_block_num FROM batch

db/historydb/historydb_test.go

@@ -203,6 +203,10 @@ func TestBatches(t *testing.T) {
 	fetchedLastBatchNum, err := historyDB.GetLastBatchNum()
 	assert.NoError(t, err)
 	assert.Equal(t, batches[len(batches)-1].BatchNum, fetchedLastBatchNum)
+	// Test GetLastBatch
+	fetchedLastBatch, err := historyDB.GetLastBatch()
+	assert.NoError(t, err)
+	assert.Equal(t, &batches[len(batches)-1], fetchedLastBatch)
 	// Test GetLastL1TxsNum
 	fetchedLastL1TxsNum, err := historyDB.GetLastL1TxsNum()
 	assert.NoError(t, err)
@@ -211,6 +215,12 @@ func TestBatches(t *testing.T) {
 	fetchedLastL1BatchBlockNum, err := historyDB.GetLastL1BatchBlockNum()
 	assert.NoError(t, err)
 	assert.Equal(t, lastL1BatchBlockNum, fetchedLastL1BatchBlockNum)
+	// Test GetBatch
+	fetchedBatch, err := historyDB.GetBatch(1)
+	require.NoError(t, err)
+	assert.Equal(t, &batches[0], fetchedBatch)
+	_, err = historyDB.GetBatch(common.BatchNum(len(batches) + 1))
+	assert.Equal(t, sql.ErrNoRows, tracerr.Unwrap(err))
 }
 
 func TestBids(t *testing.T) {

db/kvdb/kvdb.go

@@ -425,12 +425,13 @@ func (k *KVDB) MakeCheckpoint() error {
 	}
 
 	// if checkpoint BatchNum already exist in disk, delete it
-	if _, err := os.Stat(checkpointPath); !os.IsNotExist(err) {
+	if _, err := os.Stat(checkpointPath); os.IsNotExist(err) {
+	} else if err != nil {
+		return tracerr.Wrap(err)
+	} else {
 		if err := os.RemoveAll(checkpointPath); err != nil {
 			return tracerr.Wrap(err)
 		}
-	} else if err != nil && !os.IsNotExist(err) {
-		return tracerr.Wrap(err)
 	}
 
 	// execute Checkpoint
@@ -451,12 +452,25 @@ func (k *KVDB) MakeCheckpoint() error {
 	return nil
 }
 
+// CheckpointExists returns true if the checkpoint exists
+func (k *KVDB) CheckpointExists(batchNum common.BatchNum) (bool, error) {
+	source := path.Join(k.cfg.Path, fmt.Sprintf("%s%d", PathBatchNum, batchNum))
+	if _, err := os.Stat(source); os.IsNotExist(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 // DeleteCheckpoint removes if exist the checkpoint of the given batchNum
 func (k *KVDB) DeleteCheckpoint(batchNum common.BatchNum) error {
 	checkpointPath := path.Join(k.cfg.Path, fmt.Sprintf("%s%d", PathBatchNum, batchNum))
 
 	if _, err := os.Stat(checkpointPath); os.IsNotExist(err) {
 		return tracerr.Wrap(fmt.Errorf("Checkpoint with batchNum %d does not exist in DB", batchNum))
+	} else if err != nil {
+		return tracerr.Wrap(err)
 	}
 
 	return os.RemoveAll(checkpointPath)
@@ -520,6 +534,8 @@ func (k *KVDB) MakeCheckpointFromTo(fromBatchNum common.BatchNum, dest string) e
 	if _, err := os.Stat(source); os.IsNotExist(err) {
 		// if kvdb does not have checkpoint at batchNum, return err
 		return tracerr.Wrap(fmt.Errorf("Checkpoint \"%v\" does not exist", source))
+	} else if err != nil {
+		return tracerr.Wrap(err)
 	}
 	// By locking we allow calling MakeCheckpointFromTo from multiple
 	// places at the same time for the same stateDB.  This allows the
@@ -533,12 +549,13 @@ func (k *KVDB) MakeCheckpointFromTo(fromBatchNum common.BatchNum, dest string) e
 
 func pebbleMakeCheckpoint(source, dest string) error {
 	// Remove dest folder (if it exists) before doing the checkpoint
-	if _, err := os.Stat(dest); !os.IsNotExist(err) {
+	if _, err := os.Stat(dest); os.IsNotExist(err) {
+	} else if err != nil {
+		return tracerr.Wrap(err)
+	} else {
 		if err := os.RemoveAll(dest); err != nil {
 			return tracerr.Wrap(err)
 		}
-	} else if err != nil && !os.IsNotExist(err) {
-		return tracerr.Wrap(err)
 	}
 
 	sto, err := pebble.NewPebbleStorage(source, false)

db/statedb/statedb.go

@@ -498,11 +498,17 @@ func NewLocalStateDB(cfg Config, synchronizerDB *StateDB) (*LocalStateDB, error)
 	}, nil
 }
 
+// CheckpointExists returns true if the checkpoint exists
+func (l *LocalStateDB) CheckpointExists(batchNum common.BatchNum) (bool, error) {
+	return l.db.CheckpointExists(batchNum)
+}
+
 // Reset performs a reset in the LocaStateDB. If fromSynchronizer is true, it
 // gets the state from LocalStateDB.synchronizerStateDB for the given batchNum.
 // If fromSynchronizer is false, get the state from LocalStateDB checkpoints.
 func (l *LocalStateDB) Reset(batchNum common.BatchNum, fromSynchronizer bool) error {
 	if fromSynchronizer {
+		log.Debugw("Making StateDB ResetFromSynchronizer", "batch", batchNum, "type", l.cfg.Type)
 		if err := l.db.ResetFromSynchronizer(batchNum, l.synchronizerStateDB.db); err != nil {
 			return tracerr.Wrap(err)
 		}

db/utils.go

@@ -13,9 +13,9 @@ import (
 	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/tracerr"
 	"github.com/jmoiron/sqlx"
-	"github.com/marusama/semaphore/v2"
 	migrate "github.com/rubenv/sql-migrate"
 	"github.com/russross/meddler"
+	"golang.org/x/sync/semaphore"
 )
 
 var migrations *migrate.PackrMigrationSource
@@ -89,14 +89,14 @@ func InitSQLDB(port int, host, user, password, name string) (*sqlx.DB, error) {
 
 // APIConnectionController is used to limit the SQL open connections used by the API
 type APIConnectionController struct {
-	smphr   semaphore.Semaphore
+	smphr   *semaphore.Weighted
 	timeout time.Duration
 }
 
 // NewAPICnnectionController initialize APIConnectionController
 func NewAPICnnectionController(maxConnections int, timeout time.Duration) *APIConnectionController {
 	return &APIConnectionController{
-		smphr:   semaphore.New(maxConnections),
+		smphr:   semaphore.NewWeighted(int64(maxConnections)),
 		timeout: timeout,
 	}
 }

eth/rollup.go

@@ -316,7 +316,7 @@ func NewRollupClient(client *EthereumClient, address ethCommon.Address, tokenHEZ
 	}
 	consts, err := c.RollupConstants()
 	if err != nil {
-		return nil, tracerr.Wrap(err)
+		return nil, tracerr.Wrap(fmt.Errorf("RollupConstants at %v: %w", address, err))
 	}
 	c.consts = consts
 	return c, nil

go.mod

@@ -17,7 +17,6 @@ require (
 	github.com/jmoiron/sqlx v1.2.1-0.20200615141059-0794cb1f47ee
 	github.com/joho/godotenv v1.3.0
 	github.com/lib/pq v1.8.0
-	github.com/marusama/semaphore/v2 v2.4.1
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/miguelmota/go-ethereum-hdwallet v0.0.0-20200123000308-a60dcd172b4c
 	github.com/mitchellh/copystructure v1.0.0
@@ -29,5 +28,6 @@ require (
 	go.uber.org/zap v1.16.0
 	golang.org/x/crypto v0.0.0-20201217014255-9d1352758620
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	gopkg.in/go-playground/validator.v9 v9.29.1
 )

go.sum

@@ -24,6 +24,8 @@ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/uf
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
+github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
@@ -84,6 +86,8 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
+github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
+github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM=
@@ -169,6 +173,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c=
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
@@ -415,9 +421,6 @@ github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY
 github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
 github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/marusama/semaphore v0.0.0-20190110074507-6952cef993b2 h1:sq+a5mb8zHbmHhrIH06oqIMGsanjpbxNgxEgZVfgpvQ=
-github.com/marusama/semaphore/v2 v2.4.1 h1:Y29DhhFMvreVgoqF9EtaSJAF9t2E7Sk7i5VW81sqB8I=
-github.com/marusama/semaphore/v2 v2.4.1/go.mod h1:z9nMiNUekt/LTpTUQdpp+4sJeYqUGpwMHfW0Z8V8fnQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
@@ -599,6 +602,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
+github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -617,6 +622,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg=
 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM=

node/node.go

@@ -2,6 +2,7 @@ package node
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"sync"
@@ -102,8 +103,8 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) {
 	var keyStore *ethKeystore.KeyStore
 	if mode == ModeCoordinator {
 		ethCfg = eth.EthereumConfig{
-			CallGasLimit: cfg.Coordinator.EthClient.CallGasLimit,
+			CallGasLimit: 0, // cfg.Coordinator.EthClient.CallGasLimit,
-			GasPriceDiv:  cfg.Coordinator.EthClient.GasPriceDiv,
+			GasPriceDiv:  0, // cfg.Coordinator.EthClient.GasPriceDiv,
 		}
 
 		scryptN := ethKeystore.StandardScryptN
@@ -298,9 +299,15 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) {
 				ConfirmBlocks:          cfg.Coordinator.ConfirmBlocks,
 				L1BatchTimeoutPerc:     cfg.Coordinator.L1BatchTimeoutPerc,
 				ForgeRetryInterval:     cfg.Coordinator.ForgeRetryInterval.Duration,
+				ForgeDelay:             cfg.Coordinator.ForgeDelay.Duration,
+				ForgeNoTxsDelay:        cfg.Coordinator.ForgeNoTxsDelay.Duration,
 				SyncRetryInterval:      cfg.Coordinator.SyncRetryInterval.Duration,
 				EthClientAttempts:      cfg.Coordinator.EthClient.Attempts,
 				EthClientAttemptsDelay: cfg.Coordinator.EthClient.AttemptsDelay.Duration,
+				EthNoReuseNonce:        cfg.Coordinator.EthClient.NoReuseNonce,
+				EthTxResendTimeout:     cfg.Coordinator.EthClient.TxResendTimeout.Duration,
+				MaxGasPrice:            cfg.Coordinator.EthClient.MaxGasPrice,
+				GasPriceIncPerc:        cfg.Coordinator.EthClient.GasPriceIncPerc,
 				TxManagerCheckInterval: cfg.Coordinator.EthClient.CheckLoopInterval.Duration,
 				DebugBatchPath:         cfg.Coordinator.Debug.BatchPath,
 				Purger: coordinator.PurgerCfg{
@@ -487,11 +494,15 @@ func (n *Node) handleNewBlock(ctx context.Context, stats *synchronizer.Stats, va
 		if stats.Synced() {
 			if err := n.nodeAPI.api.UpdateNetworkInfo(
 				stats.Eth.LastBlock, stats.Sync.LastBlock,
-				common.BatchNum(stats.Eth.LastBatch),
+				common.BatchNum(stats.Eth.LastBatchNum),
 				stats.Sync.Auction.CurrentSlot.SlotNum,
 			); err != nil {
 				log.Errorw("API.UpdateNetworkInfo", "err", err)
 			}
+		} else {
+			n.nodeAPI.api.UpdateNetworkInfoBlock(
+				stats.Eth.LastBlock, stats.Sync.LastBlock,
+			)
 		}
 	}
 }
@@ -573,7 +584,13 @@ func (n *Node) StartSynchronizer() {
 					if n.ctx.Err() != nil {
 						continue
 					}
-					log.Errorw("Synchronizer.Sync", "err", err)
+					if errors.Is(err, eth.ErrBlockHashMismatchEvent) {
+						log.Warnw("Synchronizer.Sync", "err", err)
+					} else if errors.Is(err, synchronizer.ErrUnknownBlock) {
+						log.Warnw("Synchronizer.Sync", "err", err)
+					} else {
+						log.Errorw("Synchronizer.Sync", "err", err)
+					}
 				}
 			}
 		}

synchronizer/synchronizer.go

@@ -18,6 +18,19 @@ import (
 	"github.com/hermeznetwork/tracerr"
 )
 
+const (
+	// errStrUnknownBlock is the string returned by geth when querying an
+	// unknown block
+	errStrUnknownBlock = "unknown block"
+)
+
+var (
+	// ErrUnknownBlock is the error returned by the Synchronizer when a
+	// block is queried by hash but the ethereum node doesn't find it due
+	// to it being discarded from a reorg.
+	ErrUnknownBlock = fmt.Errorf("unknown block")
+)
+
 // Stats of the syncrhonizer
 type Stats struct {
 	Eth struct {
@@ -25,12 +38,12 @@ type Stats struct {
 		Updated       time.Time
 		FirstBlockNum int64
 		LastBlock     common.Block
-		LastBatch     int64
+		LastBatchNum  int64
 	}
 	Sync struct {
 		Updated   time.Time
 		LastBlock common.Block
-		LastBatch int64
+		LastBatch common.Batch
 		// LastL1BatchBlock is the last ethereum block in which an
 		// l1Batch was forged
 		LastL1BatchBlock  int64
@@ -77,13 +90,13 @@ func (s *StatsHolder) UpdateCurrentNextSlot(current *common.Slot, next *common.S
 }
 
 // UpdateSync updates the synchronizer stats
-func (s *StatsHolder) UpdateSync(lastBlock *common.Block, lastBatch *common.BatchNum,
+func (s *StatsHolder) UpdateSync(lastBlock *common.Block, lastBatch *common.Batch,
 	lastL1BatchBlock *int64, lastForgeL1TxsNum *int64) {
 	now := time.Now()
 	s.rw.Lock()
 	s.Sync.LastBlock = *lastBlock
 	if lastBatch != nil {
-		s.Sync.LastBatch = int64(*lastBatch)
+		s.Sync.LastBatch = *lastBatch
 	}
 	if lastL1BatchBlock != nil {
 		s.Sync.LastL1BatchBlock = *lastL1BatchBlock
@@ -105,16 +118,16 @@ func (s *StatsHolder) UpdateEth(ethClient eth.ClientInterface) error {
 
 	lastBlock, err := ethClient.EthBlockByNumber(context.TODO(), -1)
 	if err != nil {
-		return tracerr.Wrap(err)
+		return tracerr.Wrap(fmt.Errorf("EthBlockByNumber: %w", err))
 	}
-	lastBatch, err := ethClient.RollupLastForgedBatch()
+	lastBatchNum, err := ethClient.RollupLastForgedBatch()
 	if err != nil {
-		return tracerr.Wrap(err)
+		return tracerr.Wrap(fmt.Errorf("RollupLastForgedBatch: %w", err))
 	}
 	s.rw.Lock()
 	s.Eth.Updated = now
 	s.Eth.LastBlock = *lastBlock
-	s.Eth.LastBatch = lastBatch
+	s.Eth.LastBatchNum = lastBatchNum
 	s.rw.Unlock()
 	return nil
 }
@@ -139,6 +152,10 @@ func (s *StatsHolder) CopyStats() *Stats {
 		sCopy.Sync.Auction.NextSlot.DefaultSlotBid =
 			common.CopyBigInt(s.Sync.Auction.NextSlot.DefaultSlotBid)
 	}
+	if s.Sync.LastBatch.StateRoot != nil {
+		sCopy.Sync.LastBatch.StateRoot =
+			common.CopyBigInt(s.Sync.LastBatch.StateRoot)
+	}
 	s.rw.RUnlock()
 	return &sCopy
 }
@@ -152,9 +169,9 @@ func (s *StatsHolder) blocksPerc() float64 {
 		float64(s.Eth.LastBlock.Num-(s.Eth.FirstBlockNum-1))
 }
 
-func (s *StatsHolder) batchesPerc(batchNum int64) float64 {
+func (s *StatsHolder) batchesPerc(batchNum common.BatchNum) float64 {
 	return float64(batchNum) * 100.0 /
-		float64(s.Eth.LastBatch)
+		float64(s.Eth.LastBatchNum)
 }
 
 // StartBlockNums sets the first block used to start tracking the smart
@@ -329,23 +346,25 @@ func (s *Synchronizer) setSlotCoordinator(slot *common.Slot) error {
 	return nil
 }
 
-// firstBatchBlockNum is the blockNum of first batch in that block, if any
+// updateCurrentSlot updates the slot with information of the current slot.
-func (s *Synchronizer) getCurrentSlot(reset bool, firstBatchBlockNum *int64) (*common.Slot, error) {
+// The information abouth which coordinator is allowed to forge is only updated
-	slot := common.Slot{
+// when we are Synced.
-		SlotNum:          s.stats.Sync.Auction.CurrentSlot.SlotNum,
+// hasBatch is true when the last synced block contained at least one batch.
-		ForgerCommitment: s.stats.Sync.Auction.CurrentSlot.ForgerCommitment,
+func (s *Synchronizer) updateCurrentSlot(slot *common.Slot, reset bool, hasBatch bool) error {
-	}
 	// We want the next block because the current one is already mined
 	blockNum := s.stats.Sync.LastBlock.Num + 1
 	slotNum := s.consts.Auction.SlotNum(blockNum)
+	firstBatchBlockNum := s.stats.Sync.LastBlock.Num
 	if reset {
+		// Using this query only to know if there
 		dbFirstBatchBlockNum, err := s.historyDB.GetFirstBatchBlockNumBySlot(slotNum)
 		if err != nil && tracerr.Unwrap(err) != sql.ErrNoRows {
-			return nil, tracerr.Wrap(fmt.Errorf("historyDB.GetFirstBatchBySlot: %w", err))
+			return tracerr.Wrap(fmt.Errorf("historyDB.GetFirstBatchBySlot: %w", err))
 		} else if tracerr.Unwrap(err) == sql.ErrNoRows {
-			firstBatchBlockNum = nil
+			hasBatch = false
 		} else {
-			firstBatchBlockNum = &dbFirstBatchBlockNum
+			hasBatch = true
+			firstBatchBlockNum = dbFirstBatchBlockNum
 		}
 		slot.ForgerCommitment = false
 	} else if slotNum > slot.SlotNum {
@@ -356,11 +375,11 @@ func (s *Synchronizer) getCurrentSlot(reset bool, firstBatchBlockNum *int64) (*c
 	slot.StartBlock, slot.EndBlock = s.consts.Auction.SlotBlocks(slot.SlotNum)
 	// If Synced, update the current coordinator
 	if s.stats.Synced() && blockNum >= s.consts.Auction.GenesisBlockNum {
-		if err := s.setSlotCoordinator(&slot); err != nil {
+		if err := s.setSlotCoordinator(slot); err != nil {
-			return nil, tracerr.Wrap(err)
+			return tracerr.Wrap(err)
 		}
-		if firstBatchBlockNum != nil &&
+		if hasBatch &&
-			s.consts.Auction.RelativeBlock(*firstBatchBlockNum) <
+			s.consts.Auction.RelativeBlock(firstBatchBlockNum) <
 				int64(s.vars.Auction.SlotDeadline) {
 			slot.ForgerCommitment = true
 		}
@@ -369,57 +388,61 @@ func (s *Synchronizer) getCurrentSlot(reset bool, firstBatchBlockNum *int64) (*c
 		// BEGIN SANITY CHECK
 		canForge, err := s.ethClient.AuctionCanForge(slot.Forger, blockNum)
 		if err != nil {
-			return nil, tracerr.Wrap(err)
+			return tracerr.Wrap(fmt.Errorf("AuctionCanForge: %w", err))
 		}
 		if !canForge {
-			return nil, tracerr.Wrap(fmt.Errorf("Synchronized value of forger address for closed slot "+
+			return tracerr.Wrap(fmt.Errorf("Synchronized value of forger address for closed slot "+
 				"differs from smart contract: %+v", slot))
 		}
 		// END SANITY CHECK
 	}
-	return &slot, nil
+	return nil
 }
 
-func (s *Synchronizer) getNextSlot() (*common.Slot, error) {
+// updateNextSlot updates the slot with information of the next slot.
+// The information abouth which coordinator is allowed to forge is only updated
+// when we are Synced.
+func (s *Synchronizer) updateNextSlot(slot *common.Slot) error {
 	// We want the next block because the current one is already mined
 	blockNum := s.stats.Sync.LastBlock.Num + 1
 	slotNum := s.consts.Auction.SlotNum(blockNum) + 1
-	slot := common.Slot{
+	slot.SlotNum = slotNum
-		SlotNum:          slotNum,
+	slot.ForgerCommitment = false
-		ForgerCommitment: false,
-	}
 	slot.StartBlock, slot.EndBlock = s.consts.Auction.SlotBlocks(slot.SlotNum)
 	// If Synced, update the current coordinator
 	if s.stats.Synced() && blockNum >= s.consts.Auction.GenesisBlockNum {
-		if err := s.setSlotCoordinator(&slot); err != nil {
+		if err := s.setSlotCoordinator(slot); err != nil {
-			return nil, tracerr.Wrap(err)
+			return tracerr.Wrap(err)
 		}
 
 		// TODO: Remove this SANITY CHECK once this code is tested enough
 		// BEGIN SANITY CHECK
 		canForge, err := s.ethClient.AuctionCanForge(slot.Forger, slot.StartBlock)
 		if err != nil {
-			return nil, tracerr.Wrap(err)
+			return tracerr.Wrap(fmt.Errorf("AuctionCanForge: %w", err))
 		}
 		if !canForge {
-			return nil, tracerr.Wrap(fmt.Errorf("Synchronized value of forger address for closed slot "+
+			return tracerr.Wrap(fmt.Errorf("Synchronized value of forger address for closed slot "+
 				"differs from smart contract: %+v", slot))
 		}
 		// END SANITY CHECK
 	}
-	return &slot, nil
+	return nil
 }
 
-func (s *Synchronizer) updateCurrentNextSlotIfSync(reset bool, firstBatchBlockNum *int64) error {
+// updateCurrentNextSlotIfSync updates the current and next slot.  Information
-	current, err := s.getCurrentSlot(reset, firstBatchBlockNum)
+// about forger address that is allowed to forge is only updated if we are
-	if err != nil {
+// Synced.
+func (s *Synchronizer) updateCurrentNextSlotIfSync(reset bool, hasBatch bool) error {
+	current := s.stats.Sync.Auction.CurrentSlot
+	next := s.stats.Sync.Auction.NextSlot
+	if err := s.updateCurrentSlot(&current, reset, hasBatch); err != nil {
 		return tracerr.Wrap(err)
 	}
-	next, err := s.getNextSlot()
+	if err := s.updateNextSlot(&next); err != nil {
-	if err != nil {
 		return tracerr.Wrap(err)
 	}
-	s.stats.UpdateCurrentNextSlot(current, next)
+	s.stats.UpdateCurrentNextSlot(&current, &next)
 	return nil
 }
 
@@ -458,9 +481,9 @@ func (s *Synchronizer) init() error {
 		"ethLastBlock", s.stats.Eth.LastBlock,
 	)
 	log.Infow("Sync init batch",
-		"syncLastBatch", s.stats.Sync.LastBatch,
+		"syncLastBatch", s.stats.Sync.LastBatch.BatchNum,
-		"syncBatchesPerc", s.stats.batchesPerc(s.stats.Sync.LastBatch),
+		"syncBatchesPerc", s.stats.batchesPerc(s.stats.Sync.LastBatch.BatchNum),
-		"ethLastBatch", s.stats.Eth.LastBatch,
+		"ethLastBatch", s.stats.Eth.LastBatchNum,
 	)
 	return nil
 }
@@ -521,7 +544,7 @@ func (s *Synchronizer) Sync2(ctx context.Context,
 	if tracerr.Unwrap(err) == ethereum.NotFound {
 		return nil, nil, nil
 	} else if err != nil {
-		return nil, nil, tracerr.Wrap(err)
+		return nil, nil, tracerr.Wrap(fmt.Errorf("EthBlockByNumber: %w", err))
 	}
 	log.Debugf("ethBlock: num: %v, parent: %v, hash: %v",
 		ethBlock.Num, ethBlock.ParentHash.String(), ethBlock.Hash.String())
@@ -627,29 +650,29 @@ func (s *Synchronizer) Sync2(ctx context.Context,
 			}
 		}
 		s.stats.UpdateSync(ethBlock,
-			&rollupData.Batches[batchesLen-1].Batch.BatchNum,
+			&rollupData.Batches[batchesLen-1].Batch,
 			lastL1BatchBlock, lastForgeL1TxsNum)
 	}
-	var firstBatchBlockNum *int64
+	hasBatch := false
 	if len(rollupData.Batches) > 0 {
-		firstBatchBlockNum = &rollupData.Batches[0].Batch.EthBlockNum
+		hasBatch = true
 	}
-	if err := s.updateCurrentNextSlotIfSync(false, firstBatchBlockNum); err != nil {
+	if err := s.updateCurrentNextSlotIfSync(false, hasBatch); err != nil {
 		return nil, nil, tracerr.Wrap(err)
 	}
 
+	for _, batchData := range rollupData.Batches {
+		log.Debugw("Synced batch",
+			"syncLastBatch", batchData.Batch.BatchNum,
+			"syncBatchesPerc", s.stats.batchesPerc(batchData.Batch.BatchNum),
+			"ethLastBatch", s.stats.Eth.LastBatchNum,
+		)
+	}
 	log.Debugw("Synced block",
 		"syncLastBlockNum", s.stats.Sync.LastBlock.Num,
 		"syncBlocksPerc", s.stats.blocksPerc(),
 		"ethLastBlockNum", s.stats.Eth.LastBlock.Num,
 	)
-	for _, batchData := range rollupData.Batches {
-		log.Debugw("Synced batch",
-			"syncLastBatch", batchData.Batch.BatchNum,
-			"syncBatchesPerc", s.stats.batchesPerc(int64(batchData.Batch.BatchNum)),
-			"ethLastBatch", s.stats.Eth.LastBatch,
-		)
-	}
 
 	return blockData, nil, nil
 }
@@ -700,15 +723,15 @@ func getInitialVariables(ethClient eth.ClientInterface,
 	consts *SCConsts) (*SCVariables, *StartBlockNums, error) {
 	rollupInit, rollupInitBlock, err := ethClient.RollupEventInit()
 	if err != nil {
-		return nil, nil, tracerr.Wrap(err)
+		return nil, nil, tracerr.Wrap(fmt.Errorf("RollupEventInit: %w", err))
 	}
 	auctionInit, auctionInitBlock, err := ethClient.AuctionEventInit()
 	if err != nil {
-		return nil, nil, tracerr.Wrap(err)
+		return nil, nil, tracerr.Wrap(fmt.Errorf("AuctionEventInit: %w", err))
 	}
 	wDelayerInit, wDelayerInitBlock, err := ethClient.WDelayerEventInit()
 	if err != nil {
-		return nil, nil, tracerr.Wrap(err)
+		return nil, nil, tracerr.Wrap(fmt.Errorf("WDelayerEventInit: %w", err))
 	}
 	rollupVars := rollupInit.RollupVariables()
 	auctionVars := auctionInit.AuctionVariables(consts.Auction.InitialMinimalBidding)
@@ -753,15 +776,15 @@ func (s *Synchronizer) resetState(block *common.Block) error {
 		s.vars.WDelayer = *wDelayer
 	}
 
-	batchNum, err := s.historyDB.GetLastBatchNum()
+	batch, err := s.historyDB.GetLastBatch()
 	if err != nil && tracerr.Unwrap(err) != sql.ErrNoRows {
 		return tracerr.Wrap(fmt.Errorf("historyDB.GetLastBatchNum: %w", err))
 	}
 	if tracerr.Unwrap(err) == sql.ErrNoRows {
-		batchNum = 0
+		batch = &common.Batch{}
 	}
 
-	err = s.stateDB.Reset(batchNum)
+	err = s.stateDB.Reset(batch.BatchNum)
 	if err != nil {
 		return tracerr.Wrap(fmt.Errorf("stateDB.Reset: %w", err))
 	}
@@ -783,9 +806,9 @@ func (s *Synchronizer) resetState(block *common.Block) error {
 		lastForgeL1TxsNum = &n
 	}
 
-	s.stats.UpdateSync(block, &batchNum, &lastL1BatchBlockNum, lastForgeL1TxsNum)
+	s.stats.UpdateSync(block, batch, &lastL1BatchBlockNum, lastForgeL1TxsNum)
 
-	if err := s.updateCurrentNextSlotIfSync(true, nil); err != nil {
+	if err := s.updateCurrentNextSlotIfSync(true, false); err != nil {
 		return tracerr.Wrap(err)
 	}
 	return nil
@@ -801,8 +824,10 @@ func (s *Synchronizer) rollupSync(ethBlock *common.Block) (*common.RollupData, e
 	// Get rollup events in the block, and make sure the block hash matches
 	// the expected one.
 	rollupEvents, err := s.ethClient.RollupEventsByBlock(blockNum, &ethBlock.Hash)
-	if err != nil {
+	if err != nil && err.Error() == errStrUnknownBlock {
-		return nil, tracerr.Wrap(err)
+		return nil, tracerr.Wrap(ErrUnknownBlock)
+	} else if err != nil {
+		return nil, tracerr.Wrap(fmt.Errorf("RollupEventsByBlock: %w", err))
 	}
 	// No events in this block
 	if rollupEvents == nil {
@@ -919,9 +944,15 @@ func (s *Synchronizer) rollupSync(ethBlock *common.Block) (*common.RollupData, e
 			return nil, tracerr.Wrap(err)
 		}
 		if s.stateDB.CurrentBatch() != batchNum {
-			return nil, tracerr.Wrap(fmt.Errorf("stateDB.BatchNum (%v) != evtForgeBatch.BatchNum = (%v)",
+			return nil, tracerr.Wrap(fmt.Errorf("stateDB.BatchNum (%v) != "+
+				"evtForgeBatch.BatchNum = (%v)",
 				s.stateDB.CurrentBatch(), batchNum))
 		}
+		if s.stateDB.MT.Root().BigInt().Cmp(forgeBatchArgs.NewStRoot) != 0 {
+			return nil, tracerr.Wrap(fmt.Errorf("stateDB.MTRoot (%v) != "+
+				"forgeBatchArgs.NewStRoot (%v)",
+				s.stateDB.MT.Root().BigInt(), forgeBatchArgs.NewStRoot))
+		}
 
 		// Transform processed PoolL2 txs to L2 and store in BatchData
 		l2Txs, err := common.PoolL2TxsToL2Txs(poolL2Txs) // NOTE: This is a big uggly, find a better way
@@ -1105,8 +1136,10 @@ func (s *Synchronizer) auctionSync(ethBlock *common.Block) (*common.AuctionData,
 
 	// Get auction events in the block
 	auctionEvents, err := s.ethClient.AuctionEventsByBlock(blockNum, &ethBlock.Hash)
-	if err != nil {
+	if err != nil && err.Error() == errStrUnknownBlock {
-		return nil, tracerr.Wrap(err)
+		return nil, tracerr.Wrap(ErrUnknownBlock)
+	} else if err != nil {
+		return nil, tracerr.Wrap(fmt.Errorf("AuctionEventsByBlock: %w", err))
 	}
 	// No events in this block
 	if auctionEvents == nil {
@@ -1202,8 +1235,10 @@ func (s *Synchronizer) wdelayerSync(ethBlock *common.Block) (*common.WDelayerDat
 
 	// Get wDelayer events in the block
 	wDelayerEvents, err := s.ethClient.WDelayerEventsByBlock(blockNum, &ethBlock.Hash)
-	if err != nil {
+	if err != nil && err.Error() == errStrUnknownBlock {
-		return nil, tracerr.Wrap(err)
+		return nil, tracerr.Wrap(ErrUnknownBlock)
+	} else if err != nil {
+		return nil, tracerr.Wrap(fmt.Errorf("WDelayerEventsByBlock: %w", err))
 	}
 	// No events in this block
 	if wDelayerEvents == nil {

synchronizer/synchronizer_test.go

@@ -17,7 +17,6 @@ import (
 	"github.com/hermeznetwork/hermez-node/db/historydb"
 	"github.com/hermeznetwork/hermez-node/db/statedb"
 	"github.com/hermeznetwork/hermez-node/eth"
-	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/hermez-node/test"
 	"github.com/hermeznetwork/hermez-node/test/til"
 	"github.com/jinzhu/copier"
@@ -321,6 +320,14 @@ func newTestModules(t *testing.T) (*statedb.StateDB, *historydb.HistoryDB) {
 	return stateDB, historyDB
 }
 
+func newBigInt(s string) *big.Int {
+	v, ok := new(big.Int).SetString(s, 10)
+	if !ok {
+		panic(fmt.Errorf("Can't set big.Int from %s", s))
+	}
+	return v
+}
+
 func TestSyncGeneral(t *testing.T) {
 	//
 	// Setup
@@ -339,7 +346,6 @@ func TestSyncGeneral(t *testing.T) {
 	s, err := NewSynchronizer(client, historyDB, stateDB, Config{
 		StatsRefreshPeriod: 0 * time.Second,
 	})
-	log.Error(err)
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -434,12 +440,22 @@ func TestSyncGeneral(t *testing.T) {
 	require.Equal(t, 5, len(blocks[i].Rollup.L1UserTxs))
 	require.Equal(t, 2, len(blocks[i].Rollup.Batches))
 	require.Equal(t, 2, len(blocks[i].Rollup.Batches[0].L1CoordinatorTxs))
+	// Set StateRoots for batches manually (til doesn't set it)
+	blocks[i].Rollup.Batches[0].Batch.StateRoot =
+		newBigInt("18906357591508007884273218035694076596537737437965299189312069102730480717391")
+	blocks[i].Rollup.Batches[1].Batch.StateRoot =
+		newBigInt("9513185123401321669660637227182204000277156839501731093239187625486561933297")
 	// blocks 1 (blockNum=3)
 	i = 1
 	require.Equal(t, 3, int(blocks[i].Block.Num))
 	require.Equal(t, 5, len(blocks[i].Rollup.L1UserTxs))
 	require.Equal(t, 2, len(blocks[i].Rollup.Batches))
 	require.Equal(t, 3, len(blocks[i].Rollup.Batches[0].L2Txs))
+	// Set StateRoots for batches manually (til doesn't set it)
+	blocks[i].Rollup.Batches[0].Batch.StateRoot =
+		newBigInt("13060270878200012606074130020925677466793317216609491464427188889005039616594")
+	blocks[i].Rollup.Batches[1].Batch.StateRoot =
+		newBigInt("21427104994652624302859637783375978708867165042357535792408500519060088086054")
 
 	// Generate extra required data
 	ethAddTokens(blocks, client)
@@ -614,6 +630,12 @@ func TestSyncGeneral(t *testing.T) {
 	blocks, err = tc.GenerateBlocks(set2)
 	require.NoError(t, err)
 
+	// Set StateRoots for batches manually (til doesn't set it)
+	blocks[0].Rollup.Batches[0].Batch.StateRoot =
+		newBigInt("11218510534825843475100588932060366395781087435899915642332104464234485046683")
+	blocks[0].Rollup.Batches[1].Batch.StateRoot =
+		newBigInt("20283020730369146334077598087403837297563965802277806438205710455191646998983")
+
 	for i := 0; i < 4; i++ {
 		client.CtlRollback()
 	}

test/proofserver/proofserver.go

@@ -145,7 +145,7 @@ const longWaitDuration = 999 * time.Hour
 // const provingDuration = 2 * time.Second
 
 func (s *Mock) runProver(ctx context.Context) {
-	waitDuration := longWaitDuration
+	waitCh := time.After(longWaitDuration)
 	for {
 		select {
 		case <-ctx.Done():
@@ -153,21 +153,21 @@ func (s *Mock) runProver(ctx context.Context) {
 		case msg := <-s.msgCh:
 			switch msg.value {
 			case "cancel":
-				waitDuration = longWaitDuration
+				waitCh = time.After(longWaitDuration)
 				s.Lock()
 				if !s.status.IsReady() {
 					s.status = prover.StatusCodeAborted
 				}
 				s.Unlock()
 			case "prove":
-				waitDuration = s.provingDuration
+				waitCh = time.After(s.provingDuration)
 				s.Lock()
 				s.status = prover.StatusCodeBusy
 				s.Unlock()
 			}
 			msg.ackCh <- true
-		case <-time.After(waitDuration):
+		case <-waitCh:
-			waitDuration = longWaitDuration
+			waitCh = time.After(longWaitDuration)
 			s.Lock()
 			if s.status != prover.StatusCodeBusy {
 				s.Unlock()

txprocessor/txprocessor.go

@@ -405,40 +405,41 @@ func (tp *TxProcessor) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinat
 		return nil, nil
 	}
 
-	// once all txs processed (exitTree root frozen), for each Exit,
-	// generate common.ExitInfo data
-	var exitInfos []common.ExitInfo
-	exitInfosByIdx := make(map[common.Idx]*common.ExitInfo)
-	for i := 0; i < nTx; i++ {
-		if !exits[i].exit {
-			continue
-		}
-		exitIdx := exits[i].idx
-		exitAccount := exits[i].acc
-
-		// 0. generate MerkleProof
-		p, err := exitTree.GenerateSCVerifierProof(exitIdx.BigInt(), nil)
-		if err != nil {
-			return nil, tracerr.Wrap(err)
-		}
-
-		// 1. generate common.ExitInfo
-		ei := common.ExitInfo{
-			AccountIdx:  exitIdx,
-			MerkleProof: p,
-			Balance:     exitAccount.Balance,
-		}
-		if prevExit, ok := exitInfosByIdx[exitIdx]; !ok {
-			exitInfos = append(exitInfos, ei)
-			exitInfosByIdx[exitIdx] = &exitInfos[len(exitInfos)-1]
-		} else {
-			*prevExit = ei
-		}
-	}
-
 	if tp.s.Type() == statedb.TypeSynchronizer {
-		// retuTypeexitInfos, createdAccounts and collectedFees, so Synchronizer will
+		// once all txs processed (exitTree root frozen), for each
-		// be able to store it into HistoryDB for the concrete BatchNum
+		// Exit, generate common.ExitInfo data
+		var exitInfos []common.ExitInfo
+		exitInfosByIdx := make(map[common.Idx]*common.ExitInfo)
+		for i := 0; i < nTx; i++ {
+			if !exits[i].exit {
+				continue
+			}
+			exitIdx := exits[i].idx
+			exitAccount := exits[i].acc
+
+			// 0. generate MerkleProof
+			p, err := exitTree.GenerateSCVerifierProof(exitIdx.BigInt(), nil)
+			if err != nil {
+				return nil, tracerr.Wrap(err)
+			}
+
+			// 1. generate common.ExitInfo
+			ei := common.ExitInfo{
+				AccountIdx:  exitIdx,
+				MerkleProof: p,
+				Balance:     exitAccount.Balance,
+			}
+			if prevExit, ok := exitInfosByIdx[exitIdx]; !ok {
+				exitInfos = append(exitInfos, ei)
+				exitInfosByIdx[exitIdx] = &exitInfos[len(exitInfos)-1]
+			} else {
+				*prevExit = ei
+			}
+		}
+
+		// return exitInfos, createdAccounts and collectedFees, so
+		// Synchronizer will be able to store it into HistoryDB for the
+		// concrete BatchNum
 		return &ProcessTxOutput{
 			ZKInputs:           nil,
 			ExitInfos:          exitInfos,

txselector/txselector.go

@@ -88,12 +88,8 @@ func (txsel *TxSelector) LocalAccountsDB() *statedb.LocalStateDB {
 
 // Reset tells the TxSelector to get it's internal AccountsDB
 // from the required `batchNum`
-func (txsel *TxSelector) Reset(batchNum common.BatchNum) error {
+func (txsel *TxSelector) Reset(batchNum common.BatchNum, fromSynchronizer bool) error {
-	err := txsel.localAccountsDB.Reset(batchNum, true)
+	return tracerr.Wrap(txsel.localAccountsDB.Reset(batchNum, fromSynchronizer))
-	if err != nil {
-		return tracerr.Wrap(err)
-	}
-	return nil
 }
 
 func (txsel *TxSelector) getCoordIdx(tokenID common.TokenID) (common.Idx, error) {