Synchronizer main loop & reorg (#82)

* Synchronizer * mend Synchronizer main loop & reorg * mend Synchronizer main loop & reorg * mend Synchronizer main loop & reorg * Update PR and apply small changes Update PR and apply small changes: - Remove arbitrary line jumps (for example after an `err:=` there are cases with extra line before error check, and there are cases without the extra jumpline. Another example is the empty lines between a comment and the line of comment that is explained) - Fix some typo - Fix value printing of `lastSavedBlock` instead of `latestBlockNum` - Uncomment parameters of structs and use linter syntax to avoid unused checks * Update Synchr after master-pull to last types to compile Co-authored-by: Toni Ramírez <toni@iden3.com> Co-authored-by: arnaucube <root@arnaucube.com>
2026-02-06 19:06:42 +01:00 · 2020-09-06 18:12:52 +02:00
parent 4d02308057
commit 9db9508b44
5 changed files with 353 additions and 1 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -21,5 +21,6 @@ jobs:
    - name: Test
      env:
        POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }}
+        ETHCLIENT_DIAL_URL: ${{ secrets.ETHCLIENT_DIAL_URL }}
        GOARCH: ${{ matrix.goarch }}
      run: go test ./... -v
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ POSTGRES_PASS=yourpasswordhere; sudo docker run --rm --name hermez-db-test -p 54
 ```
 - Then, run the tests with the password as env var
 ```
-POSTGRES_PASS=yourpasswordhere go test ./...
+POSTGRES_PASS=yourpasswordhere ETHCLIENT_DIAL_URL=yourethereumurlhere go test ./...
 ```

 ## Lint
--- a/go.sum
+++ b/go.sum
@@ -147,6 +147,7 @@ github.com/ethereum/go-ethereum v1.9.17/go.mod h1:kihoiSg74VC4dZAXMkmoWp70oQabz4
 github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@@ -373,6 +374,7 @@ github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
@@ -380,6 +382,7 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@@ -495,6 +498,7 @@ github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rogpeppe/go-internal v1.5.2 h1:qLvObTrvO/XRCqmkKxUlOBc48bI3efyDuAZe25QiF0w=
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI=
 github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ=
 github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 h1:HXr/qUllAWv9riaI4zh2eXWKmCSDqVS/XH1MRHLKRwk=
@@ -692,6 +696,7 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -753,12 +758,14 @@ gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3M
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0=
 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
 gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
 gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
 gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
--- a/synchronizer/synchronizer.go
+++ b/synchronizer/synchronizer.go
@@ -0,0 +1,272 @@
+package synchronizer
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"math/big"
+	"sync"
+
+	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/hermeznetwork/hermez-node/db/historydb"
+	"github.com/hermeznetwork/hermez-node/db/statedb"
+	"github.com/hermeznetwork/hermez-node/eth"
+	"github.com/hermeznetwork/hermez-node/log"
+)
+
+const (
+	blocksToSync = 20 // TODO: This will be deleted once we can get the firstSavedBlock from the ethClient
+)
+
+var (
+	// ErrNotAbleToSync is used when there is not possible to find a valid block to sync
+	ErrNotAbleToSync = errors.New("it has not been possible to synchronize any block")
+)
+
+// BatchData contains information about Batches from the contracts
+//nolint:structcheck,unused
+type BatchData struct {
+	l1txs              []common.L1Tx
+	l2txs              []common.L2Tx
+	registeredAccounts []common.Account
+	exitTree           []common.ExitInfo
+}
+
+// BlockData contains information about Blocks from the contracts
+//nolint:structcheck,unused
+type BlockData struct {
+	block *common.Block
+	// Rollup
+	batches          []BatchData
+	withdrawals      []common.ExitInfo
+	registeredTokens []common.Token
+	rollupVars       *common.RollupVars
+	// Auction
+	bids         []common.Bid
+	coordinators []common.Coordinator
+	auctionVars  *common.AuctionVars
+}
+
+// Status is returned by the Status method
+type Status struct {
+	CurrentBlock      uint64
+	CurrentBatch      common.BatchNum
+	CurrentForgerAddr ethCommon.Address
+	NextForgerAddr    ethCommon.Address
+	Synchronized      bool
+}
+
+// Synchronizer implements the Synchronizer type
+type Synchronizer struct {
+	ethClient       *eth.Client
+	historyDB       *historydb.HistoryDB
+	stateDB         *statedb.StateDB
+	firstSavedBlock *common.Block
+	mux             sync.Mutex
+}
+
+// NewSynchronizer creates a new Synchronizer
+func NewSynchronizer(ethClient *eth.Client, historyDB *historydb.HistoryDB, stateDB *statedb.StateDB) *Synchronizer {
+	s := &Synchronizer{
+		ethClient: ethClient,
+		historyDB: historyDB,
+		stateDB:   stateDB,
+	}
+	return s
+}
+
+// Sync updates History and State DB with information from the blockchain
+func (s *Synchronizer) Sync() error {
+	// Avoid new sync while performing one
+	s.mux.Lock()
+	defer s.mux.Unlock()
+
+	var lastStoredForgeL1TxsNum uint64
+
+	// TODO: Get this information from ethClient once it's implemented
+	// for the moment we will get the latestblock - 20 as firstSavedBlock
+	latestBlock, err := s.ethClient.BlockByNumber(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+	s.firstSavedBlock, err = s.ethClient.BlockByNumber(context.Background(), big.NewInt(int64(latestBlock.EthBlockNum-blocksToSync)))
+	if err != nil {
+		return err
+	}
+
+	// Get lastSavedBlock from History DB
+	lastSavedBlock, err := s.historyDB.GetLastBlock()
+	if err != nil && err != sql.ErrNoRows {
+		return err
+	}
+
+	// Check if we got a block or nil
+	// In case of nil we must do a full sync
+	if lastSavedBlock == nil || lastSavedBlock.EthBlockNum == 0 {
+		lastSavedBlock = s.firstSavedBlock
+	} else {
+		// Get the latest block we have in History DB from blockchain to detect a reorg
+		ethBlock, err := s.ethClient.BlockByNumber(context.Background(), big.NewInt(int64(lastSavedBlock.EthBlockNum)))
+		if err != nil {
+			return err
+		}
+
+		if ethBlock.Hash != lastSavedBlock.Hash {
+			// Reorg detected
+			log.Debugf("Reorg Detected...")
+			err := s.reorg(lastSavedBlock)
+			if err != nil {
+				return err
+			}
+
+			lastSavedBlock, err = s.historyDB.GetLastBlock()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	log.Debugf("Syncing...")
+
+	// Get latest blockNum in blockchain
+	latestBlockNum, err := s.ethClient.CurrentBlock()
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Blocks to sync: %v (lastSavedBlock: %v, latestBlock: %v)", latestBlockNum.Uint64()-lastSavedBlock.EthBlockNum, lastSavedBlock.EthBlockNum, latestBlockNum)
+
+	for lastSavedBlock.EthBlockNum < latestBlockNum.Uint64() {
+		ethBlock, err := s.ethClient.BlockByNumber(context.Background(), big.NewInt(int64(lastSavedBlock.EthBlockNum+1)))
+		if err != nil {
+			return err
+		}
+
+		// Get data from the rollup contract
+		blockData, batchData, err := s.rollupSync(ethBlock, lastStoredForgeL1TxsNum)
+		if err != nil {
+			return err
+		}
+
+		// Get data from the auction contract
+		err = s.auctionSync(blockData, batchData)
+		if err != nil {
+			return err
+		}
+
+		// Add rollupData and auctionData once the method is updated
+		err = s.historyDB.AddBlock(ethBlock)
+		if err != nil {
+			return err
+		}
+
+		// We get the block on every iteration
+		lastSavedBlock, err = s.historyDB.GetLastBlock()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// reorg manages a reorg, updating History and State DB as needed
+func (s *Synchronizer) reorg(uncleBlock *common.Block) error {
+	var block *common.Block
+	blockNum := uncleBlock.EthBlockNum
+	found := false
+
+	log.Debugf("Reorg first uncle block: %v", blockNum)
+
+	// Iterate History DB and the blokchain looking for the latest valid block
+	for !found && blockNum > s.firstSavedBlock.EthBlockNum {
+		header, err := s.ethClient.HeaderByNumber(context.Background(), big.NewInt(int64(blockNum)))
+		if err != nil {
+			return err
+		}
+
+		block, err = s.historyDB.GetBlock(blockNum)
+		if err != nil {
+			return err
+		}
+		if block.Hash == header.Hash() {
+			found = true
+			log.Debugf("Found valid block: %v", blockNum)
+		} else {
+			log.Debugf("Discarding block: %v", blockNum)
+		}
+
+		blockNum--
+	}
+
+	if found {
+		// Set History DB and State DB to the correct state
+		err := s.historyDB.Reorg(block.EthBlockNum)
+		if err != nil {
+			return err
+		}
+
+		batchNum, err := s.historyDB.GetLastBatchNum()
+		if err != nil && err != sql.ErrNoRows {
+			return err
+		}
+		if batchNum != 0 {
+			err = s.stateDB.Reset(batchNum)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	return ErrNotAbleToSync
+}
+
+// Status returns current status values from the Synchronizer
+func (s *Synchronizer) Status() (*Status, error) {
+	// Avoid possible inconsistencies
+	s.mux.Lock()
+	defer s.mux.Unlock()
+
+	var status *Status
+
+	// Get latest block in History DB
+	lastSavedBlock, err := s.historyDB.GetLastBlock()
+	if err != nil {
+		return nil, err
+	}
+	status.CurrentBlock = lastSavedBlock.EthBlockNum
+
+	// Get latest batch in History DB
+	lastSavedBatch, err := s.historyDB.GetLastBatchNum()
+	if err != nil && err != sql.ErrNoRows {
+		return nil, err
+	}
+	status.CurrentBatch = lastSavedBatch
+
+	// Get latest blockNum in blockchain
+	latestBlockNum, err := s.ethClient.CurrentBlock()
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: Get CurrentForgerAddr & NextForgerAddr
+
+	// Check if Synchronizer is synchronized
+	status.Synchronized = status.CurrentBlock == latestBlockNum.Uint64()
+	return status, nil
+}
+
+// rollupSync gets information from the Rollup Contract
+func (s *Synchronizer) rollupSync(block *common.Block, lastStoredForgeL1TxsNum uint64) (*BlockData, []*BatchData, error) {
+	// To be implemented
+	return nil, nil, nil
+}
+
+// auctionSync gets information from the Auction Contract
+func (s *Synchronizer) auctionSync(blockData *BlockData, batchData []*BatchData) error {
+	// To be implemented
+	return nil
+}
--- a/synchronizer/synchronizer_test.go
+++ b/synchronizer/synchronizer_test.go
@@ -0,0 +1,72 @@
+package synchronizer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/hermeznetwork/hermez-node/db/historydb"
+	"github.com/hermeznetwork/hermez-node/db/statedb"
+	"github.com/hermeznetwork/hermez-node/eth"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test(t *testing.T) {
+	// Int State DB
+	dir, err := ioutil.TempDir("", "tmpdb")
+	require.Nil(t, err)
+
+	sdb, err := statedb.NewStateDB(dir, true, 32)
+	assert.Nil(t, err)
+
+	// Init History DB
+	pass := os.Getenv("POSTGRES_PASS")
+	historyDB, err := historydb.NewHistoryDB(5432, "localhost", "hermez", pass, "history")
+	require.Nil(t, err)
+	err = historyDB.Reorg(0)
+	assert.Nil(t, err)
+
+	// Init eth client
+	ehtClientDialURL := os.Getenv("ETHCLIENT_DIAL_URL")
+	ethClient, err := ethclient.Dial(ehtClientDialURL)
+	require.Nil(t, err)
+
+	client := eth.NewClient(ethClient, nil, nil, nil)
+
+	// Create Synchronizer
+	s := NewSynchronizer(client, historyDB, sdb)
+
+	// Test Sync
+	err = s.Sync()
+	require.Nil(t, err)
+
+	// TODO: Reorg will be properly tested once we have the mock ethClient implemented
+	/*
+		// Force a Reorg
+		lastSavedBlock, err := historyDB.GetLastBlock()
+		require.Nil(t, err)
+
+		lastSavedBlock.EthBlockNum++
+		err = historyDB.AddBlock(lastSavedBlock)
+		require.Nil(t, err)
+
+		lastSavedBlock.EthBlockNum++
+		err = historyDB.AddBlock(lastSavedBlock)
+		require.Nil(t, err)
+
+		log.Debugf("Wait for the blockchain to generate some blocks...")
+		time.Sleep(40 * time.Second)
+
+
+		err = s.Sync()
+		require.Nil(t, err)
+	*/
+
+	// Close History DB
+	if err := historyDB.Close(); err != nil {
+		fmt.Println("Error closing the history DB:", err)
+	}
+}