Browse Source
Synchronizer main loop & reorg (#82)
* Synchronizer * mend Synchronizer main loop & reorg * mend Synchronizer main loop & reorg * mend Synchronizer main loop & reorg * Update PR and apply small changes Update PR and apply small changes: - Remove arbitrary line jumps (for example after an `err:=` there are cases with extra line before error check, and there are cases without the extra jumpline. Another example is the empty lines between a comment and the line of comment that is explained) - Fix some typo - Fix value printing of `lastSavedBlock` instead of `latestBlockNum` - Uncomment parameters of structs and use linter syntax to avoid unused checks * Update Synchr after master-pull to last types to compile Co-authored-by: Toni Ramírez <toni@iden3.com> Co-authored-by: arnaucube <root@arnaucube.com>feature/sql-semaphore1
Toni Ramírez
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 353 additions and 1 deletions
Unified View
Diff Options
-
+1 -0.github/workflows/test.yml
-
+1 -1README.md
-
+7 -0go.sum
-
+272 -0synchronizer/synchronizer.go
-
+72 -0synchronizer/synchronizer_test.go
@ -0,0 +1,272 @@ |
|||||
|
package synchronizer |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"database/sql" |
||||
|
"errors" |
||||
|
"math/big" |
||||
|
"sync" |
||||
|
|
||||
|
ethCommon "github.com/ethereum/go-ethereum/common" |
||||
|
"github.com/hermeznetwork/hermez-node/common" |
||||
|
"github.com/hermeznetwork/hermez-node/db/historydb" |
||||
|
"github.com/hermeznetwork/hermez-node/db/statedb" |
||||
|
"github.com/hermeznetwork/hermez-node/eth" |
||||
|
"github.com/hermeznetwork/hermez-node/log" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
blocksToSync = 20 // TODO: This will be deleted once we can get the firstSavedBlock from the ethClient
|
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
// ErrNotAbleToSync is used when there is not possible to find a valid block to sync
|
||||
|
ErrNotAbleToSync = errors.New("it has not been possible to synchronize any block") |
||||
|
) |
||||
|
|
||||
|
// BatchData contains information about Batches from the contracts
|
||||
|
//nolint:structcheck,unused
|
||||
|
type BatchData struct { |
||||
|
l1txs []common.L1Tx |
||||
|
l2txs []common.L2Tx |
||||
|
registeredAccounts []common.Account |
||||
|
exitTree []common.ExitInfo |
||||
|
} |
||||
|
|
||||
|
// BlockData contains information about Blocks from the contracts
|
||||
|
//nolint:structcheck,unused
|
||||
|
type BlockData struct { |
||||
|
block *common.Block |
||||
|
// Rollup
|
||||
|
batches []BatchData |
||||
|
withdrawals []common.ExitInfo |
||||
|
registeredTokens []common.Token |
||||
|
rollupVars *common.RollupVars |
||||
|
// Auction
|
||||
|
bids []common.Bid |
||||
|
coordinators []common.Coordinator |
||||
|
auctionVars *common.AuctionVars |
||||
|
} |
||||
|
|
||||
|
// Status is returned by the Status method
|
||||
|
type Status struct { |
||||
|
CurrentBlock uint64 |
||||
|
CurrentBatch common.BatchNum |
||||
|
CurrentForgerAddr ethCommon.Address |
||||
|
NextForgerAddr ethCommon.Address |
||||
|
Synchronized bool |
||||
|
} |
||||
|
|
||||
|
// Synchronizer implements the Synchronizer type
|
||||
|
type Synchronizer struct { |
||||
|
ethClient *eth.Client |
||||
|
historyDB *historydb.HistoryDB |
||||
|
stateDB *statedb.StateDB |
||||
|
firstSavedBlock *common.Block |
||||
|
mux sync.Mutex |
||||
|
} |
||||
|
|
||||
|
// NewSynchronizer creates a new Synchronizer
|
||||
|
func NewSynchronizer(ethClient *eth.Client, historyDB *historydb.HistoryDB, stateDB *statedb.StateDB) *Synchronizer { |
||||
|
s := &Synchronizer{ |
||||
|
ethClient: ethClient, |
||||
|
historyDB: historyDB, |
||||
|
stateDB: stateDB, |
||||
|
} |
||||
|
return s |
||||
|
} |
||||
|
|
||||
|
// Sync updates History and State DB with information from the blockchain
|
||||
|
func (s *Synchronizer) Sync() error { |
||||
|
// Avoid new sync while performing one
|
||||
|
s.mux.Lock() |
||||
|
defer s.mux.Unlock() |
||||
|
|
||||
|
var lastStoredForgeL1TxsNum uint64 |
||||
|
|
||||
|
// TODO: Get this information from ethClient once it's implemented
|
||||
|
// for the moment we will get the latestblock - 20 as firstSavedBlock
|
||||
|
latestBlock, err := s.ethClient.BlockByNumber(context.Background(), nil) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
s.firstSavedBlock, err = s.ethClient.BlockByNumber(context.Background(), big.NewInt(int64(latestBlock.EthBlockNum-blocksToSync))) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Get lastSavedBlock from History DB
|
||||
|
lastSavedBlock, err := s.historyDB.GetLastBlock() |
||||
|
if err != nil && err != sql.ErrNoRows { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Check if we got a block or nil
|
||||
|
// In case of nil we must do a full sync
|
||||
|
if lastSavedBlock == nil || lastSavedBlock.EthBlockNum == 0 { |
||||
|
lastSavedBlock = s.firstSavedBlock |
||||
|
} else { |
||||
|
// Get the latest block we have in History DB from blockchain to detect a reorg
|
||||
|
ethBlock, err := s.ethClient.BlockByNumber(context.Background(), big.NewInt(int64(lastSavedBlock.EthBlockNum))) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if ethBlock.Hash != lastSavedBlock.Hash { |
||||
|
// Reorg detected
|
||||
|
log.Debugf("Reorg Detected...") |
||||
|
err := s.reorg(lastSavedBlock) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
lastSavedBlock, err = s.historyDB.GetLastBlock() |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
log.Debugf("Syncing...") |
||||
|
|
||||
|
// Get latest blockNum in blockchain
|
||||
|
latestBlockNum, err := s.ethClient.CurrentBlock() |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
log.Debugf("Blocks to sync: %v (lastSavedBlock: %v, latestBlock: %v)", latestBlockNum.Uint64()-lastSavedBlock.EthBlockNum, lastSavedBlock.EthBlockNum, latestBlockNum) |
||||
|
|
||||
|
for lastSavedBlock.EthBlockNum < latestBlockNum.Uint64() { |
||||
|
ethBlock, err := s.ethClient.BlockByNumber(context.Background(), big.NewInt(int64(lastSavedBlock.EthBlockNum+1))) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Get data from the rollup contract
|
||||
|
blockData, batchData, err := s.rollupSync(ethBlock, lastStoredForgeL1TxsNum) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Get data from the auction contract
|
||||
|
err = s.auctionSync(blockData, batchData) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Add rollupData and auctionData once the method is updated
|
||||
|
err = s.historyDB.AddBlock(ethBlock) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// We get the block on every iteration
|
||||
|
lastSavedBlock, err = s.historyDB.GetLastBlock() |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// reorg manages a reorg, updating History and State DB as needed
|
||||
|
func (s *Synchronizer) reorg(uncleBlock *common.Block) error { |
||||
|
var block *common.Block |
||||
|
blockNum := uncleBlock.EthBlockNum |
||||
|
found := false |
||||
|
|
||||
|
log.Debugf("Reorg first uncle block: %v", blockNum) |
||||
|
|
||||
|
// Iterate History DB and the blokchain looking for the latest valid block
|
||||
|
for !found && blockNum > s.firstSavedBlock.EthBlockNum { |
||||
|
header, err := s.ethClient.HeaderByNumber(context.Background(), big.NewInt(int64(blockNum))) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
block, err = s.historyDB.GetBlock(blockNum) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
if block.Hash == header.Hash() { |
||||
|
found = true |
||||
|
log.Debugf("Found valid block: %v", blockNum) |
||||
|
} else { |
||||
|
log.Debugf("Discarding block: %v", blockNum) |
||||
|
} |
||||
|
|
||||
|
blockNum-- |
||||
|
} |
||||
|
|
||||
|
if found { |
||||
|
// Set History DB and State DB to the correct state
|
||||
|
err := s.historyDB.Reorg(block.EthBlockNum) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
batchNum, err := s.historyDB.GetLastBatchNum() |
||||
|
if err != nil && err != sql.ErrNoRows { |
||||
|
return err |
||||
|
} |
||||
|
if batchNum != 0 { |
||||
|
err = s.stateDB.Reset(batchNum) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
return ErrNotAbleToSync |
||||
|
} |
||||
|
|
||||
|
// Status returns current status values from the Synchronizer
|
||||
|
func (s *Synchronizer) Status() (*Status, error) { |
||||
|
// Avoid possible inconsistencies
|
||||
|
s.mux.Lock() |
||||
|
defer s.mux.Unlock() |
||||
|
|
||||
|
var status *Status |
||||
|
|
||||
|
// Get latest block in History DB
|
||||
|
lastSavedBlock, err := s.historyDB.GetLastBlock() |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
status.CurrentBlock = lastSavedBlock.EthBlockNum |
||||
|
|
||||
|
// Get latest batch in History DB
|
||||
|
lastSavedBatch, err := s.historyDB.GetLastBatchNum() |
||||
|
if err != nil && err != sql.ErrNoRows { |
||||
|
return nil, err |
||||
|
} |
||||
|
status.CurrentBatch = lastSavedBatch |
||||
|
|
||||
|
// Get latest blockNum in blockchain
|
||||
|
latestBlockNum, err := s.ethClient.CurrentBlock() |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
// TODO: Get CurrentForgerAddr & NextForgerAddr
|
||||
|
|
||||
|
// Check if Synchronizer is synchronized
|
||||
|
status.Synchronized = status.CurrentBlock == latestBlockNum.Uint64() |
||||
|
return status, nil |
||||
|
} |
||||
|
|
||||
|
// rollupSync gets information from the Rollup Contract
|
||||
|
func (s *Synchronizer) rollupSync(block *common.Block, lastStoredForgeL1TxsNum uint64) (*BlockData, []*BatchData, error) { |
||||
|
// To be implemented
|
||||
|
return nil, nil, nil |
||||
|
} |
||||
|
|
||||
|
// auctionSync gets information from the Auction Contract
|
||||
|
func (s *Synchronizer) auctionSync(blockData *BlockData, batchData []*BatchData) error { |
||||
|
// To be implemented
|
||||
|
return nil |
||||
|
} |
||||
@ -0,0 +1,72 @@ |
|||||
|
package synchronizer |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"io/ioutil" |
||||
|
"os" |
||||
|
"testing" |
||||
|
|
||||
|
"github.com/ethereum/go-ethereum/ethclient" |
||||
|
"github.com/hermeznetwork/hermez-node/db/historydb" |
||||
|
"github.com/hermeznetwork/hermez-node/db/statedb" |
||||
|
"github.com/hermeznetwork/hermez-node/eth" |
||||
|
"github.com/stretchr/testify/assert" |
||||
|
"github.com/stretchr/testify/require" |
||||
|
) |
||||
|
|
||||
|
func Test(t *testing.T) { |
||||
|
// Int State DB
|
||||
|
dir, err := ioutil.TempDir("", "tmpdb") |
||||
|
require.Nil(t, err) |
||||
|
|
||||
|
sdb, err := statedb.NewStateDB(dir, true, 32) |
||||
|
assert.Nil(t, err) |
||||
|
|
||||
|
// Init History DB
|
||||
|
pass := os.Getenv("POSTGRES_PASS") |
||||
|
historyDB, err := historydb.NewHistoryDB(5432, "localhost", "hermez", pass, "history") |
||||
|
require.Nil(t, err) |
||||
|
err = historyDB.Reorg(0) |
||||
|
assert.Nil(t, err) |
||||
|
|
||||
|
// Init eth client
|
||||
|
ehtClientDialURL := os.Getenv("ETHCLIENT_DIAL_URL") |
||||
|
ethClient, err := ethclient.Dial(ehtClientDialURL) |
||||
|
require.Nil(t, err) |
||||
|
|
||||
|
client := eth.NewClient(ethClient, nil, nil, nil) |
||||
|
|
||||
|
// Create Synchronizer
|
||||
|
s := NewSynchronizer(client, historyDB, sdb) |
||||
|
|
||||
|
// Test Sync
|
||||
|
err = s.Sync() |
||||
|
require.Nil(t, err) |
||||
|
|
||||
|
// TODO: Reorg will be properly tested once we have the mock ethClient implemented
|
||||
|
/* |
||||
|
// Force a Reorg
|
||||
|
lastSavedBlock, err := historyDB.GetLastBlock() |
||||
|
require.Nil(t, err) |
||||
|
|
||||
|
lastSavedBlock.EthBlockNum++ |
||||
|
err = historyDB.AddBlock(lastSavedBlock) |
||||
|
require.Nil(t, err) |
||||
|
|
||||
|
lastSavedBlock.EthBlockNum++ |
||||
|
err = historyDB.AddBlock(lastSavedBlock) |
||||
|
require.Nil(t, err) |
||||
|
|
||||
|
log.Debugf("Wait for the blockchain to generate some blocks...") |
||||
|
time.Sleep(40 * time.Second) |
||||
|
|
||||
|
|
||||
|
err = s.Sync() |
||||
|
require.Nil(t, err) |
||||
|
*/ |
||||
|
|
||||
|
// Close History DB
|
||||
|
if err := historyDB.Close(); err != nil { |
||||
|
fmt.Println("Error closing the history DB:", err) |
||||
|
} |
||||
|
} |
||||