arnaucube
arnaucube/hermez-nodePublic Self-hosted repository
Merge pull request #50 from hermeznetwork/feature/txselector-int-statedb

Integrate TxSelector with StateDB
a_bennassar arnaubennassar5@gmail.com
Thu, 13 Aug 2020 13:15:18 +0200
commit

9df30affc6ec92d5fd8b151e6efddb03212e87de

parent

b28b13d62013f7ecf834b0b9e9ba3d427511a164

M README.md → README.md

@@ -1,3 +1,14 @@

# hermez-node [![Go Report Card](https://goreportcard.com/badge/github.com/hermeznetwork/hermez-node)](https://goreportcard.com/report/github.com/hermeznetwork/hermez-node) [![Test Status](https://github.com/hermeznetwork/hermez-node/workflows/Test/badge.svg)](https://github.com/hermeznetwork/hermez-node/actions?query=workflow%3ATest) [![Lint Status](https://github.com/hermeznetwork/hermez-node/workflows/Lint/badge.svg)](https://github.com/hermeznetwork/hermez-node/actions?query=workflow%3ALint) [![GoDoc](https://godoc.org/github.com/hermeznetwork/hermez-node?status.svg)](https://godoc.org/github.com/hermeznetwork/hermez-node) Go implementation of the Hermez node. + + +## Test +- First run a docker instance of the PostgresSQL (where `yourpasswordhere` should be your password) +``` +POSTGRES_PASS=yourpasswordhere; sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=history -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$POSTGRES_PASS" -d postgres && sleep 2s && sudo docker exec hermez-db-test psql -a history -U hermez -c "CREATE DATABASE l2;" +``` +- Then, run the tests with the password as env var +``` +POSTGRES_PASS=yourpasswordhere go test ./... +```
M batchbuilder/batchbuilder_test.go → batchbuilder/batchbuilder_test.go

@@ -14,10 +14,10 @@ func TestBatchBuilder(t *testing.T) {

dir, err := ioutil.TempDir("", "tmpdb") require.Nil(t, err) - sdb, err := statedb.NewStateDB(dir, false, false, 0) + synchDB, err := statedb.NewStateDB(dir, false, false, 0) assert.Nil(t, err) - bb, err := NewBatchBuilder(sdb, nil, 0, 0, 32) + bb, err := NewBatchBuilder(synchDB, nil, 0, 0, 32) assert.Nil(t, err) fmt.Println(bb) }
M common/pooll2tx.go → common/pooll2tx.go

@@ -18,6 +18,7 @@ RqTx RqTx

Timestamp time.Time // time when added to the tx pool Signature babyjub.Signature // tx signature ToEthAddr eth.Address + AbsoluteFee float64 // TODO add methods to calculate this value from Tx.Fee tables + priceupdater tables } // RqTx Transaction Data used to indicate that a transaction depends on another transaction
M db/statedb/statedb.go → db/statedb/statedb.go

@@ -210,6 +210,12 @@ // Reset performs a reset, getting the state from

// LocalStateDB.synchronizerStateDB for the given batchNum func (l *LocalStateDB) Reset(batchNum int, fromSynchronizer bool) error { // TODO - + // if fromSynchronizer==true: + // make copy from l.synchronizerStateDB at the batchNum to the localStateDB + // if synchronizerStateDB does not have batchNum, return err + // the localStateDB checkpoint is set to batchNum + // else fromSynchronizer==false: + // the localStateDB checkpoint is set to batchNum + // if localStateDB does not have batchNum, return err return nil }
D txselector/common/tmp.go

@@ -1,40 +0,0 @@

-package common - -import ( - "math/big" - - ethCommon "github.com/ethereum/go-ethereum/common" -) - -// WIP this will be from hermeznetwork/common -type Tx struct { - FromAx [32]byte - FromAy [32]byte - FromEthAddr ethCommon.Address - ToAx [32]byte - ToAy [32]byte - ToEthAddr ethCommon.Address - OnChain bool - RqOffset []byte - NewAccount bool - TokenID uint32 - LoadAmount [3]byte - Amount [3]byte - Nonce uint64 - UserFee uint8 - UserFeeAbsolute uint64 - R8x [32]byte - R8y [32]byte - S [32]byte - RqTxData [32]byte -} - -// WIP this will be from hermeznetwork/common -type Account struct { - EthAddr ethCommon.Address - TokenID uint32 - Idx uint32 - Nonce uint64 - Balance *big.Int - // Ax, Ay -}
D txselector/mock/mock.go

@@ -1,33 +0,0 @@

-package mock - -import ( - "github.com/hermeznetwork/hermez-node/txselector/common" -) - -type MockDB struct { - Txs map[uint64][]common.Tx - - // AccountDB is the LocalAccountDB copy of the original AccountDB - AccountDB map[[36]byte]common.Account // [36]byte is tx.ToEthAddr + tx.TokenID - - PendingRegistersDB map[[36]byte]common.Account // [36]byte is tx.ToEthAddr + tx.TokenID -} - -func New() *MockDB { - return &MockDB{ - Txs: make(map[uint64][]common.Tx), - AccountDB: make(map[[36]byte]common.Account), - PendingRegistersDB: make(map[[36]byte]common.Account), - } -} - -func (m *MockDB) AddTx(batchID uint64, tx common.Tx) { - if _, ok := m.Txs[batchID]; !ok { - m.Txs[batchID] = []common.Tx{} - } - m.Txs[batchID] = append(m.Txs[batchID], tx) -} - -func (m *MockDB) GetTxs(batchID uint64) []common.Tx { - return m.Txs[batchID] -}
M txselector/txselector.go → txselector/txselector.go

@@ -3,12 +3,13 @@

import ( "sort" - "github.com/hermeznetwork/hermez-node/txselector/common" - "github.com/hermeznetwork/hermez-node/txselector/mock" + "github.com/hermeznetwork/hermez-node/common" + "github.com/hermeznetwork/hermez-node/db/l2db" + "github.com/hermeznetwork/hermez-node/db/statedb" ) // txs implements the interface Sort for an array of Tx -type txs []common.Tx +type txs []common.PoolL2Tx func (t txs) Len() int { return len(t)

@@ -17,57 +18,68 @@ func (t txs) Swap(i, j int) {

t[i], t[j] = t[j], t[i] } func (t txs) Less(i, j int) bool { - return t[i].UserFeeAbsolute > t[j].UserFeeAbsolute + return t[i].AbsoluteFee > t[j].AbsoluteFee } +// TxSelector implements all the functionalities to select the txs for the next batch type TxSelector struct { - // NMax is the maximum L1-user-tx for a batch - NMax uint64 - // MMax is the maximum L1-operator-tx for a batch - MMax uint64 - // PMax is the maximum L2-tx for a batch - PMax uint64 - // DB is a pointer to the database interface - DB *mock.MockDB + // MaxL1UserTxs is the maximum L1-user-tx for a batch + MaxL1UserTxs uint64 + // MaxL1OperatorTxs is the maximum L1-operator-tx for a batch + MaxL1OperatorTxs uint64 + // MaxTxs is the maximum txs for a batch + MaxTxs uint64 + + l2db *l2db.L2DB + localAccountsDB *statedb.LocalStateDB } -func NewTxSelector(db *mock.MockDB, nMax, mMax, pMax uint64) *TxSelector { - return &TxSelector{ - NMax: nMax, - MMax: mMax, - PMax: pMax, - DB: db, +// NewTxSelector returns a *TxSelector +func NewTxSelector(synchronizerStateDB *statedb.StateDB, l2 *l2db.L2DB, maxL1UserTxs, maxL1OperatorTxs, maxTxs uint64) (*TxSelector, error) { + localAccountsDB, err := statedb.NewLocalStateDB(synchronizerStateDB, false, 0) // without merkletree + if err != nil { + return nil, err } -} -func (txsel *TxSelector) updateLocalAccountDB(batchId uint64) error { - // if batchID > max(localAccountDB.BatchID) + 1 - // make a checkpoint of AccountDB at BatchID to a localAccountDB - // use localAccountDB[inputBatchID-1] + return &TxSelector{ + MaxL1UserTxs: maxL1UserTxs, + MaxL1OperatorTxs: maxL1OperatorTxs, + MaxTxs: maxTxs, + l2db: l2, + localAccountsDB: localAccountsDB, + }, nil +} +// Reset tells the TxSelector to get it's internal AccountsDB +// from the required `batchNum` +func (txsel *TxSelector) Reset(batchNum int) error { + err := txsel.localAccountsDB.Reset(batchNum, true) + if err != nil { + return err + } return nil } -func (txsel *TxSelector) GetL2TxSelection(batchID uint64) ([]common.Tx, error) { - err := txsel.updateLocalAccountDB(batchID) +// GetL2TxSelection returns a selection of the L2Txs for the next batch, from the L2DB pool +func (txsel *TxSelector) GetL2TxSelection(batchNum int) ([]common.PoolL2Tx, error) { + // get pending l2-tx from tx-pool + l2TxsRaw, err := txsel.l2db.GetPendingTxs() // once l2db ready, maybe use parameter 'batchNum' if err != nil { return nil, err } - // get pending l2-tx from tx-pool - txsRaw := txsel.DB.GetTxs(batchID) - // discard the txs that don't have an Account in the AccountDB var validTxs txs - for _, tx := range txsRaw { - accountID := getAccountID(tx.ToEthAddr, tx.TokenID) - if _, ok := txsel.DB.AccountDB[accountID]; ok { + for _, tx := range l2TxsRaw { + _, err = txsel.localAccountsDB.GetAccount(tx.FromIdx) + if err == nil { + // if FromIdx has an account into the AccountsDB validTxs = append(validTxs, tx) } } - // get most profitable L2-tx (len<NMax) - txs := txsel.getL2Profitable(validTxs) + // get most profitable L2-tx + txs := txsel.getL2Profitable(validTxs, txsel.MaxTxs) // apply L2-tx to local AccountDB, make checkpoint tagged with BatchID // update balances

@@ -77,66 +89,73 @@ // return selected txs

return txs, nil } -func (txsel *TxSelector) GetL1L2TxSelection(batchID uint64, l1txs []common.Tx) ([]common.Tx, []common.Tx, []common.Tx, error) { - err := txsel.updateLocalAccountDB(batchID) - if err != nil { - return nil, nil, nil, err - } - +// GetL1L2TxSelection returns the selection of L1 + L2 txs +func (txsel *TxSelector) GetL1L2TxSelection(batchNum int, l1txs []common.Tx) ([]common.Tx, []common.PoolL2Tx, []common.Tx, error) { // apply l1-user-tx to localAccountDB // create new leaves // update balances // update nonces // get pending l2-tx from tx-pool - txsRaw := txsel.DB.GetTxs(batchID) + l2TxsRaw, err := txsel.l2db.GetPendingTxs() // (batchID) + if err != nil { + return nil, nil, nil, err + } // discard the txs that don't have an Account in the AccountDB - // neither appear in the PendingRegistersDB + // neither appear in the AccountCreationAuthsDB var validTxs txs - for _, tx := range txsRaw { - accountID := getAccountID(tx.ToEthAddr, tx.TokenID) - exist := txsel.checkIfAccountExist(accountID) - if exist { + for _, tx := range l2TxsRaw { + if txsel.checkIfAccountExistOrPending(tx.FromIdx) { + // if FromIdx has an account into the AccountsDB validTxs = append(validTxs, tx) } } - // get most profitable L2-tx (len<NMax) - l2txs := txsel.getL2Profitable(validTxs) + // prepare (from the selected l2txs) pending to create from the AccountCreationAuthsDB + var accountCreationAuths []common.Account + // TODO once DB ready: + // if tx.ToIdx is in AccountCreationAuthsDB, take it and add it to + // the array 'accountCreationAuths' + // for _, tx := range l2txs { + // account, err := txsel.localAccountsDB.GetAccount(tx.ToIdx) + // if err != nil { + // return nil, nil, nil, err + // } + // if accountToCreate, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok { + // accountCreationAuths = append(accountCreationAuths, accountToCreate) + // } + // } - // prepare (from the selected l2txs) pending to register from the PendingRegistersDB - var pendingRegisters []common.Account - for _, tx := range l2txs { - accountID := getAccountID(tx.ToEthAddr, tx.TokenID) - if toRegister, ok := txsel.DB.PendingRegistersDB[accountID]; ok { - pendingRegisters = append(pendingRegisters, toRegister) - } - } + // create L1-operator-tx for each L2-tx selected in which the recipient does not have account + l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(accountCreationAuths) // only with the L2-tx selected ones - // create L1-operator-tx for each L2-tx selected in which the recipient does not have account - l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(pendingRegisters) // only with the L2-tx selected ones + // get most profitable L2-tx + maxL2Txs := txsel.MaxTxs - uint64(len(l1OperatorTxs)) // - len(l1UserTxs) + l2txs := txsel.getL2Profitable(validTxs, maxL2Txs) return l1txs, l2txs, l1OperatorTxs, nil } -func (txsel *TxSelector) checkIfAccountExist(accountID [36]byte) bool { +func (txsel *TxSelector) checkIfAccountExistOrPending(idx common.Idx) bool { // check if account exist in AccountDB - if _, ok := txsel.DB.AccountDB[accountID]; ok { - return true - } - // check if account is pending to register - if _, ok := txsel.DB.PendingRegistersDB[accountID]; ok { - return true + _, err := txsel.localAccountsDB.GetAccount(idx) + if err != nil { + return false } + // check if account is pending to create + // TODO need a method in the DB to get the PendingRegisters + // if _, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok { + // return true + // } return false } -func (txsel *TxSelector) getL2Profitable(txs txs) txs { +func (txsel *TxSelector) getL2Profitable(txs txs, max uint64) txs { sort.Sort(txs) - return txs[:txsel.PMax] + return txs[:max] } -func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []common.Account) txs { +func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []common.Account) []common.Tx { // - return txs{} + return nil }
M txselector/txselector_test.go → txselector/txselector_test.go

@@ -2,76 +2,75 @@ package txselector

import ( "fmt" - "math/big" + "io/ioutil" "testing" - "github.com/hermeznetwork/hermez-node/txselector/common" - "github.com/hermeznetwork/hermez-node/txselector/mock" + "github.com/hermeznetwork/hermez-node/db/l2db" + "github.com/hermeznetwork/hermez-node/db/statedb" - ethCommon "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func initMockDB() *mock.MockDB { - m := mock.New() - +/* +func initTestDB(l2 *l2db.L2DB, sdb *statedb.StateDB) *mock.MockDB { txs := []common.Tx{ { - FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"), - ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - TokenID: 1, - Nonce: 1, - UserFeeAbsolute: 1, + FromIdx: common.Idx(0), + ToIdx: common.Idx(1), + TokenID: 1, + Nonce: 1, + AbsoluteFee: 1, }, { - FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"), - ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - TokenID: 1, - Nonce: 2, - UserFeeAbsolute: 3, + FromIdx: common.Idx(0), + ToIdx: common.Idx(1), + TokenID: 1, + Nonce: 2, + AbsoluteFee: 3, }, { - FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"), - ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - TokenID: 1, - Nonce: 4, - UserFeeAbsolute: 6, + FromIdx: common.Idx(0), + ToIdx: common.Idx(1), + TokenID: 1, + Nonce: 4, + AbsoluteFee: 6, }, { - FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"), - ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - TokenID: 1, - Nonce: 4, - UserFeeAbsolute: 4, + FromIdx: common.Idx(0), + ToIdx: common.Idx(1), + TokenID: 1, + Nonce: 4, + AbsoluteFee: 4, }, { - FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - ToEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"), - TokenID: 1, - Nonce: 1, - UserFeeAbsolute: 4, + ToIdx: common.Idx(1), + FromIdx: common.Idx(0), + TokenID: 1, + Nonce: 1, + AbsoluteFee: 4, }, { - FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - ToEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"), - TokenID: 1, - Nonce: 2, - UserFeeAbsolute: 3, + ToIdx: common.Idx(1), + FromIdx: common.Idx(0), + TokenID: 1, + Nonce: 2, + AbsoluteFee: 3, }, { - FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - ToEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"), - TokenID: 1, - Nonce: 3, - UserFeeAbsolute: 5, + ToIdx: common.Idx(1), + FromIdx: common.Idx(0), + TokenID: 1, + Nonce: 3, + AbsoluteFee: 5, }, { // this tx will not be selected, as the ToEthAddr does not have an account - FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"), - ToEthAddr: ethCommon.HexToAddress("0x4a2CFDF534725D8D6e07Af97B237Fff19BDb3c93"), - TokenID: 1, - Nonce: 4, - UserFeeAbsolute: 5, + FromIdx: common.Idx(1), + ToIdx: common.Idx(2), + TokenID: 1, + Nonce: 4, + AbsoluteFee: 5, }, }

@@ -81,13 +80,13 @@ for i := 0; i < len(txs); i++ {

// for i := 0; i < nBatch; i++ { // for j := 0; j < len(txs)/nBatch; j++ { // store tx - m.AddTx(uint64(nBatch), txs[i]) + l2db.AddTx(uint64(nBatch), txs[i]) // store account if not yet accountID := getAccountID(txs[i].FromEthAddr, txs[i].TokenID) if _, ok := m.AccountDB[accountID]; !ok { account := common.Account{ - EthAddr: txs[i].FromEthAddr, + // EthAddr: txs[i].FromEthAddr, TokenID: txs[i].TokenID, Nonce: 0, Balance: big.NewInt(0),

@@ -100,18 +99,27 @@ }

return m } +*/ func TestGetL2TxSelection(t *testing.T) { - mockDB := initMockDB() - txsel := NewTxSelector(mockDB, 3, 3, 3) + dir, err := ioutil.TempDir("", "tmpdb") + require.Nil(t, err) + sdb, err := statedb.NewStateDB(dir, false, false, 0) + assert.Nil(t, err) + testL2DB := &l2db.L2DB{} + // initTestDB(testL2DB, sdb) - txs, err := txsel.GetL2TxSelection(0) + txsel, err := NewTxSelector(sdb, testL2DB, 3, 3, 3) assert.Nil(t, err) - for _, tx := range txs { - fmt.Println(tx.FromEthAddr.String(), tx.ToEthAddr.String(), tx.UserFeeAbsolute) - } - assert.Equal(t, 3, len(txs)) - assert.Equal(t, uint64(6), txs[0].UserFeeAbsolute) - assert.Equal(t, uint64(5), txs[1].UserFeeAbsolute) - assert.Equal(t, uint64(4), txs[2].UserFeeAbsolute) + fmt.Println(txsel) + + // txs, err := txsel.GetL2TxSelection(0) + // assert.Nil(t, err) + // for _, tx := range txs { + // fmt.Println(tx.FromIdx, tx.ToIdx, tx.AbsoluteFee) + // } + // assert.Equal(t, 3, len(txs)) + // assert.Equal(t, uint64(6), txs[0].AbsoluteFee) + // assert.Equal(t, uint64(5), txs[1].AbsoluteFee) + // assert.Equal(t, uint64(4), txs[2].AbsoluteFee) }
D txselector/utils.go

@@ -1,16 +0,0 @@

-package txselector - -import ( - "encoding/binary" - - ethCommon "github.com/ethereum/go-ethereum/common" -) - -func getAccountID(addr ethCommon.Address, tokenID uint32) [36]byte { - var tokenIDBytes [4]byte - binary.LittleEndian.PutUint32(tokenIDBytes[:], tokenID) - accountIDBytes := append(addr[:], tokenIDBytes[:]...) - var accountID [36]byte - copy(accountID[:], accountIDBytes[:36]) - return accountID -}