Browse Source

Merge pull request #50 from hermeznetwork/feature/txselector-int-statedb

Integrate TxSelector with StateDB
feature/sql-semaphore1
a_bennassar 4 years ago
committed by GitHub
parent
commit
9df30affc6
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 175 additions and 219 deletions
  1. +11
    -0
      README.md
  2. +2
    -2
      batchbuilder/batchbuilder_test.go
  3. +1
    -0
      common/pooll2tx.go
  4. +7
    -1
      db/statedb/statedb.go
  5. +0
    -40
      txselector/common/tmp.go
  6. +0
    -33
      txselector/mock/mock.go
  7. +87
    -68
      txselector/txselector.go
  8. +67
    -59
      txselector/txselector_test.go
  9. +0
    -16
      txselector/utils.go

+ 11
- 0
README.md

@ -1,3 +1,14 @@
# hermez-node [![Go Report Card](https://goreportcard.com/badge/github.com/hermeznetwork/hermez-node)](https://goreportcard.com/report/github.com/hermeznetwork/hermez-node) [![Test Status](https://github.com/hermeznetwork/hermez-node/workflows/Test/badge.svg)](https://github.com/hermeznetwork/hermez-node/actions?query=workflow%3ATest) [![Lint Status](https://github.com/hermeznetwork/hermez-node/workflows/Lint/badge.svg)](https://github.com/hermeznetwork/hermez-node/actions?query=workflow%3ALint) [![GoDoc](https://godoc.org/github.com/hermeznetwork/hermez-node?status.svg)](https://godoc.org/github.com/hermeznetwork/hermez-node)
Go implementation of the Hermez node.
## Test
- First run a docker instance of the PostgresSQL (where `yourpasswordhere` should be your password)
```
POSTGRES_PASS=yourpasswordhere; sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=history -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$POSTGRES_PASS" -d postgres && sleep 2s && sudo docker exec hermez-db-test psql -a history -U hermez -c "CREATE DATABASE l2;"
```
- Then, run the tests with the password as env var
```
POSTGRES_PASS=yourpasswordhere go test ./...
```

+ 2
- 2
batchbuilder/batchbuilder_test.go

@ -14,10 +14,10 @@ func TestBatchBuilder(t *testing.T) {
dir, err := ioutil.TempDir("", "tmpdb")
require.Nil(t, err)
sdb, err := statedb.NewStateDB(dir, false, false, 0)
synchDB, err := statedb.NewStateDB(dir, false, false, 0)
assert.Nil(t, err)
bb, err := NewBatchBuilder(sdb, nil, 0, 0, 32)
bb, err := NewBatchBuilder(synchDB, nil, 0, 0, 32)
assert.Nil(t, err)
fmt.Println(bb)
}

+ 1
- 0
common/pooll2tx.go

@ -18,6 +18,7 @@ type PoolL2Tx struct {
Timestamp time.Time // time when added to the tx pool
Signature babyjub.Signature // tx signature
ToEthAddr eth.Address
AbsoluteFee float64 // TODO add methods to calculate this value from Tx.Fee tables + priceupdater tables
}
// RqTx Transaction Data used to indicate that a transaction depends on another transaction

+ 7
- 1
db/statedb/statedb.go

@ -210,6 +210,12 @@ func NewLocalStateDB(synchronizerDB *StateDB, withMT bool, nLevels int) (*LocalS
// LocalStateDB.synchronizerStateDB for the given batchNum
func (l *LocalStateDB) Reset(batchNum int, fromSynchronizer bool) error {
// TODO
// if fromSynchronizer==true:
// make copy from l.synchronizerStateDB at the batchNum to the localStateDB
// if synchronizerStateDB does not have batchNum, return err
// the localStateDB checkpoint is set to batchNum
// else fromSynchronizer==false:
// the localStateDB checkpoint is set to batchNum
// if localStateDB does not have batchNum, return err
return nil
}

+ 0
- 40
txselector/common/tmp.go

@ -1,40 +0,0 @@
package common
import (
"math/big"
ethCommon "github.com/ethereum/go-ethereum/common"
)
// WIP this will be from hermeznetwork/common
type Tx struct {
FromAx [32]byte
FromAy [32]byte
FromEthAddr ethCommon.Address
ToAx [32]byte
ToAy [32]byte
ToEthAddr ethCommon.Address
OnChain bool
RqOffset []byte
NewAccount bool
TokenID uint32
LoadAmount [3]byte
Amount [3]byte
Nonce uint64
UserFee uint8
UserFeeAbsolute uint64
R8x [32]byte
R8y [32]byte
S [32]byte
RqTxData [32]byte
}
// WIP this will be from hermeznetwork/common
type Account struct {
EthAddr ethCommon.Address
TokenID uint32
Idx uint32
Nonce uint64
Balance *big.Int
// Ax, Ay
}

+ 0
- 33
txselector/mock/mock.go

@ -1,33 +0,0 @@
package mock
import (
"github.com/hermeznetwork/hermez-node/txselector/common"
)
type MockDB struct {
Txs map[uint64][]common.Tx
// AccountDB is the LocalAccountDB copy of the original AccountDB
AccountDB map[[36]byte]common.Account // [36]byte is tx.ToEthAddr + tx.TokenID
PendingRegistersDB map[[36]byte]common.Account // [36]byte is tx.ToEthAddr + tx.TokenID
}
func New() *MockDB {
return &MockDB{
Txs: make(map[uint64][]common.Tx),
AccountDB: make(map[[36]byte]common.Account),
PendingRegistersDB: make(map[[36]byte]common.Account),
}
}
func (m *MockDB) AddTx(batchID uint64, tx common.Tx) {
if _, ok := m.Txs[batchID]; !ok {
m.Txs[batchID] = []common.Tx{}
}
m.Txs[batchID] = append(m.Txs[batchID], tx)
}
func (m *MockDB) GetTxs(batchID uint64) []common.Tx {
return m.Txs[batchID]
}

+ 87
- 68
txselector/txselector.go

@ -3,12 +3,13 @@ package txselector
import (
"sort"
"github.com/hermeznetwork/hermez-node/txselector/common"
"github.com/hermeznetwork/hermez-node/txselector/mock"
"github.com/hermeznetwork/hermez-node/common"
"github.com/hermeznetwork/hermez-node/db/l2db"
"github.com/hermeznetwork/hermez-node/db/statedb"
)
// txs implements the interface Sort for an array of Tx
type txs []common.Tx
type txs []common.PoolL2Tx
func (t txs) Len() int {
return len(t)
@ -17,57 +18,68 @@ func (t txs) Swap(i, j int) {
t[i], t[j] = t[j], t[i]
}
func (t txs) Less(i, j int) bool {
return t[i].UserFeeAbsolute > t[j].UserFeeAbsolute
return t[i].AbsoluteFee > t[j].AbsoluteFee
}
// TxSelector implements all the functionalities to select the txs for the next batch
type TxSelector struct {
// NMax is the maximum L1-user-tx for a batch
NMax uint64
// MMax is the maximum L1-operator-tx for a batch
MMax uint64
// PMax is the maximum L2-tx for a batch
PMax uint64
// DB is a pointer to the database interface
DB *mock.MockDB
// MaxL1UserTxs is the maximum L1-user-tx for a batch
MaxL1UserTxs uint64
// MaxL1OperatorTxs is the maximum L1-operator-tx for a batch
MaxL1OperatorTxs uint64
// MaxTxs is the maximum txs for a batch
MaxTxs uint64
l2db *l2db.L2DB
localAccountsDB *statedb.LocalStateDB
}
func NewTxSelector(db *mock.MockDB, nMax, mMax, pMax uint64) *TxSelector {
return &TxSelector{
NMax: nMax,
MMax: mMax,
PMax: pMax,
DB: db,
// NewTxSelector returns a *TxSelector
func NewTxSelector(synchronizerStateDB *statedb.StateDB, l2 *l2db.L2DB, maxL1UserTxs, maxL1OperatorTxs, maxTxs uint64) (*TxSelector, error) {
localAccountsDB, err := statedb.NewLocalStateDB(synchronizerStateDB, false, 0) // without merkletree
if err != nil {
return nil, err
}
}
func (txsel *TxSelector) updateLocalAccountDB(batchId uint64) error {
// if batchID > max(localAccountDB.BatchID) + 1
// make a checkpoint of AccountDB at BatchID to a localAccountDB
// use localAccountDB[inputBatchID-1]
return &TxSelector{
MaxL1UserTxs: maxL1UserTxs,
MaxL1OperatorTxs: maxL1OperatorTxs,
MaxTxs: maxTxs,
l2db: l2,
localAccountsDB: localAccountsDB,
}, nil
}
// Reset tells the TxSelector to get it's internal AccountsDB
// from the required `batchNum`
func (txsel *TxSelector) Reset(batchNum int) error {
err := txsel.localAccountsDB.Reset(batchNum, true)
if err != nil {
return err
}
return nil
}
func (txsel *TxSelector) GetL2TxSelection(batchID uint64) ([]common.Tx, error) {
err := txsel.updateLocalAccountDB(batchID)
// GetL2TxSelection returns a selection of the L2Txs for the next batch, from the L2DB pool
func (txsel *TxSelector) GetL2TxSelection(batchNum int) ([]common.PoolL2Tx, error) {
// get pending l2-tx from tx-pool
l2TxsRaw, err := txsel.l2db.GetPendingTxs() // once l2db ready, maybe use parameter 'batchNum'
if err != nil {
return nil, err
}
// get pending l2-tx from tx-pool
txsRaw := txsel.DB.GetTxs(batchID)
// discard the txs that don't have an Account in the AccountDB
var validTxs txs
for _, tx := range txsRaw {
accountID := getAccountID(tx.ToEthAddr, tx.TokenID)
if _, ok := txsel.DB.AccountDB[accountID]; ok {
for _, tx := range l2TxsRaw {
_, err = txsel.localAccountsDB.GetAccount(tx.FromIdx)
if err == nil {
// if FromIdx has an account into the AccountsDB
validTxs = append(validTxs, tx)
}
}
// get most profitable L2-tx (len<NMax)
txs := txsel.getL2Profitable(validTxs)
// get most profitable L2-tx
txs := txsel.getL2Profitable(validTxs, txsel.MaxTxs)
// apply L2-tx to local AccountDB, make checkpoint tagged with BatchID
// update balances
@ -77,66 +89,73 @@ func (txsel *TxSelector) GetL2TxSelection(batchID uint64) ([]common.Tx, error) {
return txs, nil
}
func (txsel *TxSelector) GetL1L2TxSelection(batchID uint64, l1txs []common.Tx) ([]common.Tx, []common.Tx, []common.Tx, error) {
err := txsel.updateLocalAccountDB(batchID)
if err != nil {
return nil, nil, nil, err
}
// GetL1L2TxSelection returns the selection of L1 + L2 txs
func (txsel *TxSelector) GetL1L2TxSelection(batchNum int, l1txs []common.Tx) ([]common.Tx, []common.PoolL2Tx, []common.Tx, error) {
// apply l1-user-tx to localAccountDB
// create new leaves
// update balances
// update nonces
// get pending l2-tx from tx-pool
txsRaw := txsel.DB.GetTxs(batchID)
l2TxsRaw, err := txsel.l2db.GetPendingTxs() // (batchID)
if err != nil {
return nil, nil, nil, err
}
// discard the txs that don't have an Account in the AccountDB
// neither appear in the PendingRegistersDB
// neither appear in the AccountCreationAuthsDB
var validTxs txs
for _, tx := range txsRaw {
accountID := getAccountID(tx.ToEthAddr, tx.TokenID)
exist := txsel.checkIfAccountExist(accountID)
if exist {
for _, tx := range l2TxsRaw {
if txsel.checkIfAccountExistOrPending(tx.FromIdx) {
// if FromIdx has an account into the AccountsDB
validTxs = append(validTxs, tx)
}
}
// get most profitable L2-tx (len<NMax)
l2txs := txsel.getL2Profitable(validTxs)
// prepare (from the selected l2txs) pending to register from the PendingRegistersDB
var pendingRegisters []common.Account
for _, tx := range l2txs {
accountID := getAccountID(tx.ToEthAddr, tx.TokenID)
if toRegister, ok := txsel.DB.PendingRegistersDB[accountID]; ok {
pendingRegisters = append(pendingRegisters, toRegister)
}
}
// prepare (from the selected l2txs) pending to create from the AccountCreationAuthsDB
var accountCreationAuths []common.Account
// TODO once DB ready:
// if tx.ToIdx is in AccountCreationAuthsDB, take it and add it to
// the array 'accountCreationAuths'
// for _, tx := range l2txs {
// account, err := txsel.localAccountsDB.GetAccount(tx.ToIdx)
// if err != nil {
// return nil, nil, nil, err
// }
// if accountToCreate, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok {
// accountCreationAuths = append(accountCreationAuths, accountToCreate)
// }
// }
// create L1-operator-tx for each L2-tx selected in which the recipient does not have account
l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(pendingRegisters) // only with the L2-tx selected ones
l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(accountCreationAuths) // only with the L2-tx selected ones
// get most profitable L2-tx
maxL2Txs := txsel.MaxTxs - uint64(len(l1OperatorTxs)) // - len(l1UserTxs)
l2txs := txsel.getL2Profitable(validTxs, maxL2Txs)
return l1txs, l2txs, l1OperatorTxs, nil
}
func (txsel *TxSelector) checkIfAccountExist(accountID [36]byte) bool {
func (txsel *TxSelector) checkIfAccountExistOrPending(idx common.Idx) bool {
// check if account exist in AccountDB
if _, ok := txsel.DB.AccountDB[accountID]; ok {
return true
}
// check if account is pending to register
if _, ok := txsel.DB.PendingRegistersDB[accountID]; ok {
return true
_, err := txsel.localAccountsDB.GetAccount(idx)
if err != nil {
return false
}
// check if account is pending to create
// TODO need a method in the DB to get the PendingRegisters
// if _, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok {
// return true
// }
return false
}
func (txsel *TxSelector) getL2Profitable(txs txs) txs {
func (txsel *TxSelector) getL2Profitable(txs txs, max uint64) txs {
sort.Sort(txs)
return txs[:txsel.PMax]
return txs[:max]
}
func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []common.Account) txs {
func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []common.Account) []common.Tx {
//
return txs{}
return nil
}

+ 67
- 59
txselector/txselector_test.go

@ -2,76 +2,75 @@ package txselector
import (
"fmt"
"math/big"
"io/ioutil"
"testing"
"github.com/hermeznetwork/hermez-node/txselector/common"
"github.com/hermeznetwork/hermez-node/txselector/mock"
"github.com/hermeznetwork/hermez-node/db/l2db"
"github.com/hermeznetwork/hermez-node/db/statedb"
ethCommon "github.com/ethereum/go-ethereum/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func initMockDB() *mock.MockDB {
m := mock.New()
/*
func initTestDB(l2 *l2db.L2DB, sdb *statedb.StateDB) *mock.MockDB {
txs := []common.Tx{
{
FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
TokenID: 1,
Nonce: 1,
UserFeeAbsolute: 1,
FromIdx: common.Idx(0),
ToIdx: common.Idx(1),
TokenID: 1,
Nonce: 1,
AbsoluteFee: 1,
},
{
FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
TokenID: 1,
Nonce: 2,
UserFeeAbsolute: 3,
FromIdx: common.Idx(0),
ToIdx: common.Idx(1),
TokenID: 1,
Nonce: 2,
AbsoluteFee: 3,
},
{
FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
TokenID: 1,
Nonce: 4,
UserFeeAbsolute: 6,
FromIdx: common.Idx(0),
ToIdx: common.Idx(1),
TokenID: 1,
Nonce: 4,
AbsoluteFee: 6,
},
{
FromEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
ToEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
TokenID: 1,
Nonce: 4,
UserFeeAbsolute: 4,
FromIdx: common.Idx(0),
ToIdx: common.Idx(1),
TokenID: 1,
Nonce: 4,
AbsoluteFee: 4,
},
{
FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
ToEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
TokenID: 1,
Nonce: 1,
UserFeeAbsolute: 4,
ToIdx: common.Idx(1),
FromIdx: common.Idx(0),
TokenID: 1,
Nonce: 1,
AbsoluteFee: 4,
},
{
FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
ToEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
TokenID: 1,
Nonce: 2,
UserFeeAbsolute: 3,
ToIdx: common.Idx(1),
FromIdx: common.Idx(0),
TokenID: 1,
Nonce: 2,
AbsoluteFee: 3,
},
{
FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
ToEthAddr: ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
TokenID: 1,
Nonce: 3,
UserFeeAbsolute: 5,
ToIdx: common.Idx(1),
FromIdx: common.Idx(0),
TokenID: 1,
Nonce: 3,
AbsoluteFee: 5,
},
{
// this tx will not be selected, as the ToEthAddr does not have an account
FromEthAddr: ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
ToEthAddr: ethCommon.HexToAddress("0x4a2CFDF534725D8D6e07Af97B237Fff19BDb3c93"),
TokenID: 1,
Nonce: 4,
UserFeeAbsolute: 5,
FromIdx: common.Idx(1),
ToIdx: common.Idx(2),
TokenID: 1,
Nonce: 4,
AbsoluteFee: 5,
},
}
@ -81,13 +80,13 @@ func initMockDB() *mock.MockDB {
// for i := 0; i < nBatch; i++ {
// for j := 0; j < len(txs)/nBatch; j++ {
// store tx
m.AddTx(uint64(nBatch), txs[i])
l2db.AddTx(uint64(nBatch), txs[i])
// store account if not yet
accountID := getAccountID(txs[i].FromEthAddr, txs[i].TokenID)
if _, ok := m.AccountDB[accountID]; !ok {
account := common.Account{
EthAddr: txs[i].FromEthAddr,
// EthAddr: txs[i].FromEthAddr,
TokenID: txs[i].TokenID,
Nonce: 0,
Balance: big.NewInt(0),
@ -100,18 +99,27 @@ func initMockDB() *mock.MockDB {
return m
}
*/
func TestGetL2TxSelection(t *testing.T) {
mockDB := initMockDB()
txsel := NewTxSelector(mockDB, 3, 3, 3)
dir, err := ioutil.TempDir("", "tmpdb")
require.Nil(t, err)
sdb, err := statedb.NewStateDB(dir, false, false, 0)
assert.Nil(t, err)
testL2DB := &l2db.L2DB{}
// initTestDB(testL2DB, sdb)
txs, err := txsel.GetL2TxSelection(0)
txsel, err := NewTxSelector(sdb, testL2DB, 3, 3, 3)
assert.Nil(t, err)
for _, tx := range txs {
fmt.Println(tx.FromEthAddr.String(), tx.ToEthAddr.String(), tx.UserFeeAbsolute)
}
assert.Equal(t, 3, len(txs))
assert.Equal(t, uint64(6), txs[0].UserFeeAbsolute)
assert.Equal(t, uint64(5), txs[1].UserFeeAbsolute)
assert.Equal(t, uint64(4), txs[2].UserFeeAbsolute)
fmt.Println(txsel)
// txs, err := txsel.GetL2TxSelection(0)
// assert.Nil(t, err)
// for _, tx := range txs {
// fmt.Println(tx.FromIdx, tx.ToIdx, tx.AbsoluteFee)
// }
// assert.Equal(t, 3, len(txs))
// assert.Equal(t, uint64(6), txs[0].AbsoluteFee)
// assert.Equal(t, uint64(5), txs[1].AbsoluteFee)
// assert.Equal(t, uint64(4), txs[2].AbsoluteFee)
}

+ 0
- 16
txselector/utils.go

@ -1,16 +0,0 @@
package txselector
import (
"encoding/binary"
ethCommon "github.com/ethereum/go-ethereum/common"
)
func getAccountID(addr ethCommon.Address, tokenID uint32) [36]byte {
var tokenIDBytes [4]byte
binary.LittleEndian.PutUint32(tokenIDBytes[:], tokenID)
accountIDBytes := append(addr[:], tokenIDBytes[:]...)
var accountID [36]byte
copy(accountID[:], accountIDBytes[:36])
return accountID
}

Loading…
Cancel
Save