Merge pull request #50 from hermeznetwork/feature/txselector-int-statedb

Integrate TxSelector with StateDB

README.md

@@ -1,3 +1,14 @@
 # hermez-node [![Go Report Card](https://goreportcard.com/badge/github.com/hermeznetwork/hermez-node)](https://goreportcard.com/report/github.com/hermeznetwork/hermez-node) [![Test Status](https://github.com/hermeznetwork/hermez-node/workflows/Test/badge.svg)](https://github.com/hermeznetwork/hermez-node/actions?query=workflow%3ATest) [![Lint Status](https://github.com/hermeznetwork/hermez-node/workflows/Lint/badge.svg)](https://github.com/hermeznetwork/hermez-node/actions?query=workflow%3ALint) [![GoDoc](https://godoc.org/github.com/hermeznetwork/hermez-node?status.svg)](https://godoc.org/github.com/hermeznetwork/hermez-node)
 
 Go implementation of the Hermez node.
+
+
+## Test
+- First run a docker instance of the PostgresSQL (where `yourpasswordhere` should be your password)
+```
+POSTGRES_PASS=yourpasswordhere; sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=history -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$POSTGRES_PASS" -d postgres && sleep 2s && sudo docker exec hermez-db-test psql -a history -U hermez -c "CREATE DATABASE l2;"
+```
+- Then, run the tests with the password as env var
+```
+POSTGRES_PASS=yourpasswordhere go test ./...
+```

batchbuilder/batchbuilder_test.go

@@ -14,10 +14,10 @@ func TestBatchBuilder(t *testing.T) {
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
 
-	sdb, err := statedb.NewStateDB(dir, false, false, 0)
+	synchDB, err := statedb.NewStateDB(dir, false, false, 0)
 	assert.Nil(t, err)
 
-	bb, err := NewBatchBuilder(sdb, nil, 0, 0, 32)
+	bb, err := NewBatchBuilder(synchDB, nil, 0, 0, 32)
 	assert.Nil(t, err)
 	fmt.Println(bb)
 }

common/pooll2tx.go

@@ -18,6 +18,7 @@ type PoolL2Tx struct {
 	Timestamp          time.Time         // time when added to the tx pool
 	Signature          babyjub.Signature // tx signature
 	ToEthAddr          eth.Address
+	AbsoluteFee        float64 // TODO add methods to calculate this value from Tx.Fee tables + priceupdater tables
 }
 
 // RqTx Transaction Data used to indicate that a transaction depends on another transaction

db/statedb/statedb.go

@@ -210,6 +210,12 @@ func NewLocalStateDB(synchronizerDB *StateDB, withMT bool, nLevels int) (*LocalS
 // LocalStateDB.synchronizerStateDB for the given batchNum
 func (l *LocalStateDB) Reset(batchNum int, fromSynchronizer bool) error {
 	// TODO
-
+	// if fromSynchronizer==true:
+	// 	make copy from l.synchronizerStateDB at the batchNum to the localStateDB
+	// 		if synchronizerStateDB does not have batchNum, return err
+	// 	the localStateDB checkpoint is set to batchNum
+	// else fromSynchronizer==false:
+	// 	the localStateDB checkpoint is set to batchNum
+	// 		if localStateDB does not have batchNum, return err
 	return nil
 }

txselector/common/tmp.go

@@ -1,40 +0,0 @@
-package common
-
-import (
-	"math/big"
-
-	ethCommon "github.com/ethereum/go-ethereum/common"
-)
-
-// WIP this will be from hermeznetwork/common
-type Tx struct {
-	FromAx          [32]byte
-	FromAy          [32]byte
-	FromEthAddr     ethCommon.Address
-	ToAx            [32]byte
-	ToAy            [32]byte
-	ToEthAddr       ethCommon.Address
-	OnChain         bool
-	RqOffset        []byte
-	NewAccount      bool
-	TokenID         uint32
-	LoadAmount      [3]byte
-	Amount          [3]byte
-	Nonce           uint64
-	UserFee         uint8
-	UserFeeAbsolute uint64
-	R8x             [32]byte
-	R8y             [32]byte
-	S               [32]byte
-	RqTxData        [32]byte
-}
-
-// WIP this will be from hermeznetwork/common
-type Account struct {
-	EthAddr ethCommon.Address
-	TokenID uint32
-	Idx     uint32
-	Nonce   uint64
-	Balance *big.Int
-	// Ax, Ay
-}

txselector/mock/mock.go

@@ -1,33 +0,0 @@
-package mock
-
-import (
-	"github.com/hermeznetwork/hermez-node/txselector/common"
-)
-
-type MockDB struct {
-	Txs map[uint64][]common.Tx
-
-	// AccountDB is the LocalAccountDB copy of the original AccountDB
-	AccountDB map[[36]byte]common.Account // [36]byte is tx.ToEthAddr + tx.TokenID
-
-	PendingRegistersDB map[[36]byte]common.Account // [36]byte is tx.ToEthAddr + tx.TokenID
-}
-
-func New() *MockDB {
-	return &MockDB{
-		Txs:                make(map[uint64][]common.Tx),
-		AccountDB:          make(map[[36]byte]common.Account),
-		PendingRegistersDB: make(map[[36]byte]common.Account),
-	}
-}
-
-func (m *MockDB) AddTx(batchID uint64, tx common.Tx) {
-	if _, ok := m.Txs[batchID]; !ok {
-		m.Txs[batchID] = []common.Tx{}
-	}
-	m.Txs[batchID] = append(m.Txs[batchID], tx)
-}
-
-func (m *MockDB) GetTxs(batchID uint64) []common.Tx {
-	return m.Txs[batchID]
-}

txselector/txselector.go

@@ -3,12 +3,13 @@ package txselector
 import (
 	"sort"
 
-	"github.com/hermeznetwork/hermez-node/txselector/common"
-	"github.com/hermeznetwork/hermez-node/txselector/mock"
+	"github.com/hermeznetwork/hermez-node/common"
+	"github.com/hermeznetwork/hermez-node/db/l2db"
+	"github.com/hermeznetwork/hermez-node/db/statedb"
 )
 
 // txs implements the interface Sort for an array of Tx
-type txs []common.Tx
+type txs []common.PoolL2Tx
 
 func (t txs) Len() int {
 	return len(t)
@@ -17,57 +18,68 @@ func (t txs) Swap(i, j int) {
 	t[i], t[j] = t[j], t[i]
 }
 func (t txs) Less(i, j int) bool {
-	return t[i].UserFeeAbsolute > t[j].UserFeeAbsolute
+	return t[i].AbsoluteFee > t[j].AbsoluteFee
 }
 
+// TxSelector implements all the functionalities to select the txs for the next batch
 type TxSelector struct {
-	// NMax is the maximum L1-user-tx for a batch
-	NMax uint64
-	// MMax is the maximum L1-operator-tx for a batch
-	MMax uint64
-	// PMax is the maximum L2-tx for a batch
-	PMax uint64
-	// DB is a pointer to the database interface
-	DB *mock.MockDB
+	// MaxL1UserTxs is the maximum L1-user-tx for a batch
+	MaxL1UserTxs uint64
+	// MaxL1OperatorTxs is the maximum L1-operator-tx for a batch
+	MaxL1OperatorTxs uint64
+	// MaxTxs is the maximum txs for a batch
+	MaxTxs uint64
+
+	l2db            *l2db.L2DB
+	localAccountsDB *statedb.LocalStateDB
 }
 
-func NewTxSelector(db *mock.MockDB, nMax, mMax, pMax uint64) *TxSelector {
-	return &TxSelector{
-		NMax: nMax,
-		MMax: mMax,
-		PMax: pMax,
-		DB:   db,
+// NewTxSelector returns a *TxSelector
+func NewTxSelector(synchronizerStateDB *statedb.StateDB, l2 *l2db.L2DB, maxL1UserTxs, maxL1OperatorTxs, maxTxs uint64) (*TxSelector, error) {
+	localAccountsDB, err := statedb.NewLocalStateDB(synchronizerStateDB, false, 0) // without merkletree
+	if err != nil {
+		return nil, err
 	}
-}
 
-func (txsel *TxSelector) updateLocalAccountDB(batchId uint64) error {
-	// if batchID > max(localAccountDB.BatchID) + 1
-	//     make a checkpoint of AccountDB at BatchID to a localAccountDB
-	// use localAccountDB[inputBatchID-1]
+	return &TxSelector{
+		MaxL1UserTxs:     maxL1UserTxs,
+		MaxL1OperatorTxs: maxL1OperatorTxs,
+		MaxTxs:           maxTxs,
+		l2db:             l2,
+		localAccountsDB:  localAccountsDB,
+	}, nil
+}
 
+// Reset tells the TxSelector to get it's internal AccountsDB
+// from the required `batchNum`
+func (txsel *TxSelector) Reset(batchNum int) error {
+	err := txsel.localAccountsDB.Reset(batchNum, true)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
-func (txsel *TxSelector) GetL2TxSelection(batchID uint64) ([]common.Tx, error) {
-	err := txsel.updateLocalAccountDB(batchID)
+// GetL2TxSelection returns a selection of the L2Txs for the next batch, from the L2DB pool
+func (txsel *TxSelector) GetL2TxSelection(batchNum int) ([]common.PoolL2Tx, error) {
+	// get pending l2-tx from tx-pool
+	l2TxsRaw, err := txsel.l2db.GetPendingTxs() // once l2db ready, maybe use parameter 'batchNum'
 	if err != nil {
 		return nil, err
 	}
 
-	// get pending l2-tx from tx-pool
-	txsRaw := txsel.DB.GetTxs(batchID)
-
 	// discard the txs that don't have an Account in the AccountDB
 	var validTxs txs
-	for _, tx := range txsRaw {
-		accountID := getAccountID(tx.ToEthAddr, tx.TokenID)
-		if _, ok := txsel.DB.AccountDB[accountID]; ok {
+	for _, tx := range l2TxsRaw {
+		_, err = txsel.localAccountsDB.GetAccount(tx.FromIdx)
+		if err == nil {
+			// if FromIdx has an account into the AccountsDB
 			validTxs = append(validTxs, tx)
 		}
 	}
 
-	// get most profitable L2-tx (len<NMax)
-	txs := txsel.getL2Profitable(validTxs)
+	// get most profitable L2-tx
+	txs := txsel.getL2Profitable(validTxs, txsel.MaxTxs)
 
 	// apply L2-tx to local AccountDB, make checkpoint tagged with BatchID
 	//     update balances
@@ -77,66 +89,73 @@ func (txsel *TxSelector) GetL2TxSelection(batchID uint64) ([]common.Tx, error) {
 	return txs, nil
 }
 
-func (txsel *TxSelector) GetL1L2TxSelection(batchID uint64, l1txs []common.Tx) ([]common.Tx, []common.Tx, []common.Tx, error) {
-	err := txsel.updateLocalAccountDB(batchID)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
+// GetL1L2TxSelection returns the selection of L1 + L2 txs
+func (txsel *TxSelector) GetL1L2TxSelection(batchNum int, l1txs []common.Tx) ([]common.Tx, []common.PoolL2Tx, []common.Tx, error) {
 	// apply l1-user-tx to localAccountDB
 	//     create new leaves
 	//     update balances
 	//     update nonces
 
 	// get pending l2-tx from tx-pool
-	txsRaw := txsel.DB.GetTxs(batchID)
+	l2TxsRaw, err := txsel.l2db.GetPendingTxs() // (batchID)
+	if err != nil {
+		return nil, nil, nil, err
+	}
 
 	// discard the txs that don't have an Account in the AccountDB
-	// neither appear in the PendingRegistersDB
+	// neither appear in the AccountCreationAuthsDB
 	var validTxs txs
-	for _, tx := range txsRaw {
-		accountID := getAccountID(tx.ToEthAddr, tx.TokenID)
-		exist := txsel.checkIfAccountExist(accountID)
-		if exist {
+	for _, tx := range l2TxsRaw {
+		if txsel.checkIfAccountExistOrPending(tx.FromIdx) {
+			// if FromIdx has an account into the AccountsDB
 			validTxs = append(validTxs, tx)
 		}
 	}
 
-	// get most profitable L2-tx (len<NMax)
-	l2txs := txsel.getL2Profitable(validTxs)
-
-	// prepare (from the selected l2txs) pending to register from the PendingRegistersDB
-	var pendingRegisters []common.Account
-	for _, tx := range l2txs {
-		accountID := getAccountID(tx.ToEthAddr, tx.TokenID)
-		if toRegister, ok := txsel.DB.PendingRegistersDB[accountID]; ok {
-			pendingRegisters = append(pendingRegisters, toRegister)
-		}
-	}
+	// prepare (from the selected l2txs) pending to create from the AccountCreationAuthsDB
+	var accountCreationAuths []common.Account
+	// TODO once DB ready:
+	// if tx.ToIdx is in AccountCreationAuthsDB, take it and add it to
+	// the array 'accountCreationAuths'
+	// for _, tx := range l2txs {
+	//         account, err := txsel.localAccountsDB.GetAccount(tx.ToIdx)
+	//         if err != nil {
+	//                 return nil, nil, nil, err
+	//         }
+	//         if accountToCreate, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok {
+	//                 accountCreationAuths = append(accountCreationAuths, accountToCreate)
+	//         }
+	// }
 
 	// create L1-operator-tx for each L2-tx selected in which the recipient does not have account
-	l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(pendingRegisters) // only with the L2-tx selected ones
+	l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(accountCreationAuths) // only with the L2-tx selected ones
+
+	// get most profitable L2-tx
+	maxL2Txs := txsel.MaxTxs - uint64(len(l1OperatorTxs)) // - len(l1UserTxs)
+	l2txs := txsel.getL2Profitable(validTxs, maxL2Txs)
 
 	return l1txs, l2txs, l1OperatorTxs, nil
 }
 
-func (txsel *TxSelector) checkIfAccountExist(accountID [36]byte) bool {
+func (txsel *TxSelector) checkIfAccountExistOrPending(idx common.Idx) bool {
 	// check if account exist in AccountDB
-	if _, ok := txsel.DB.AccountDB[accountID]; ok {
-		return true
-	}
-	// check if account is pending to register
-	if _, ok := txsel.DB.PendingRegistersDB[accountID]; ok {
-		return true
+	_, err := txsel.localAccountsDB.GetAccount(idx)
+	if err != nil {
+		return false
 	}
+	// check if account is pending to create
+	// TODO need a method in the DB to get the PendingRegisters
+	// if _, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok {
+	//         return true
+	// }
 	return false
 }
 
-func (txsel *TxSelector) getL2Profitable(txs txs) txs {
+func (txsel *TxSelector) getL2Profitable(txs txs, max uint64) txs {
 	sort.Sort(txs)
-	return txs[:txsel.PMax]
+	return txs[:max]
 }
-func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []common.Account) txs {
+func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []common.Account) []common.Tx {
 	//
-	return txs{}
+	return nil
 }

txselector/txselector_test.go

@@ -2,76 +2,75 @@ package txselector
 
 import (
 	"fmt"
-	"math/big"
+	"io/ioutil"
 	"testing"
 
-	"github.com/hermeznetwork/hermez-node/txselector/common"
-	"github.com/hermeznetwork/hermez-node/txselector/mock"
+	"github.com/hermeznetwork/hermez-node/db/l2db"
+	"github.com/hermeznetwork/hermez-node/db/statedb"
 
-	ethCommon "github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func initMockDB() *mock.MockDB {
-	m := mock.New()
-
+/*
+func initTestDB(l2 *l2db.L2DB, sdb *statedb.StateDB) *mock.MockDB {
 	txs := []common.Tx{
 		{
-			FromEthAddr:     ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
-			ToEthAddr:       ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			TokenID:         1,
-			Nonce:           1,
-			UserFeeAbsolute: 1,
+			FromIdx:     common.Idx(0),
+			ToIdx:       common.Idx(1),
+			TokenID:     1,
+			Nonce:       1,
+			AbsoluteFee: 1,
 		},
 		{
-			FromEthAddr:     ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
-			ToEthAddr:       ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			TokenID:         1,
-			Nonce:           2,
-			UserFeeAbsolute: 3,
+			FromIdx:     common.Idx(0),
+			ToIdx:       common.Idx(1),
+			TokenID:     1,
+			Nonce:       2,
+			AbsoluteFee: 3,
 		},
 		{
-			FromEthAddr:     ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
-			ToEthAddr:       ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			TokenID:         1,
-			Nonce:           4,
-			UserFeeAbsolute: 6,
+			FromIdx:     common.Idx(0),
+			ToIdx:       common.Idx(1),
+			TokenID:     1,
+			Nonce:       4,
+			AbsoluteFee: 6,
 		},
 		{
-			FromEthAddr:     ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
-			ToEthAddr:       ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			TokenID:         1,
-			Nonce:           4,
-			UserFeeAbsolute: 4,
+			FromIdx:     common.Idx(0),
+			ToIdx:       common.Idx(1),
+			TokenID:     1,
+			Nonce:       4,
+			AbsoluteFee: 4,
 		},
 		{
-			FromEthAddr:     ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			ToEthAddr:       ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
-			TokenID:         1,
-			Nonce:           1,
-			UserFeeAbsolute: 4,
+			ToIdx:       common.Idx(1),
+			FromIdx:     common.Idx(0),
+			TokenID:     1,
+			Nonce:       1,
+			AbsoluteFee: 4,
 		},
 		{
-			FromEthAddr:     ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			ToEthAddr:       ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
-			TokenID:         1,
-			Nonce:           2,
-			UserFeeAbsolute: 3,
+			ToIdx:       common.Idx(1),
+			FromIdx:     common.Idx(0),
+			TokenID:     1,
+			Nonce:       2,
+			AbsoluteFee: 3,
 		},
 		{
-			FromEthAddr:     ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			ToEthAddr:       ethCommon.HexToAddress("0x859c3d0d5aD917F146fF6654A4C676f1ddeCE26a"),
-			TokenID:         1,
-			Nonce:           3,
-			UserFeeAbsolute: 5,
+			ToIdx:       common.Idx(1),
+			FromIdx:     common.Idx(0),
+			TokenID:     1,
+			Nonce:       3,
+			AbsoluteFee: 5,
 		},
 		{
 			// this tx will not be selected, as the ToEthAddr does not have an account
-			FromEthAddr:     ethCommon.HexToAddress("0x6950E814B82d276DB5Fa7f34253CfeE1387fe03E"),
-			ToEthAddr:       ethCommon.HexToAddress("0x4a2CFDF534725D8D6e07Af97B237Fff19BDb3c93"),
-			TokenID:         1,
-			Nonce:           4,
-			UserFeeAbsolute: 5,
+			FromIdx:     common.Idx(1),
+			ToIdx:       common.Idx(2),
+			TokenID:     1,
+			Nonce:       4,
+			AbsoluteFee: 5,
 		},
 	}
 
@@ -81,13 +80,13 @@ func initMockDB() *mock.MockDB {
 		// for i := 0; i < nBatch; i++ {
 		//         for j := 0; j < len(txs)/nBatch; j++ {
 		// store tx
-		m.AddTx(uint64(nBatch), txs[i])
+		l2db.AddTx(uint64(nBatch), txs[i])
 
 		// store account if not yet
 		accountID := getAccountID(txs[i].FromEthAddr, txs[i].TokenID)
 		if _, ok := m.AccountDB[accountID]; !ok {
 			account := common.Account{
-				EthAddr: txs[i].FromEthAddr,
+				// EthAddr: txs[i].FromEthAddr,
 				TokenID: txs[i].TokenID,
 				Nonce:   0,
 				Balance: big.NewInt(0),
@@ -100,18 +99,27 @@ func initMockDB() *mock.MockDB {
 
 	return m
 }
+*/
 
 func TestGetL2TxSelection(t *testing.T) {
-	mockDB := initMockDB()
-	txsel := NewTxSelector(mockDB, 3, 3, 3)
+	dir, err := ioutil.TempDir("", "tmpdb")
+	require.Nil(t, err)
+	sdb, err := statedb.NewStateDB(dir, false, false, 0)
+	assert.Nil(t, err)
+	testL2DB := &l2db.L2DB{}
+	// initTestDB(testL2DB, sdb)
 
-	txs, err := txsel.GetL2TxSelection(0)
+	txsel, err := NewTxSelector(sdb, testL2DB, 3, 3, 3)
 	assert.Nil(t, err)
-	for _, tx := range txs {
-		fmt.Println(tx.FromEthAddr.String(), tx.ToEthAddr.String(), tx.UserFeeAbsolute)
-	}
-	assert.Equal(t, 3, len(txs))
-	assert.Equal(t, uint64(6), txs[0].UserFeeAbsolute)
-	assert.Equal(t, uint64(5), txs[1].UserFeeAbsolute)
-	assert.Equal(t, uint64(4), txs[2].UserFeeAbsolute)
+	fmt.Println(txsel)
+
+	// txs, err := txsel.GetL2TxSelection(0)
+	// assert.Nil(t, err)
+	// for _, tx := range txs {
+	//         fmt.Println(tx.FromIdx, tx.ToIdx, tx.AbsoluteFee)
+	// }
+	// assert.Equal(t, 3, len(txs))
+	// assert.Equal(t, uint64(6), txs[0].AbsoluteFee)
+	// assert.Equal(t, uint64(5), txs[1].AbsoluteFee)
+	// assert.Equal(t, uint64(4), txs[2].AbsoluteFee)
 }

txselector/utils.go

@@ -1,16 +0,0 @@
-package txselector
-
-import (
-	"encoding/binary"
-
-	ethCommon "github.com/ethereum/go-ethereum/common"
-)
-
-func getAccountID(addr ethCommon.Address, tokenID uint32) [36]byte {
-	var tokenIDBytes [4]byte
-	binary.LittleEndian.PutUint32(tokenIDBytes[:], tokenID)
-	accountIDBytes := append(addr[:], tokenIDBytes[:]...)
-	var accountID [36]byte
-	copy(accountID[:], accountIDBytes[:36])
-	return accountID
-}