TxSel integrated with L2DB, Add L1CoordTx creation

- GetL2TxSelection & GetL1L2TxSelection integrated with dbs
- Create L1CoordinatorTx of type CreateAccountDeposit when a L2
requires it (and the AccountCreationAuth exists)

common/account.go

@@ -19,8 +19,50 @@ const (
 	maxNonceValue = 0xffffffffff
 	// maxBalanceBytes is the maximum bytes that can use the Account.Balance *big.Int
 	maxBalanceBytes = 24
+
+	idxBytesLen = 4
+	// maxIdxValue is the maximum value that Idx can have (32 bits: maxIdxValue=2**32-1)
+	maxIdxValue = 0xffffffff
+
+	// userThreshold determines the threshold from the User Idxs can be
+	userThreshold = 256
+	// IdxUserThreshold is a Idx type value that determines the threshold
+	// from the User Idxs can be
+	IdxUserThreshold = Idx(userThreshold)
 )
 
+// Idx represents the account Index in the MerkleTree
+type Idx uint32
+
+// Bytes returns a byte array representing the Idx
+func (idx Idx) Bytes() []byte {
+	var b [4]byte
+	binary.LittleEndian.PutUint32(b[:], uint32(idx))
+	return b[:]
+}
+
+// BigInt returns a *big.Int representing the Idx
+func (idx Idx) BigInt() *big.Int {
+	return big.NewInt(int64(idx))
+}
+
+// IdxFromBytes returns Idx from a byte array
+func IdxFromBytes(b []byte) (Idx, error) {
+	if len(b) != idxBytesLen {
+		return 0, fmt.Errorf("can not parse Idx, bytes len %d, expected 4", len(b))
+	}
+	idx := binary.LittleEndian.Uint32(b[:4])
+	return Idx(idx), nil
+}
+
+// IdxFromBigInt converts a *big.Int to Idx type
+func IdxFromBigInt(b *big.Int) (Idx, error) {
+	if b.Int64() > maxIdxValue {
+		return 0, ErrNumOverflow
+	}
+	return Idx(uint32(b.Int64())), nil
+}
+
 // Account is a struct that gives information of the holdings of an address and a specific token. Is the data structure that generates the Value stored in the leaf of the MerkleTree
 type Account struct {
 	TokenID   TokenID

common/accountcreationauths.go

@@ -11,6 +11,6 @@ import (
 type AccountCreationAuth struct {
 	Timestamp time.Time
 	EthAddr   ethCommon.Address
-	BJJ       babyjub.PublicKey
+	BJJ       *babyjub.PublicKey
 	Signature []byte
 }

common/tx.go

@@ -1,49 +1,9 @@
 package common
 
 import (
-	"encoding/binary"
-	"fmt"
 	"math/big"
 )
 
-const (
-	idxBytesLen = 4
-	// maxIdxValue is the maximum value that Idx can have (32 bits: maxIdxValue=2**32-1)
-	maxIdxValue = 0xffffffff
-)
-
-// Idx represents the account Index in the MerkleTree
-type Idx uint32
-
-// Bytes returns a byte array representing the Idx
-func (idx Idx) Bytes() []byte {
-	var b [4]byte
-	binary.LittleEndian.PutUint32(b[:], uint32(idx))
-	return b[:]
-}
-
-// BigInt returns a *big.Int representing the Idx
-func (idx Idx) BigInt() *big.Int {
-	return big.NewInt(int64(idx))
-}
-
-// IdxFromBytes returns Idx from a byte array
-func IdxFromBytes(b []byte) (Idx, error) {
-	if len(b) != idxBytesLen {
-		return 0, fmt.Errorf("can not parse Idx, bytes len %d, expected 4", len(b))
-	}
-	idx := binary.LittleEndian.Uint32(b[:4])
-	return Idx(idx), nil
-}
-
-// IdxFromBigInt converts a *big.Int to Idx type
-func IdxFromBigInt(b *big.Int) (Idx, error) {
-	if b.Int64() > maxIdxValue {
-		return 0, ErrNumOverflow
-	}
-	return Idx(uint32(b.Int64())), nil
-}
-
 // TxID is the identifier of a Hermez network transaction
 type TxID Hash // Hash is a guess
 

db/l2db/l2db.go

@@ -65,6 +65,12 @@ func NewL2DB(
 	}, nil
 }
 
+// DB returns a pointer to the L2DB.db. This method should be used only for
+// internal testing purposes.
+func (l2db *L2DB) DB() *sqlx.DB {
+	return l2db.db
+}
+
 // AddTx inserts a tx into the L2DB
 func (l2db *L2DB) AddTx(tx *common.PoolL2Tx) error {
 	return meddler.Insert(l2db.db, "tx_pool", tx)

db/statedb/txprocessors.go

@@ -64,10 +64,12 @@ func (s *StateDB) ProcessTxs(cmpExitTree, cmpZKInputs bool, l1usertxs, l1coordin
 
 	// TBD if ExitTree is only in memory or stored in disk, for the moment
 	// only needed in memory
+	if cmpExitTree {
 		exitTree, err = merkletree.NewMerkleTree(memory.NewMemoryStorage(), s.mt.MaxLevels())
 		if err != nil {
 			return nil, nil, err
 		}
+	}
 
 	// assumption: l1usertx are sorted by L1Tx.Position
 	for _, tx := range l1usertxs {
@@ -333,12 +335,12 @@ func (s *StateDB) processL2Tx(exitTree *merkletree.MerkleTree, tx *common.PoolL2
 			// Idx not set in the Tx, get it from DB through ToEthAddr or ToBJJ
 			var idx common.Idx
 			if !bytes.Equal(tx.ToEthAddr.Bytes(), ffAddr.Bytes()) {
-				idx = s.getIdxByEthAddr(tx.ToEthAddr)
+				idx = s.GetIdxByEthAddr(tx.ToEthAddr)
 				if idx == common.Idx(0) {
 					return nil, nil, false, fmt.Errorf("Idx can not be found for given tx.FromEthAddr")
 				}
 			} else {
-				idx = s.getIdxByBJJ(tx.ToBJJ)
+				idx = s.GetIdxByBJJ(tx.ToBJJ)
 				if idx == common.Idx(0) {
 					return nil, nil, false, fmt.Errorf("Idx can not be found for given tx.FromBJJ")
 				}
@@ -629,6 +631,9 @@ func (s *StateDB) applyExit(exitTree *merkletree.MerkleTree, tx *common.Tx) (*co
 		s.zki.Siblings1[s.i] = siblingsToZKInputFormat(p.Siblings)
 	}
 
+	if exitTree == nil {
+		return nil, false, nil
+	}
 	exitAccount, err := getAccountInTreeDB(exitTree.DB(), tx.FromIdx)
 	if err == db.ErrNotFound {
 		// 1a. if idx does not exist in exitTree:

db/statedb/utils.go

@@ -9,13 +9,27 @@ import (
 	"github.com/iden3/go-merkletree"
 )
 
-// TODO
+// GetIdxByEthAddr returns the smallest Idx in the StateDB for the given
-func (s *StateDB) getIdxByEthAddr(addr ethCommon.Address) common.Idx {
+// Ethereum Address. Will return 0 in case that Idx is not found in the
+// StateDB.
+func (s *StateDB) GetIdxByEthAddr(addr ethCommon.Address) common.Idx {
+	// TODO
 	return common.Idx(0)
 }
 
-// TODO
+// GetIdxByBJJ returns the smallest Idx in the StateDB for the given BabyJubJub
-func (s *StateDB) getIdxByBJJ(pk *babyjub.PublicKey) common.Idx {
+// PublicKey. Will return 0 in case that Idx is not found in the StateDB.
+func (s *StateDB) GetIdxByBJJ(pk *babyjub.PublicKey) common.Idx {
+	// TODO
+	return common.Idx(0)
+}
+
+// GetIdxByEthAddrBJJ returns the smallest Idx in the StateDB for the given
+// Ethereum Address AND the given BabyJubJub PublicKey. If `addr` is the zero
+// address, it's ignored in the query.  If `pk` is nil, it's ignored in the
+// query.  Will return 0 in case that Idx is not found in the StateDB.
+func (s *StateDB) GetIdxByEthAddrBJJ(addr ethCommon.Address, pk *babyjub.PublicKey) common.Idx {
+	// TODO
 	return common.Idx(0)
 }
 

test/l2db.go

@@ -0,0 +1,13 @@
+package test
+
+import "github.com/jmoiron/sqlx"
+
+// CleanL2DB deletes 'tx_pool' and 'account_creation_auth' from the given DB
+func CleanL2DB(db *sqlx.DB) {
+	if _, err := db.Exec("DELETE FROM tx_pool"); err != nil {
+		panic(err)
+	}
+	if _, err := db.Exec("DELETE FROM account_creation_auth"); err != nil {
+		panic(err)
+	}
+}

test/txs.go

@@ -63,7 +63,7 @@ func GenerateTestTxs(t *testing.T, instructions Instructions) ([][]*common.L1Tx,
 	var coordinatorL1Txs [][]*common.L1Tx
 	var poolL2Txs [][]*common.PoolL2Tx
 	idx := 1
-	for _, inst := range instructions.Instructions {
+	for i, inst := range instructions.Instructions {
 		switch inst.Type {
 		case common.TxTypeCreateAccountDeposit:
 			tx := common.L1Tx{
@@ -98,7 +98,7 @@ func GenerateTestTxs(t *testing.T, instructions Instructions) ([][]*common.L1Tx,
 			}
 
 			tx := common.PoolL2Tx{
-				// TxID: nil,
+				TxID:        common.TxID([]byte{byte(i)}), // TODO this is for the moment, once TxID Hash is implemented use it
 				FromIdx:     accounts[idxTokenIDToString(inst.From, inst.TokenID)].Idx,
 				ToIdx:       accounts[idxTokenIDToString(inst.To, inst.TokenID)].Idx,
 				ToEthAddr:   accounts[idxTokenIDToString(inst.To, inst.TokenID)].Addr,
@@ -109,7 +109,7 @@ func GenerateTestTxs(t *testing.T, instructions Instructions) ([][]*common.L1Tx,
 				Nonce:       accounts[idxTokenIDToString(inst.From, inst.TokenID)].Nonce,
 				State:       common.PoolL2TxStatePending,
 				Timestamp:   time.Now(),
-				BatchNum:    0,
+				BatchNum:    common.BatchNum(0),
 				RqToEthAddr: accounts[idxTokenIDToString(inst.To, inst.TokenID)].Addr,
 				RqToBJJ:     accounts[idxTokenIDToString(inst.To, inst.TokenID)].BJJ.Public(),
 				Type:        common.TxTypeTransfer,

txselector/txselector.go

@@ -1,6 +1,9 @@
 package txselector
 
+// current: very simple version of TxSelector
+
 import (
+	"math/big"
 	"sort"
 
 	"github.com/hermeznetwork/hermez-node/common"
@@ -81,16 +84,17 @@ func (txsel *TxSelector) GetL2TxSelection(batchNum common.BatchNum) ([]*common.P
 	// get most profitable L2-tx
 	txs := txsel.getL2Profitable(validTxs, txsel.MaxTxs)
 
-	// apply L2-tx to local AccountDB, make checkpoint tagged with BatchID
+	// process the txs in the local AccountsDB
-	//     update balances
+	_, _, err = txsel.localAccountsDB.ProcessTxs(false, false, nil, nil, txs)
-	//     update nonces
+	if err != nil {
-
+		return nil, err
-	// return selected txs
+	}
-	return txs, nil
+	err = txsel.localAccountsDB.MakeCheckpoint()
+	return txs, err
 }
 
 // GetL1L2TxSelection returns the selection of L1 + L2 txs
-func (txsel *TxSelector) GetL1L2TxSelection(batchNum common.BatchNum, l1txs []*common.L1Tx) ([]*common.L1Tx, []*common.L1Tx, []*common.PoolL2Tx, error) {
+func (txsel *TxSelector) GetL1L2TxSelection(batchNum common.BatchNum, l1Txs []*common.L1Tx) ([]*common.L1Tx, []*common.L1Tx, []*common.PoolL2Tx, error) {
 	// apply l1-user-tx to localAccountDB
 	//     create new leaves
 	//     update balances
@@ -102,53 +106,72 @@ func (txsel *TxSelector) GetL1L2TxSelection(batchNum common.BatchNum, l1txs []*c
 		return nil, nil, nil, err
 	}
 
-	// discard the txs that don't have an Account in the AccountDB
-	// neither appear in the AccountCreationAuthsDB
 	var validTxs txs
-	for _, tx := range l2TxsRaw {
+	var l1CoordinatorTxs []*common.L1Tx
-		if txsel.checkIfAccountExistOrPending(tx.FromIdx) {
+
-			// if FromIdx has an account into the AccountsDB
+	// if tx.ToIdx>=256, tx.ToIdx should exist to localAccountsDB, if so,
-			validTxs = append(validTxs, tx)
+	// tx is used.  if tx.ToIdx==0, check if tx.ToEthAddr/tx.ToBJJ exist in
+	// localAccountsDB, if yes tx is used; if not, check if tx.ToEthAddr is
+	// in AccountCreationAuthDB, if so, tx is used and L1CoordinatorTx of
+	// CreateAccountAndDeposit is created.
+	for i := 0; i < len(l2TxsRaw); i++ {
+		if l2TxsRaw[i].ToIdx >= common.IdxUserThreshold {
+			_, err = txsel.localAccountsDB.GetAccount(l2TxsRaw[i].ToIdx)
+			if err != nil {
+				// tx not valid
+				continue
+			}
+			// Account found in the DB, include the l2Tx in the selection
+			validTxs = append(validTxs, l2TxsRaw[i])
+		} else if l2TxsRaw[i].ToIdx == common.Idx(0) {
+			idx := txsel.localAccountsDB.GetIdxByEthAddrBJJ(l2TxsRaw[i].ToEthAddr, l2TxsRaw[i].ToBJJ)
+			if idx != common.Idx(0) {
+				// account for ToEthAddr/ToBJJ already exist,
+				// there is no need to create a new one.
+				// tx valid, StateDB will use the ToIdx==0 to define the AuxToIdx
+				validTxs = append(validTxs, l2TxsRaw[i])
+				continue
+			}
+			// check if ToEthAddr is in AccountCreationAuths
+			_, err := txsel.l2db.GetAccountCreationAuth(l2TxsRaw[i].ToEthAddr) // TODO once l2db.GetAccountCreationAuth is ready, use the value returned as 'accAuth'
+			if err != nil {
+				// not found, l2Tx will not be added in the selection
+				continue
+			}
+			validTxs = append(validTxs, l2TxsRaw[i])
+
+			// create L1CoordinatorTx for the accountCreation
+			l1CoordinatorTx := &common.L1Tx{
+				UserOrigin: false,
+				// FromEthAddr: accAuth.EthAddr, // TODO This 2 lines will panic, as l2db.GetAccountCreationAuth is not implemented yet and returns nil. Uncomment this 2 lines once l2db method is done.
+				// FromBJJ:     accAuth.BJJ,
+				TokenID:    l2TxsRaw[i].TokenID,
+				LoadAmount: big.NewInt(0),
+				Type:       common.TxTypeCreateAccountDeposit,
+			}
+			l1CoordinatorTxs = append(l1CoordinatorTxs, l1CoordinatorTx)
+		} else if l2TxsRaw[i].ToIdx == common.Idx(1) {
+			// valid txs (of Exit type)
+			validTxs = append(validTxs, l2TxsRaw[i])
 		}
 	}
 
-	// prepare (from the selected l2txs) pending to create from the AccountCreationAuthsDB
+	// get most profitable L2-tx
-	var accountCreationAuths []*common.Account
+	maxL2Txs := txsel.MaxTxs - uint64(len(l1CoordinatorTxs)) // - len(l1UserTxs)
-	// TODO once DB ready:
+	l2Txs := txsel.getL2Profitable(validTxs, maxL2Txs)
-	// if tx.ToIdx is in AccountCreationAuthsDB, take it and add it to
+
-	// the array 'accountCreationAuths'
+	// TODO This 3 lines will panic, as l2db.GetAccountCreationAuth is not implemented yet and returns nil. Uncomment this lines once l2db method is done.
-	// for _, tx := range l2txs {
+	// process the txs in the local AccountsDB
-	//         account, err := txsel.localAccountsDB.GetAccount(tx.ToIdx)
+	// _, _, err = txsel.localAccountsDB.ProcessTxs(false, false, l1Txs, l1CoordinatorTxs, l2Txs)
 	// if err != nil {
 	//         return nil, nil, nil, err
 	// }
-	//         if accountToCreate, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok {
+	err = txsel.localAccountsDB.MakeCheckpoint()
-	//                 accountCreationAuths = append(accountCreationAuths, accountToCreate)
-	//         }
-	// }
-
-	// create L1-operator-tx for each L2-tx selected in which the recipient does not have account
-	l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(accountCreationAuths) // only with the L2-tx selected ones
-
-	// get most profitable L2-tx
-	maxL2Txs := txsel.MaxTxs - uint64(len(l1OperatorTxs)) // - len(l1UserTxs)
-	l2txs := txsel.getL2Profitable(validTxs, maxL2Txs)
-
-	return l1txs, l1OperatorTxs, l2txs, nil
-}
-
-func (txsel *TxSelector) checkIfAccountExistOrPending(idx common.Idx) bool {
-	// check if account exist in AccountDB
-	_, err := txsel.localAccountsDB.GetAccount(idx)
 	if err != nil {
-		return false
+		return nil, nil, nil, err
 	}
-	// check if account is pending to create
+
-	// TODO need a method in the DB to get the PendingRegisters
+	return l1Txs, l1CoordinatorTxs, l2Txs, nil
-	// if _, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok {
-	//         return true
-	// }
-	return false
 }
 
 func (txsel *TxSelector) getL2Profitable(txs txs, max uint64) txs {
@@ -158,7 +181,3 @@ func (txsel *TxSelector) getL2Profitable(txs txs, max uint64) txs {
 	}
 	return txs[:max]
 }
-func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []*common.Account) []*common.L1Tx {
-	//
-	return nil
-}

txselector/txselector_test.go

@@ -1,29 +1,58 @@
 package txselector
 
 import (
-	"fmt"
 	"io/ioutil"
+	"os"
 	"testing"
+	"time"
 
+	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db/l2db"
 	"github.com/hermeznetwork/hermez-node/db/statedb"
+	"github.com/hermeznetwork/hermez-node/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestGetL2TxSelection(t *testing.T) {
+func initTest(t *testing.T, testSet string) *TxSelector {
+	pass := os.Getenv("POSTGRES_PASS")
+	l2DB, err := l2db.NewL2DB(5432, "localhost", "hermez", pass, "l2", 10, 512, 24*time.Hour)
+	require.Nil(t, err)
+
 	dir, err := ioutil.TempDir("", "tmpdb")
 	require.Nil(t, err)
 	sdb, err := statedb.NewStateDB(dir, false, 0)
-	assert.Nil(t, err)
+	require.Nil(t, err)
-	testL2DB := &l2db.L2DB{}
-	// initTestDB(testL2DB, sdb)
 
 	txselDir, err := ioutil.TempDir("", "tmpTxSelDB")
 	require.Nil(t, err)
-	txsel, err := NewTxSelector(txselDir, sdb, testL2DB, 3, 3, 3)
+	txsel, err := NewTxSelector(txselDir, sdb, l2DB, 100, 100, 1000)
+	require.Nil(t, err)
+
+	return txsel
+}
+func addL2Txs(t *testing.T, txsel *TxSelector, poolL2Txs []*common.PoolL2Tx) {
+	for i := 0; i < len(poolL2Txs); i++ {
+		err := txsel.l2db.AddTx(poolL2Txs[i])
+		require.Nil(t, err)
+	}
+}
+
+func TestGetL2TxSelection(t *testing.T) {
+	txsel := initTest(t, test.SetTest0)
+	test.CleanL2DB(txsel.l2db.DB())
+
+	// generate test transactions
+	l1Txs, _, poolL2Txs := test.GenerateTestTxsFromSet(t, test.SetTest0)
+
+	// add the first batch of transactions to the TxSelector
+	addL2Txs(t, txsel, poolL2Txs[0])
+
+	_, err := txsel.GetL2TxSelection(0)
+	assert.Nil(t, err)
+
+	_, _, _, err = txsel.GetL1L2TxSelection(0, l1Txs[0])
 	assert.Nil(t, err)
-	fmt.Println(txsel)
 
 	// txs, err := txsel.GetL2TxSelection(0)
 	// assert.Nil(t, err)