TxSel integrated with L2DB, Add L1CoordTx creation - GetL2TxSelection & GetL1L2TxSelection integrated with dbs - Create L1CoordinatorTx of type CreateAccountDeposit when a L2 requires it (and the AccountCreationAuth exists)
@@ -19,7 +19,49 @@ // maxNonceValue is the maximum value that the Account.Nonce can have (40 bits: maxNonceValue=2**40-1)
maxNonceValue = 0xffffffffff // maxBalanceBytes is the maximum bytes that can use the Account.Balance *big.Int maxBalanceBytes = 24 + + idxBytesLen = 4 + // maxIdxValue is the maximum value that Idx can have (32 bits: maxIdxValue=2**32-1) + maxIdxValue = 0xffffffff + + // userThreshold determines the threshold from the User Idxs can be + userThreshold = 256 + // IdxUserThreshold is a Idx type value that determines the threshold + // from the User Idxs can be + IdxUserThreshold = Idx(userThreshold) ) + +// Idx represents the account Index in the MerkleTree +type Idx uint32 + +// Bytes returns a byte array representing the Idx +func (idx Idx) Bytes() []byte { + var b [4]byte + binary.LittleEndian.PutUint32(b[:], uint32(idx)) + return b[:] +} + +// BigInt returns a *big.Int representing the Idx +func (idx Idx) BigInt() *big.Int { + return big.NewInt(int64(idx)) +} + +// IdxFromBytes returns Idx from a byte array +func IdxFromBytes(b []byte) (Idx, error) { + if len(b) != idxBytesLen { + return 0, fmt.Errorf("can not parse Idx, bytes len %d, expected 4", len(b)) + } + idx := binary.LittleEndian.Uint32(b[:4]) + return Idx(idx), nil +} + +// IdxFromBigInt converts a *big.Int to Idx type +func IdxFromBigInt(b *big.Int) (Idx, error) { + if b.Int64() > maxIdxValue { + return 0, ErrNumOverflow + } + return Idx(uint32(b.Int64())), nil +} // Account is a struct that gives information of the holdings of an address and a specific token. Is the data structure that generates the Value stored in the leaf of the MerkleTree type Account struct {
@@ -11,6 +11,6 @@ // AccountCreationAuth authorizations sent by users to the L2DB, to be used for account creations when necessary
type AccountCreationAuth struct { Timestamp time.Time EthAddr ethCommon.Address - BJJ babyjub.PublicKey + BJJ *babyjub.PublicKey Signature []byte }
@@ -1,48 +1,8 @@
package common import ( - "encoding/binary" - "fmt" "math/big" ) - -const ( - idxBytesLen = 4 - // maxIdxValue is the maximum value that Idx can have (32 bits: maxIdxValue=2**32-1) - maxIdxValue = 0xffffffff -) - -// Idx represents the account Index in the MerkleTree -type Idx uint32 - -// Bytes returns a byte array representing the Idx -func (idx Idx) Bytes() []byte { - var b [4]byte - binary.LittleEndian.PutUint32(b[:], uint32(idx)) - return b[:] -} - -// BigInt returns a *big.Int representing the Idx -func (idx Idx) BigInt() *big.Int { - return big.NewInt(int64(idx)) -} - -// IdxFromBytes returns Idx from a byte array -func IdxFromBytes(b []byte) (Idx, error) { - if len(b) != idxBytesLen { - return 0, fmt.Errorf("can not parse Idx, bytes len %d, expected 4", len(b)) - } - idx := binary.LittleEndian.Uint32(b[:4]) - return Idx(idx), nil -} - -// IdxFromBigInt converts a *big.Int to Idx type -func IdxFromBigInt(b *big.Int) (Idx, error) { - if b.Int64() > maxIdxValue { - return 0, ErrNumOverflow - } - return Idx(uint32(b.Int64())), nil -} // TxID is the identifier of a Hermez network transaction type TxID Hash // Hash is a guess
@@ -65,6 +65,12 @@ maxTxs: maxTxs,
}, nil } +// DB returns a pointer to the L2DB.db. This method should be used only for +// internal testing purposes. +func (l2db *L2DB) DB() *sqlx.DB { + return l2db.db +} + // AddTx inserts a tx into the L2DB func (l2db *L2DB) AddTx(tx *common.PoolL2Tx) error { return meddler.Insert(l2db.db, "tx_pool", tx)
@@ -64,9 +64,11 @@ }
// TBD if ExitTree is only in memory or stored in disk, for the moment // only needed in memory - exitTree, err = merkletree.NewMerkleTree(memory.NewMemoryStorage(), s.mt.MaxLevels()) - if err != nil { - return nil, nil, err + if cmpExitTree { + exitTree, err = merkletree.NewMerkleTree(memory.NewMemoryStorage(), s.mt.MaxLevels()) + if err != nil { + return nil, nil, err + } } // assumption: l1usertx are sorted by L1Tx.Position@@ -333,12 +335,12 @@ if tx.ToIdx == common.Idx(0) {
// Idx not set in the Tx, get it from DB through ToEthAddr or ToBJJ var idx common.Idx if !bytes.Equal(tx.ToEthAddr.Bytes(), ffAddr.Bytes()) { - idx = s.getIdxByEthAddr(tx.ToEthAddr) + idx = s.GetIdxByEthAddr(tx.ToEthAddr) if idx == common.Idx(0) { return nil, nil, false, fmt.Errorf("Idx can not be found for given tx.FromEthAddr") } } else { - idx = s.getIdxByBJJ(tx.ToBJJ) + idx = s.GetIdxByBJJ(tx.ToBJJ) if idx == common.Idx(0) { return nil, nil, false, fmt.Errorf("Idx can not be found for given tx.FromBJJ") }@@ -629,6 +631,9 @@ s.zki.EthAddr1[s.i] = common.EthAddrToBigInt(acc.EthAddr)
s.zki.Siblings1[s.i] = siblingsToZKInputFormat(p.Siblings) } + if exitTree == nil { + return nil, false, nil + } exitAccount, err := getAccountInTreeDB(exitTree.DB(), tx.FromIdx) if err == db.ErrNotFound { // 1a. if idx does not exist in exitTree:
@@ -9,13 +9,27 @@ "github.com/iden3/go-iden3-crypto/babyjub"
"github.com/iden3/go-merkletree" ) -// TODO -func (s *StateDB) getIdxByEthAddr(addr ethCommon.Address) common.Idx { +// GetIdxByEthAddr returns the smallest Idx in the StateDB for the given +// Ethereum Address. Will return 0 in case that Idx is not found in the +// StateDB. +func (s *StateDB) GetIdxByEthAddr(addr ethCommon.Address) common.Idx { + // TODO return common.Idx(0) } -// TODO -func (s *StateDB) getIdxByBJJ(pk *babyjub.PublicKey) common.Idx { +// GetIdxByBJJ returns the smallest Idx in the StateDB for the given BabyJubJub +// PublicKey. Will return 0 in case that Idx is not found in the StateDB. +func (s *StateDB) GetIdxByBJJ(pk *babyjub.PublicKey) common.Idx { + // TODO + return common.Idx(0) +} + +// GetIdxByEthAddrBJJ returns the smallest Idx in the StateDB for the given +// Ethereum Address AND the given BabyJubJub PublicKey. If `addr` is the zero +// address, it's ignored in the query. If `pk` is nil, it's ignored in the +// query. Will return 0 in case that Idx is not found in the StateDB. +func (s *StateDB) GetIdxByEthAddrBJJ(addr ethCommon.Address, pk *babyjub.PublicKey) common.Idx { + // TODO return common.Idx(0) }
@@ -0,0 +1,13 @@
+package test + +import "github.com/jmoiron/sqlx" + +// CleanL2DB deletes 'tx_pool' and 'account_creation_auth' from the given DB +func CleanL2DB(db *sqlx.DB) { + if _, err := db.Exec("DELETE FROM tx_pool"); err != nil { + panic(err) + } + if _, err := db.Exec("DELETE FROM account_creation_auth"); err != nil { + panic(err) + } +}
@@ -63,7 +63,7 @@ var l1Txs [][]*common.L1Tx
var coordinatorL1Txs [][]*common.L1Tx var poolL2Txs [][]*common.PoolL2Tx idx := 1 - for _, inst := range instructions.Instructions { + for i, inst := range instructions.Instructions { switch inst.Type { case common.TxTypeCreateAccountDeposit: tx := common.L1Tx{@@ -98,7 +98,7 @@ idx++
} tx := common.PoolL2Tx{ - // TxID: nil, + TxID: common.TxID([]byte{byte(i)}), // TODO this is for the moment, once TxID Hash is implemented use it FromIdx: accounts[idxTokenIDToString(inst.From, inst.TokenID)].Idx, ToIdx: accounts[idxTokenIDToString(inst.To, inst.TokenID)].Idx, ToEthAddr: accounts[idxTokenIDToString(inst.To, inst.TokenID)].Addr,@@ -109,7 +109,7 @@ Fee: common.FeeSelector(inst.Fee),
Nonce: accounts[idxTokenIDToString(inst.From, inst.TokenID)].Nonce, State: common.PoolL2TxStatePending, Timestamp: time.Now(), - BatchNum: 0, + BatchNum: common.BatchNum(0), RqToEthAddr: accounts[idxTokenIDToString(inst.To, inst.TokenID)].Addr, RqToBJJ: accounts[idxTokenIDToString(inst.To, inst.TokenID)].BJJ.Public(), Type: common.TxTypeTransfer,
@@ -1,6 +1,9 @@
package txselector +// current: very simple version of TxSelector + import ( + "math/big" "sort" "github.com/hermeznetwork/hermez-node/common"@@ -81,16 +84,17 @@
// get most profitable L2-tx txs := txsel.getL2Profitable(validTxs, txsel.MaxTxs) - // apply L2-tx to local AccountDB, make checkpoint tagged with BatchID - // update balances - // update nonces - - // return selected txs - return txs, nil + // process the txs in the local AccountsDB + _, _, err = txsel.localAccountsDB.ProcessTxs(false, false, nil, nil, txs) + if err != nil { + return nil, err + } + err = txsel.localAccountsDB.MakeCheckpoint() + return txs, err } // GetL1L2TxSelection returns the selection of L1 + L2 txs -func (txsel *TxSelector) GetL1L2TxSelection(batchNum common.BatchNum, l1txs []*common.L1Tx) ([]*common.L1Tx, []*common.L1Tx, []*common.PoolL2Tx, error) { +func (txsel *TxSelector) GetL1L2TxSelection(batchNum common.BatchNum, l1Txs []*common.L1Tx) ([]*common.L1Tx, []*common.L1Tx, []*common.PoolL2Tx, error) { // apply l1-user-tx to localAccountDB // create new leaves // update balances@@ -102,53 +106,72 @@ if err != nil {
return nil, nil, nil, err } - // discard the txs that don't have an Account in the AccountDB - // neither appear in the AccountCreationAuthsDB var validTxs txs - for _, tx := range l2TxsRaw { - if txsel.checkIfAccountExistOrPending(tx.FromIdx) { - // if FromIdx has an account into the AccountsDB - validTxs = append(validTxs, tx) + var l1CoordinatorTxs []*common.L1Tx + + // if tx.ToIdx>=256, tx.ToIdx should exist to localAccountsDB, if so, + // tx is used. if tx.ToIdx==0, check if tx.ToEthAddr/tx.ToBJJ exist in + // localAccountsDB, if yes tx is used; if not, check if tx.ToEthAddr is + // in AccountCreationAuthDB, if so, tx is used and L1CoordinatorTx of + // CreateAccountAndDeposit is created. + for i := 0; i < len(l2TxsRaw); i++ { + if l2TxsRaw[i].ToIdx >= common.IdxUserThreshold { + _, err = txsel.localAccountsDB.GetAccount(l2TxsRaw[i].ToIdx) + if err != nil { + // tx not valid + continue + } + // Account found in the DB, include the l2Tx in the selection + validTxs = append(validTxs, l2TxsRaw[i]) + } else if l2TxsRaw[i].ToIdx == common.Idx(0) { + idx := txsel.localAccountsDB.GetIdxByEthAddrBJJ(l2TxsRaw[i].ToEthAddr, l2TxsRaw[i].ToBJJ) + if idx != common.Idx(0) { + // account for ToEthAddr/ToBJJ already exist, + // there is no need to create a new one. + // tx valid, StateDB will use the ToIdx==0 to define the AuxToIdx + validTxs = append(validTxs, l2TxsRaw[i]) + continue + } + // check if ToEthAddr is in AccountCreationAuths + _, err := txsel.l2db.GetAccountCreationAuth(l2TxsRaw[i].ToEthAddr) // TODO once l2db.GetAccountCreationAuth is ready, use the value returned as 'accAuth' + if err != nil { + // not found, l2Tx will not be added in the selection + continue + } + validTxs = append(validTxs, l2TxsRaw[i]) + + // create L1CoordinatorTx for the accountCreation + l1CoordinatorTx := &common.L1Tx{ + UserOrigin: false, + // FromEthAddr: accAuth.EthAddr, // TODO This 2 lines will panic, as l2db.GetAccountCreationAuth is not implemented yet and returns nil. Uncomment this 2 lines once l2db method is done. + // FromBJJ: accAuth.BJJ, + TokenID: l2TxsRaw[i].TokenID, + LoadAmount: big.NewInt(0), + Type: common.TxTypeCreateAccountDeposit, + } + l1CoordinatorTxs = append(l1CoordinatorTxs, l1CoordinatorTx) + } else if l2TxsRaw[i].ToIdx == common.Idx(1) { + // valid txs (of Exit type) + validTxs = append(validTxs, l2TxsRaw[i]) } } - // prepare (from the selected l2txs) pending to create from the AccountCreationAuthsDB - var accountCreationAuths []*common.Account - // TODO once DB ready: - // if tx.ToIdx is in AccountCreationAuthsDB, take it and add it to - // the array 'accountCreationAuths' - // for _, tx := range l2txs { - // account, err := txsel.localAccountsDB.GetAccount(tx.ToIdx) - // if err != nil { - // return nil, nil, nil, err - // } - // if accountToCreate, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok { - // accountCreationAuths = append(accountCreationAuths, accountToCreate) - // } - // } - - // create L1-operator-tx for each L2-tx selected in which the recipient does not have account - l1OperatorTxs := txsel.createL1OperatorTxForL2Tx(accountCreationAuths) // only with the L2-tx selected ones - // get most profitable L2-tx - maxL2Txs := txsel.MaxTxs - uint64(len(l1OperatorTxs)) // - len(l1UserTxs) - l2txs := txsel.getL2Profitable(validTxs, maxL2Txs) + maxL2Txs := txsel.MaxTxs - uint64(len(l1CoordinatorTxs)) // - len(l1UserTxs) + l2Txs := txsel.getL2Profitable(validTxs, maxL2Txs) - return l1txs, l1OperatorTxs, l2txs, nil -} - -func (txsel *TxSelector) checkIfAccountExistOrPending(idx common.Idx) bool { - // check if account exist in AccountDB - _, err := txsel.localAccountsDB.GetAccount(idx) + // TODO This 3 lines will panic, as l2db.GetAccountCreationAuth is not implemented yet and returns nil. Uncomment this lines once l2db method is done. + // process the txs in the local AccountsDB + // _, _, err = txsel.localAccountsDB.ProcessTxs(false, false, l1Txs, l1CoordinatorTxs, l2Txs) + // if err != nil { + // return nil, nil, nil, err + // } + err = txsel.localAccountsDB.MakeCheckpoint() if err != nil { - return false + return nil, nil, nil, err } - // check if account is pending to create - // TODO need a method in the DB to get the PendingRegisters - // if _, ok := txsel.DB.AccountCreationAuthsDB[accountID]; ok { - // return true - // } - return false + + return l1Txs, l1CoordinatorTxs, l2Txs, nil } func (txsel *TxSelector) getL2Profitable(txs txs, max uint64) txs {@@ -158,7 +181,3 @@ return txs
} return txs[:max] } -func (txsel *TxSelector) createL1OperatorTxForL2Tx(accounts []*common.Account) []*common.L1Tx { - // - return nil -}
@@ -1,29 +1,58 @@
package txselector import ( - "fmt" "io/ioutil" + "os" "testing" + "time" + "github.com/hermeznetwork/hermez-node/common" "github.com/hermeznetwork/hermez-node/db/l2db" "github.com/hermeznetwork/hermez-node/db/statedb" + "github.com/hermeznetwork/hermez-node/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestGetL2TxSelection(t *testing.T) { +func initTest(t *testing.T, testSet string) *TxSelector { + pass := os.Getenv("POSTGRES_PASS") + l2DB, err := l2db.NewL2DB(5432, "localhost", "hermez", pass, "l2", 10, 512, 24*time.Hour) + require.Nil(t, err) + dir, err := ioutil.TempDir("", "tmpdb") require.Nil(t, err) sdb, err := statedb.NewStateDB(dir, false, 0) - assert.Nil(t, err) - testL2DB := &l2db.L2DB{} - // initTestDB(testL2DB, sdb) + require.Nil(t, err) txselDir, err := ioutil.TempDir("", "tmpTxSelDB") require.Nil(t, err) - txsel, err := NewTxSelector(txselDir, sdb, testL2DB, 3, 3, 3) + txsel, err := NewTxSelector(txselDir, sdb, l2DB, 100, 100, 1000) + require.Nil(t, err) + + return txsel +} +func addL2Txs(t *testing.T, txsel *TxSelector, poolL2Txs []*common.PoolL2Tx) { + for i := 0; i < len(poolL2Txs); i++ { + err := txsel.l2db.AddTx(poolL2Txs[i]) + require.Nil(t, err) + } +} + +func TestGetL2TxSelection(t *testing.T) { + txsel := initTest(t, test.SetTest0) + test.CleanL2DB(txsel.l2db.DB()) + + // generate test transactions + l1Txs, _, poolL2Txs := test.GenerateTestTxsFromSet(t, test.SetTest0) + + // add the first batch of transactions to the TxSelector + addL2Txs(t, txsel, poolL2Txs[0]) + + _, err := txsel.GetL2TxSelection(0) assert.Nil(t, err) - fmt.Println(txsel) + + _, _, _, err = txsel.GetL1L2TxSelection(0, l1Txs[0]) + assert.Nil(t, err) // txs, err := txsel.GetL2TxSelection(0) // assert.Nil(t, err)