arnaucube
arnaucube/hermez-nodePublic Self-hosted repository
Merge pull request #233 from hermeznetwork/feature/sdb-idxdb-tokenid

StateDB store Idxs with TokenID at the db-key & TxSelector include L1CoordinatorTxs in selections
Eduard S eduard@iden3.io
Tue, 27 Oct 2020 17:58:48 +0100
commit

c1b9b0be90b412e0893d77a81ab173757782c906

parent

d6d1a9c8984799571179f4295ec58f2d2f86074f

M common/pooll2tx.go → common/pooll2tx.go

@@ -22,7 +22,7 @@ TxID TxID `meddler:"tx_id"`

FromIdx Idx `meddler:"from_idx"` ToIdx Idx `meddler:"to_idx,zeroisnull"` AuxToIdx Idx `meddler:"-"` // AuxToIdx is only used internally at the StateDB to avoid repeated computation when processing transactions (from Synchronizer, TxSelector, BatchBuilder) - ToEthAddr ethCommon.Address `meddler:"to_eth_addr"` + ToEthAddr ethCommon.Address `meddler:"to_eth_addr,zeroisnull"` ToBJJ *babyjub.PublicKey `meddler:"to_bjj"` TokenID TokenID `meddler:"token_id"` Amount *big.Int `meddler:"amount,bigint"` // TODO: change to float16

@@ -34,7 +34,7 @@ Timestamp time.Time `meddler:"timestamp,utctime"` // time when added to the tx pool

// Stored in DB: optional fileds, may be uninitialized RqFromIdx Idx `meddler:"rq_from_idx,zeroisnull"` // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit) RqToIdx Idx `meddler:"rq_to_idx,zeroisnull"` // FromIdx is used by L1Tx/Deposit to indicate the Idx receiver of the L1Tx.LoadAmount (deposit) - RqToEthAddr ethCommon.Address `meddler:"rq_to_eth_addr"` + RqToEthAddr ethCommon.Address `meddler:"rq_to_eth_addr,zeroisnull"` RqToBJJ *babyjub.PublicKey `meddler:"rq_to_bjj"` // TODO: stop using json, use scanner/valuer RqTokenID TokenID `meddler:"rq_token_id,zeroisnull"` RqAmount *big.Int `meddler:"rq_amount,bigintnull"` // TODO: change to float16
M coordinator/coordinator.go → coordinator/coordinator.go

@@ -220,7 +220,7 @@ return nil, err

} } else { // 2b: only L2 txs - poolL2Txs, err = c.txsel.GetL2TxSelection([]common.Idx{}, c.batchNum) // TODO once feesInfo is added to method return, add the var + _, poolL2Txs, err = c.txsel.GetL2TxSelection([]common.Idx{}, c.batchNum) // TODO once feesInfo is added to method return, add the var if err != nil { return nil, err }
M db/l2db/l2db.go → db/l2db/l2db.go

@@ -82,7 +82,7 @@ TokenID: tx.TokenID,

Amount: tx.Amount, Fee: tx.Fee, Nonce: tx.Nonce, - State: tx.State, + State: common.PoolL2TxStatePending, Signature: tx.Signature, RqToBJJ: tx.RqToBJJ, RqAmount: tx.RqAmount,
M db/statedb/statedb.go → db/statedb/statedb.go

@@ -372,7 +372,7 @@ if err != nil {

return cpp, err } // store idx by EthAddr & BJJ - err = s.setIdxByEthAddrBJJ(idx, account.EthAddr, account.PublicKey) + err = s.setIdxByEthAddrBJJ(idx, account.EthAddr, account.PublicKey, account.TokenID) return cpp, err }
M db/statedb/txprocessors.go → db/statedb/txprocessors.go

@@ -378,7 +378,7 @@ var err error

// if tx.ToIdx==0, get toIdx by ToEthAddr or ToBJJ if tx.ToIdx == common.Idx(0) && tx.AuxToIdx == common.Idx(0) { // case when tx.Type== common.TxTypeTransferToEthAddr or common.TxTypeTransferToBJJ - tx.AuxToIdx, err = s.GetIdxByEthAddrBJJ(tx.ToEthAddr, tx.ToBJJ) + tx.AuxToIdx, err = s.GetIdxByEthAddrBJJ(tx.ToEthAddr, tx.ToBJJ, tx.TokenID) if err != nil { log.Error(err) return nil, nil, false, err
M db/statedb/txprocessors_test.go → db/statedb/txprocessors_test.go

@@ -44,6 +44,8 @@

// Idx of user 'A' idxA1 := tc.Users["A"].Accounts[common.TokenID(1)].Idx + // Process the 1st batch, which contains the L1CoordinatorTxs necessary + // to create the Coordinator accounts to receive the fees log.Debug("1st batch, 1st block, only L1CoordinatorTxs") ptOut, err := sdb.ProcessTxs(nil, nil, blocks[0].Batches[0].L1CoordinatorTxs, nil) require.Nil(t, err)
M db/statedb/utils.go → db/statedb/utils.go

@@ -12,11 +12,18 @@ "github.com/iden3/go-iden3-crypto/babyjub"

"github.com/iden3/go-merkletree" ) -func concatEthAddrBJJ(addr ethCommon.Address, pk *babyjub.PublicKey) []byte { +func concatEthAddrTokenID(addr ethCommon.Address, tokenID common.TokenID) []byte { + var b []byte + b = append(b, addr.Bytes()...) + b = append(b[:], tokenID.Bytes()[:]...) + return b +} +func concatEthAddrBJJTokenID(addr ethCommon.Address, pk *babyjub.PublicKey, tokenID common.TokenID) []byte { pkComp := pk.Compress() var b []byte b = append(b, addr.Bytes()...) b = append(b[:], pkComp[:]...) + b = append(b[:], tokenID.Bytes()[:]...) return b }

@@ -25,8 +32,8 @@ // - key: Eth Address, value: idx

// - key: EthAddr & BabyJubJub PublicKey Compressed, value: idx // If Idx already exist for the given EthAddr & BJJ, the remaining Idx will be // always the smallest one. -func (s *StateDB) setIdxByEthAddrBJJ(idx common.Idx, addr ethCommon.Address, pk *babyjub.PublicKey) error { - oldIdx, err := s.GetIdxByEthAddrBJJ(addr, pk) +func (s *StateDB) setIdxByEthAddrBJJ(idx common.Idx, addr ethCommon.Address, pk *babyjub.PublicKey, tokenID common.TokenID) error { + oldIdx, err := s.GetIdxByEthAddrBJJ(addr, pk, tokenID) if err == nil { // EthAddr & BJJ already have an Idx // check which Idx is smaller

@@ -46,7 +53,7 @@ tx, err := s.db.NewTx()

if err != nil { return err } - k := concatEthAddrBJJ(addr, pk) + k := concatEthAddrBJJTokenID(addr, pk, tokenID) // store Addr&BJJ-idx idxBytes, err := idx.Bytes() if err != nil {

@@ -57,7 +64,8 @@ if err != nil {

return err } // store Addr-idx - err = tx.Put(append(PrefixKeyAddr, addr.Bytes()...), idxBytes[:]) + k = concatEthAddrTokenID(addr, tokenID) + err = tx.Put(append(PrefixKeyAddr, k...), idxBytes[:]) if err != nil { return err }

@@ -71,8 +79,9 @@

// GetIdxByEthAddr returns the smallest Idx in the StateDB for the given // Ethereum Address. Will return common.Idx(0) and error in case that Idx is // not found in the StateDB. -func (s *StateDB) GetIdxByEthAddr(addr ethCommon.Address) (common.Idx, error) { - b, err := s.db.Get(append(PrefixKeyAddr, addr.Bytes()...)) +func (s *StateDB) GetIdxByEthAddr(addr ethCommon.Address, tokenID common.TokenID) (common.Idx, error) { + k := concatEthAddrTokenID(addr, tokenID) + b, err := s.db.Get(append(PrefixKeyAddr, k...)) if err != nil { return common.Idx(0), ErrToIdxNotFound }

@@ -88,13 +97,13 @@ // Ethereum Address AND the given BabyJubJub PublicKey. If `addr` is the zero

// address, it's ignored in the query. If `pk` is nil, it's ignored in the // query. Will return common.Idx(0) and error in case that Idx is not found in // the StateDB. -func (s *StateDB) GetIdxByEthAddrBJJ(addr ethCommon.Address, pk *babyjub.PublicKey) (common.Idx, error) { +func (s *StateDB) GetIdxByEthAddrBJJ(addr ethCommon.Address, pk *babyjub.PublicKey, tokenID common.TokenID) (common.Idx, error) { if !bytes.Equal(addr.Bytes(), common.EmptyAddr.Bytes()) && pk == nil { // case ToEthAddr!=0 && ToBJJ=0 - return s.GetIdxByEthAddr(addr) + return s.GetIdxByEthAddr(addr, tokenID) } else if !bytes.Equal(addr.Bytes(), common.EmptyAddr.Bytes()) && pk != nil { // case ToEthAddr!=0 && ToBJJ!=0 - k := concatEthAddrBJJ(addr, pk) + k := concatEthAddrBJJTokenID(addr, pk, tokenID) b, err := s.db.Get(append(PrefixKeyAddrBJJ, k...)) if err != nil { return common.Idx(0), ErrToIdxNotFound
M db/statedb/utils_test.go → db/statedb/utils_test.go

@@ -33,51 +33,60 @@ idx := common.Idx(1234)

idx2 := common.Idx(12345) idx3 := common.Idx(1233) + tokenID0 := common.TokenID(0) + tokenID1 := common.TokenID(1) + // store the keys for idx by Addr & BJJ - err = sdb.setIdxByEthAddrBJJ(idx, addr, pk) + err = sdb.setIdxByEthAddrBJJ(idx, addr, pk, tokenID0) require.Nil(t, err) - idxR, err := sdb.GetIdxByEthAddrBJJ(addr, pk) + idxR, err := sdb.GetIdxByEthAddrBJJ(addr, pk, tokenID0) assert.Nil(t, err) assert.Equal(t, idx, idxR) // expect error when getting only by EthAddr, as value does not exist // in the db for only EthAddr - _, err = sdb.GetIdxByEthAddr(addr) + _, err = sdb.GetIdxByEthAddr(addr, tokenID0) assert.Nil(t, err) - _, err = sdb.GetIdxByEthAddr(addr2) + _, err = sdb.GetIdxByEthAddr(addr2, tokenID0) + assert.NotNil(t, err) + // expect error when getting by EthAddr and BJJ, but for another TokenID + _, err = sdb.GetIdxByEthAddrBJJ(addr, pk, tokenID1) assert.NotNil(t, err) // expect to fail - idxR, err = sdb.GetIdxByEthAddrBJJ(addr2, pk) + idxR, err = sdb.GetIdxByEthAddrBJJ(addr2, pk, tokenID0) assert.NotNil(t, err) assert.Equal(t, common.Idx(0), idxR) - idxR, err = sdb.GetIdxByEthAddrBJJ(addr, pk2) + idxR, err = sdb.GetIdxByEthAddrBJJ(addr, pk2, tokenID0) assert.NotNil(t, err) assert.Equal(t, common.Idx(0), idxR) // try to store bigger idx, will not affect as already exist a smaller // Idx for that Addr & BJJ - err = sdb.setIdxByEthAddrBJJ(idx2, addr, pk) + err = sdb.setIdxByEthAddrBJJ(idx2, addr, pk, tokenID0) assert.Nil(t, err) // store smaller idx - err = sdb.setIdxByEthAddrBJJ(idx3, addr, pk) + err = sdb.setIdxByEthAddrBJJ(idx3, addr, pk, tokenID0) assert.Nil(t, err) - idxR, err = sdb.GetIdxByEthAddrBJJ(addr, pk) + idxR, err = sdb.GetIdxByEthAddrBJJ(addr, pk, tokenID0) assert.Nil(t, err) assert.Equal(t, idx3, idxR) // by EthAddr should work - idxR, err = sdb.GetIdxByEthAddr(addr) + idxR, err = sdb.GetIdxByEthAddr(addr, tokenID0) assert.Nil(t, err) assert.Equal(t, idx3, idxR) // expect error when trying to get Idx by addr2 & pk2 - idxR, err = sdb.GetIdxByEthAddrBJJ(addr2, pk2) + idxR, err = sdb.GetIdxByEthAddrBJJ(addr2, pk2, tokenID0) assert.NotNil(t, err) assert.Equal(t, ErrToIdxNotFound, err) assert.Equal(t, common.Idx(0), idxR) + // expect error when trying to get Idx by addr with not used TokenID + _, err = sdb.GetIdxByEthAddr(addr, tokenID1) + assert.NotNil(t, err) } func TestBJJCompressedTo256BigInt(t *testing.T) {
M test/til/txs.go → test/til/txs.go

@@ -30,7 +30,7 @@ type Context struct {

Instructions []instruction userNames []string Users map[string]*User - lastRegisteredTokenID common.TokenID + LastRegisteredTokenID common.TokenID l1CreatedAccounts map[string]*Account // rollupConstMaxL1UserTx Maximum L1-user transactions allowed to be queued in a batch

@@ -56,7 +56,7 @@ currBatchNum := 1 // The protocol defines the first batchNum to be 1

return &Context{ Users: make(map[string]*User), l1CreatedAccounts: make(map[string]*Account), - lastRegisteredTokenID: 0, + LastRegisteredTokenID: 0, rollupConstMaxL1UserTx: rollupConstMaxL1UserTx, idx: common.UserThreshold,

@@ -323,10 +323,10 @@ // Decimals: 18,

TokenID: inst.tokenID, EthBlockNum: tc.blockNum, } - if inst.tokenID != tc.lastRegisteredTokenID+1 { - return nil, fmt.Errorf("Line %d: AddToken TokenID should be sequential, expected TokenID: %d, defined TokenID: %d", inst.lineNum, tc.lastRegisteredTokenID+1, inst.tokenID) + if inst.tokenID != tc.LastRegisteredTokenID+1 { + return nil, fmt.Errorf("Line %d: AddToken TokenID should be sequential, expected TokenID: %d, defined TokenID: %d", inst.lineNum, tc.LastRegisteredTokenID+1, inst.tokenID) } - tc.lastRegisteredTokenID++ + tc.LastRegisteredTokenID++ tc.currBlock.AddedTokens = append(tc.currBlock.AddedTokens, newToken) default: return nil, fmt.Errorf("Line %d: Unexpected type: %s", inst.lineNum, inst.typ)

@@ -457,8 +457,8 @@ }

return nil } func (tc *Context) checkIfTokenIsRegistered(inst instruction) error { - if inst.tokenID > tc.lastRegisteredTokenID { - return fmt.Errorf("Can not process %s: TokenID %d not registered, last registered TokenID: %d", inst.typ, inst.tokenID, tc.lastRegisteredTokenID) + if inst.tokenID > tc.LastRegisteredTokenID { + return fmt.Errorf("Can not process %s: TokenID %d not registered, last registered TokenID: %d", inst.typ, inst.tokenID, tc.LastRegisteredTokenID) } return nil }
M txselector/txselector.go → txselector/txselector.go

@@ -67,34 +67,11 @@ }

return nil } -// GetL2TxSelection returns a selection of the L2Txs for the next batch, from the L2DB pool -func (txsel *TxSelector) GetL2TxSelection(coordIdxs []common.Idx, batchNum common.BatchNum) ([]common.PoolL2Tx, error) { - // get pending l2-tx from tx-pool - l2TxsRaw, err := txsel.l2db.GetPendingTxs() // once l2db ready, maybe use parameter 'batchNum' - if err != nil { - return nil, err - } - - // discard the txs that don't have an Account in the AccountDB - var validTxs txs - for _, tx := range l2TxsRaw { - _, err = txsel.localAccountsDB.GetAccount(tx.FromIdx) - if err == nil { - // if FromIdx has an account into the AccountsDB - validTxs = append(validTxs, tx) - } - } - - // get most profitable L2-tx - txs := txsel.getL2Profitable(validTxs, txsel.MaxTxs) - - // process the txs in the local AccountsDB - _, err = txsel.localAccountsDB.ProcessTxs(coordIdxs, nil, nil, txs) - if err != nil { - return nil, err - } - err = txsel.localAccountsDB.MakeCheckpoint() - return txs, err +// GetL2TxSelection returns the L1CoordinatorTxs and a selection of the L2Txs +// for the next batch, from the L2DB pool +func (txsel *TxSelector) GetL2TxSelection(coordIdxs []common.Idx, batchNum common.BatchNum) ([]common.L1Tx, []common.PoolL2Tx, error) { + _, l1CoordinatorTxs, l2Txs, err := txsel.GetL1L2TxSelection(coordIdxs, batchNum, []common.L1Tx{}) + return l1CoordinatorTxs, l2Txs, err } // GetL1L2TxSelection returns the selection of L1 + L2 txs

@@ -138,7 +115,7 @@ var accAuth *common.AccountCreationAuth

if l2TxsRaw[i].ToBJJ != nil { // case: ToBJJ!=0: // if idx exist for EthAddr&BJJ use it - _, err := txsel.localAccountsDB.GetIdxByEthAddrBJJ(l2TxsRaw[i].ToEthAddr, l2TxsRaw[i].ToBJJ) + _, err := txsel.localAccountsDB.GetIdxByEthAddrBJJ(l2TxsRaw[i].ToEthAddr, l2TxsRaw[i].ToBJJ, l2TxsRaw[i].TokenID) if err == nil { // account for ToEthAddr&ToBJJ already exist, // there is no need to create a new one.

@@ -163,7 +140,7 @@ validTxs = append(validTxs, l2TxsRaw[i])

} else { // case: ToBJJ==0: // if idx exist for EthAddr use it - _, err := txsel.localAccountsDB.GetIdxByEthAddr(l2TxsRaw[i].ToEthAddr) + _, err := txsel.localAccountsDB.GetIdxByEthAddr(l2TxsRaw[i].ToEthAddr, l2TxsRaw[i].TokenID) if err == nil { // account for ToEthAddr already exist, // there is no need to create a new one.

@@ -194,7 +171,7 @@ positionL1++

l1CoordinatorTxs = append(l1CoordinatorTxs, l1CoordinatorTx) } else if bytes.Equal(l2TxsRaw[i].ToEthAddr.Bytes(), common.FFAddr.Bytes()) && l2TxsRaw[i].ToBJJ != nil { // if idx exist for EthAddr&BJJ use it - _, err := txsel.localAccountsDB.GetIdxByEthAddrBJJ(l2TxsRaw[i].ToEthAddr, l2TxsRaw[i].ToBJJ) + _, err := txsel.localAccountsDB.GetIdxByEthAddrBJJ(l2TxsRaw[i].ToEthAddr, l2TxsRaw[i].ToBJJ, l2TxsRaw[i].TokenID) if err == nil { // account for ToEthAddr&ToBJJ already exist, (where ToEthAddr==0xff) // there is no need to create a new one.
M txselector/txselector_test.go → txselector/txselector_test.go

@@ -1,7 +1,26 @@

package txselector -/* - TODO update transactions generation +import ( + "io/ioutil" + "os" + "strconv" + "testing" + "time" + + ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/common" + dbUtils "github.com/hermeznetwork/hermez-node/db" + "github.com/hermeznetwork/hermez-node/db/historydb" + "github.com/hermeznetwork/hermez-node/db/l2db" + "github.com/hermeznetwork/hermez-node/db/statedb" + "github.com/hermeznetwork/hermez-node/eth" + "github.com/hermeznetwork/hermez-node/test" + "github.com/hermeznetwork/hermez-node/test/til" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + func initTest(t *testing.T, testSet string, maxL1UserTxs, maxL1OperatorTxs, maxTxs uint64) *TxSelector { pass := os.Getenv("POSTGRES_PASS") db, err := dbUtils.InitSQLDB(5432, "localhost", "hermez", pass, "hermez")

@@ -42,34 +61,67 @@ func TestGetL2TxSelection(t *testing.T) {

txsel := initTest(t, til.SetPool0, 5, 5, 10) test.CleanL2DB(txsel.l2db.DB()) - // generate test transactions - l1Txs, _, poolL2Txs, tokens := test.GenerateTestTxsFromSet(t, test.SetTest0) + tc := til.NewContext(eth.RollupConstMaxL1UserTx) + // generate test transactions + blocks, err := tc.GenerateBlocks(til.SetBlockchain0) + assert.Nil(t, err) + // poolL2Txs, err := tc.GeneratePoolL2Txs(til.SetPool0) + // assert.Nil(t, err) - // add tokens to HistoryDB to avoid breaking FK constrains - addTokens(t, tokens, txsel.l2db.DB()) - // add the first batch of transactions to the TxSelector - addL2Txs(t, txsel, poolL2Txs[0]) + coordIdxs := []common.Idx{256, 257, 258, 259} - _, err := txsel.GetL2TxSelection(0) - assert.Nil(t, err) + // add tokens to HistoryDB to avoid breaking FK constrains + var tokens []common.Token + for i := 0; i < int(tc.LastRegisteredTokenID); i++ { + tokens = append(tokens, common.Token{ + TokenID: common.TokenID(i), + EthBlockNum: 1, + EthAddr: ethCommon.BytesToAddress([]byte{byte(i)}), + Name: strconv.Itoa(i), + Symbol: strconv.Itoa(i), + Decimals: 18, + }) + } + addTokens(t, tokens, txsel.l2db.DB()) - _, _, _, err = txsel.GetL1L2TxSelection(0, l1Txs[0]) - assert.Nil(t, err) + // Process the 1st batch, which contains the L1CoordinatorTxs necessary + // to create the Coordinator accounts to receive the fees + _, err = txsel.localAccountsDB.ProcessTxs(nil, nil, blocks[0].Batches[0].L1CoordinatorTxs, nil) + require.Nil(t, err) - // TODO once L2DB is updated to return error in case that AddTxTest - // fails, and the Til is updated, update this test, checking that the - // selected PoolL2Tx are correctly sorted by Nonce + // add the 1st batch of transactions to the TxSelector + addL2Txs(t, txsel, common.L2TxsToPoolL2Txs(blocks[0].Batches[0].L2Txs)) + l1CoordTxs, l2Txs, err := txsel.GetL2TxSelection(coordIdxs, 0) + assert.Nil(t, err) + assert.Equal(t, 0, len(l2Txs)) + assert.Equal(t, 0, len(l1CoordTxs)) + _, _, _, err = txsel.GetL1L2TxSelection(coordIdxs, 0, blocks[0].L1UserTxs) + assert.Nil(t, err) - // txs, err := txsel.GetL2TxSelection(0) - // assert.Nil(t, err) - // for _, tx := range txs { - // fmt.Println(tx.FromIdx, tx.ToIdx, tx.AbsoluteFee) - // } - // assert.Equal(t, 3, len(txs)) - // assert.Equal(t, uint64(6), txs[0].AbsoluteFee) - // assert.Equal(t, uint64(5), txs[1].AbsoluteFee) - // assert.Equal(t, uint64(4), txs[2].AbsoluteFee) + // TODO once L2DB is updated to return error in case that AddTxTest + // fails, and the Til is updated, update this test, checking that the + // selected PoolL2Tx are correctly sorted by Nonce + + // TODO once L2DB is updated to store the parameter AbsoluteFee (which + // is used by TxSelector to sort L2Txs), uncomment this next lines of + // test, and put the expected value for + // l2Txs[len(l2Txs)-1].AbsoluteFee, which is the Tx which has the + // Fee==192. + /* + // add the 3rd batch of transactions to the TxSelector + addL2Txs(t, txsel, common.L2TxsToPoolL2Txs(blocks[0].Batches[2].L2Txs)) + + _, l2Txs, err = txsel.GetL2TxSelection(coordIdxs, 0) + assert.Nil(t, err) + for _, tx := range l2Txs { + fmt.Println(tx.FromIdx, tx.ToIdx, tx.AbsoluteFee) + } + require.Equal(t, 10, len(l2Txs)) + assert.Equal(t, float64(0), l2Txs[0].AbsoluteFee) + + fmt.Println(l2Txs[len(l2Txs)-1].Amount) + assert.Equal(t, float64(4), l2Txs[len(l2Txs)-1].AbsoluteFee) + */ } -*/