From 900494fd8e0ac52fd08a6b5e9ec9bab5893b26ef Mon Sep 17 00:00:00 2001 From: arnaucube Date: Fri, 6 Nov 2020 16:28:57 +0100 Subject: [PATCH] StateDB group Fee txs and ZKInputs gen of Fee txs --- common/l2tx.go | 2 +- common/zk.go | 89 ++++++++++++++----------- db/statedb/statedb.go | 6 +- db/statedb/txprocessors.go | 132 ++++++++++++++++++++++++++----------- 4 files changed, 150 insertions(+), 79 deletions(-) diff --git a/common/l2tx.go b/common/l2tx.go index 2ce279c..d2a8f31 100644 --- a/common/l2tx.go +++ b/common/l2tx.go @@ -107,7 +107,7 @@ func L2TxsToPoolL2Txs(txs []L2Tx) []PoolL2Tx { func (tx *L2Tx) Bytes(nLevels int) ([]byte, error) { idxLen := nLevels / 8 //nolint:gomnd - b := make([]byte, ((nLevels*2)+16+8)/8) + b := make([]byte, ((nLevels*2)+16+8)/8) //nolint:gomnd fromIdxBytes, err := tx.FromIdx.Bytes() if err != nil { diff --git a/common/zk.go b/common/zk.go index a3e965f..17c6afb 100644 --- a/common/zk.go +++ b/common/zk.go @@ -1,6 +1,5 @@ // Package common contains all the common data structures used at the // hermez-node, zk.go contains the zkSnark inputs used to generate the proof -//nolint:deadcode,structcheck,unused package common import ( @@ -11,21 +10,30 @@ import ( "github.com/mitchellh/mapstructure" ) -// circuit parameters -// absolute maximum of L1 or L2 transactions allowed -type nTx uint32 - -// merkle tree depth -type nLevels uint32 - -// absolute maximum of L1 transaction allowed -type maxL1Tx uint32 - -//absolute maximum of fee transactions allowed -type maxFeeTx uint32 +// ZKMetadata contains ZKInputs metadata that is not used directly in the +// ZKInputs result, but to calculate values for Hash check +type ZKMetadata struct { + // Circuit parameters + // absolute maximum of L1 or L2 transactions allowed + NTx uint32 + // merkle tree depth + NLevels uint32 + MaxLevels uint32 + // absolute maximum of L1 transaction allowed + MaxL1Tx uint32 + // Maximum number of Idxs where Fees can be send in a batch (currently + // is constant for all circuits: 64) + MaxFeeIdxs uint32 + + L1TxsData [][]byte + L2TxsData [][]byte + ChainID uint16 +} // ZKInputs represents the inputs that will be used to generate the zkSNARK proof type ZKInputs struct { + Metadata ZKMetadata + // // General // @@ -38,11 +46,11 @@ type ZKInputs struct { // GlobalChainID is the blockchain ID (0 for Ethereum mainnet). This value can be get from the smart contract. GlobalChainID *big.Int `json:"globalChainID"` // uint16 // FeeIdxs is an array of merkle tree indexes where the coordinator will receive the accumulated fees - FeeIdxs []*big.Int `json:"feeIdxs"` // uint64 (max nLevels bits), len: [maxFeeTx] + FeeIdxs []*big.Int `json:"feeIdxs"` // uint64 (max nLevels bits), len: [maxFeeIdxs] // accumulate fees // FeePlanTokens contains all the tokenIDs for which the fees are being accumulated - FeePlanTokens []*big.Int `json:"feePlanTokens"` // uint32 (max 32 bits), len: [maxFeeTx] + FeePlanTokens []*big.Int `json:"feePlanTokens"` // uint32 (max 32 bits), len: [maxFeeIdxs] // // Txs (L1&L2) @@ -142,13 +150,13 @@ type ZKInputs struct { // state 3, value of the account leaf receiver of the Fees // fee tx // State fees - TokenID3 []*big.Int `json:"tokenID3"` // uint32, len: [maxFeeTx] - Nonce3 []*big.Int `json:"nonce3"` // uint64 (max 40 bits), len: [maxFeeTx] - Sign3 []*big.Int `json:"sign3"` // bool, len: [maxFeeTx] - Ay3 []*big.Int `json:"ay3"` // big.Int, len: [maxFeeTx] - Balance3 []*big.Int `json:"balance3"` // big.Int (max 192 bits), len: [maxFeeTx] - EthAddr3 []*big.Int `json:"ethAddr3"` // ethCommon.Address, len: [maxFeeTx] - Siblings3 [][]*big.Int `json:"siblings3"` // Hash, len: [maxFeeTx][nLevels + 1] + TokenID3 []*big.Int `json:"tokenID3"` // uint32, len: [maxFeeIdxs] + Nonce3 []*big.Int `json:"nonce3"` // uint64 (max 40 bits), len: [maxFeeIdxs] + Sign3 []*big.Int `json:"sign3"` // bool, len: [maxFeeIdxs] + Ay3 []*big.Int `json:"ay3"` // big.Int, len: [maxFeeIdxs] + Balance3 []*big.Int `json:"balance3"` // big.Int (max 192 bits), len: [maxFeeIdxs] + EthAddr3 []*big.Int `json:"ethAddr3"` // ethCommon.Address, len: [maxFeeIdxs] + Siblings3 [][]*big.Int `json:"siblings3"` // Hash, len: [maxFeeIdxs][nLevels + 1] // // Intermediate States @@ -174,14 +182,14 @@ type ZKInputs struct { // ISExitTree root at the moment of the Tx the value once the Tx is processed into the exit tree ISExitRoot []*big.Int `json:"imExitRoot"` // Hash, len: [nTx - 1] // ISAccFeeOut accumulated fees once the Tx is processed - ISAccFeeOut [][]*big.Int `json:"imAccFeeOut"` // big.Int, len: [nTx - 1][maxFeeTx] + ISAccFeeOut [][]*big.Int `json:"imAccFeeOut"` // big.Int, len: [nTx - 1][maxFeeIdxs] // fee-tx // ISStateRootFee root at the moment of the Tx, the state root value once the Tx is processed into the state tree - ISStateRootFee []*big.Int `json:"imStateRootFee"` // Hash, len: [maxFeeTx - 1] + ISStateRootFee []*big.Int `json:"imStateRootFee"` // Hash, len: [maxFeeIdxs - 1] // ISInitStateRootFee state root once all L1-L2 tx are processed (before computing the fees-tx) ISInitStateRootFee *big.Int `json:"imInitStateRootFee"` // Hash // ISFinalAccFee final accumulated fees (before computing the fees-tx) - ISFinalAccFee []*big.Int `json:"imFinalAccFee"` // big.Int, len: [maxFeeTx - 1] + ISFinalAccFee []*big.Int `json:"imFinalAccFee"` // big.Int, len: [maxFeeIdxs - 1] } func bigIntsToStrings(v interface{}) interface{} { @@ -212,6 +220,8 @@ func bigIntsToStrings(v interface{}) interface{} { r[i] = bigIntsToStrings(c[i]) } return r + case map[string]interface{}: + // avoid printing a warning when there is a struct type default: log.Warnf("bigIntsToStrings unexpected type: %T\n", v) } @@ -240,15 +250,18 @@ func (z ZKInputs) MarshalJSON() ([]byte, error) { } // NewZKInputs returns a pointer to an initialized struct of ZKInputs -func NewZKInputs(nTx, maxFeeTx, nLevels int) *ZKInputs { +func NewZKInputs(nTx, maxFeeIdxs, nLevels int) *ZKInputs { zki := &ZKInputs{} + zki.Metadata.NTx = uint32(nTx) + zki.Metadata.MaxFeeIdxs = uint32(maxFeeIdxs) + zki.Metadata.NLevels = uint32(nLevels) // General zki.OldLastIdx = big.NewInt(0) zki.OldStateRoot = big.NewInt(0) zki.GlobalChainID = big.NewInt(0) - zki.FeeIdxs = newSlice(maxFeeTx) - zki.FeePlanTokens = newSlice(maxFeeTx) + zki.FeeIdxs = newSlice(maxFeeIdxs) + zki.FeePlanTokens = newSlice(maxFeeIdxs) // Txs zki.TxCompressedData = newSlice(nTx) @@ -312,13 +325,13 @@ func NewZKInputs(nTx, maxFeeTx, nLevels int) *ZKInputs { zki.OldKey2 = newSlice(nTx) zki.OldValue2 = newSlice(nTx) - zki.TokenID3 = newSlice(maxFeeTx) - zki.Nonce3 = newSlice(maxFeeTx) - zki.Sign3 = newSlice(maxFeeTx) - zki.Ay3 = newSlice(maxFeeTx) - zki.Balance3 = newSlice(maxFeeTx) - zki.EthAddr3 = newSlice(maxFeeTx) - zki.Siblings3 = make([][]*big.Int, maxFeeTx) + zki.TokenID3 = newSlice(maxFeeIdxs) + zki.Nonce3 = newSlice(maxFeeIdxs) + zki.Sign3 = newSlice(maxFeeIdxs) + zki.Ay3 = newSlice(maxFeeIdxs) + zki.Balance3 = newSlice(maxFeeIdxs) + zki.EthAddr3 = newSlice(maxFeeIdxs) + zki.Siblings3 = make([][]*big.Int, maxFeeIdxs) for i := 0; i < len(zki.Siblings3); i++ { zki.Siblings3[i] = newSlice(nLevels + 1) } @@ -330,11 +343,11 @@ func NewZKInputs(nTx, maxFeeTx, nLevels int) *ZKInputs { zki.ISExitRoot = newSlice(nTx - 1) zki.ISAccFeeOut = make([][]*big.Int, nTx-1) for i := 0; i < len(zki.ISAccFeeOut); i++ { - zki.ISAccFeeOut[i] = newSlice(maxFeeTx) + zki.ISAccFeeOut[i] = newSlice(maxFeeIdxs) } - zki.ISStateRootFee = newSlice(maxFeeTx - 1) + zki.ISStateRootFee = newSlice(maxFeeIdxs - 1) zki.ISInitStateRootFee = big.NewInt(0) - zki.ISFinalAccFee = newSlice(maxFeeTx - 1) + zki.ISFinalAccFee = newSlice(maxFeeIdxs - 1) return zki } diff --git a/db/statedb/statedb.go b/db/statedb/statedb.go index f4931db..c3c0096 100644 --- a/db/statedb/statedb.go +++ b/db/statedb/statedb.go @@ -78,7 +78,11 @@ type StateDB struct { // idx holds the current Idx that the BatchBuilder is using idx common.Idx zki *common.ZKInputs - i int // i is the current transaction index in the ZKInputs generation (zki) + // i is the current transaction index in the ZKInputs generation (zki) + i int + // accumulatedFees contains the accumulated fees for each token (Coord + // Idx) in the processed batch + accumulatedFees map[common.Idx]*big.Int } // NewStateDB creates a new StateDB, allowing to use an in-memory or in-disk diff --git a/db/statedb/txprocessors.go b/db/statedb/txprocessors.go index 532de30..fbaf1c5 100644 --- a/db/statedb/txprocessors.go +++ b/db/statedb/txprocessors.go @@ -64,6 +64,8 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs } defer s.resetZKInputs() + s.accumulatedFees = make(map[common.Idx]*big.Int) + nTx := len(l1usertxs) + len(l1coordinatortxs) + len(l2txs) if nTx == 0 { // TODO return ZKInputs of batch without txs @@ -77,22 +79,8 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs } exits := make([]processedExit, nTx) - // get TokenIDs of coordIdxs - coordIdxsMap, err := s.getTokenIDsFromIdxs(coordIdxs) - if err != nil { - return nil, err - } - - var collectedFees map[common.TokenID]*big.Int - if s.typ == TypeSynchronizer { - collectedFees = make(map[common.TokenID]*big.Int) - for tokenID := range coordIdxsMap { - collectedFees[tokenID] = big.NewInt(0) - } - } - if s.typ == TypeBatchBuilder { - maxFeeTx := 2 // TODO this value will be a parameter + maxFeeTx := 64 // TODO this value will be a parameter s.zki = common.NewZKInputs(nTx, maxFeeTx, s.mt.MaxLevels()) s.zki.OldLastIdx = (s.idx - 1).BigInt() s.zki.OldStateRoot = s.mt.Root().BigInt() @@ -120,8 +108,9 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs } } - // assumption: l1usertx are sorted by L1Tx.Position + // Process L1UserTxs for i := 0; i < len(l1usertxs); i++ { + // assumption: l1usertx are sorted by L1Tx.Position exitIdx, exitAccount, newExit, createdAccount, err := s.processL1Tx(exitTree, &l1usertxs[i]) if err != nil { return nil, err @@ -140,7 +129,17 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs if s.typ == TypeSynchronizer && createdAccount != nil { createdAccounts = append(createdAccounts, *createdAccount) } + + if s.zki != nil { + l1TxData, err := l1usertxs[i].BytesGeneric() + if err != nil { + return nil, err + } + s.zki.Metadata.L1TxsData = append(s.zki.Metadata.L1TxsData, l1TxData) + } } + + // Process L1CoordinatorTxs for i := 0; i < len(l1coordinatortxs); i++ { exitIdx, _, _, createdAccount, err := s.processL1Tx(exitTree, &l1coordinatortxs[i]) if err != nil { @@ -152,7 +151,36 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs if s.typ == TypeSynchronizer && createdAccount != nil { createdAccounts = append(createdAccounts, *createdAccount) } + if s.zki != nil { + l1TxData, err := l1coordinatortxs[i].BytesGeneric() + if err != nil { + return nil, err + } + s.zki.Metadata.L1TxsData = append(s.zki.Metadata.L1TxsData, l1TxData) + } + } + + s.accumulatedFees = make(map[common.Idx]*big.Int) + for _, idx := range coordIdxs { + s.accumulatedFees[idx] = big.NewInt(0) + } + + // once L1UserTxs & L1CoordinatorTxs are processed, get TokenIDs of + // coordIdxs. In this way, if a coordIdx uses an Idx that is being + // created in the current batch, at this point the Idx will be created + coordIdxsMap, err := s.getTokenIDsFromIdxs(coordIdxs) + if err != nil { + return nil, err } + var collectedFees map[common.TokenID]*big.Int + if s.typ == TypeSynchronizer { + collectedFees = make(map[common.TokenID]*big.Int) + for tokenID := range coordIdxsMap { + collectedFees[tokenID] = big.NewInt(0) + } + } + + // Process L2Txs for i := 0; i < len(l2txs); i++ { exitIdx, exitAccount, newExit, err := s.processL2Tx(coordIdxsMap, collectedFees, exitTree, &l2txs[i]) if err != nil { @@ -171,6 +199,39 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs } } + // distribute the AccumulatedFees from the processed L2Txs into the + // Coordinator Idxs + iFee := 0 + for idx, accumulatedFee := range s.accumulatedFees { + // send the fee to the Idx of the Coordinator for the TokenID + accCoord, err := s.GetAccount(idx) + if err != nil { + log.Errorw("Can not distribute accumulated fees to coordinator account: No coord Idx to receive fee", "idx", idx) + return nil, err + } + accCoord.Balance = new(big.Int).Add(accCoord.Balance, accumulatedFee) + pFee, err := s.UpdateAccount(idx, accCoord) + if err != nil { + log.Error(err) + return nil, err + } + if s.zki != nil { + s.zki.TokenID3[iFee] = accCoord.TokenID.BigInt() + s.zki.Nonce3[iFee] = accCoord.Nonce.BigInt() + if babyjub.PointCoordSign(accCoord.PublicKey.X) { + s.zki.Sign3[iFee] = big.NewInt(1) + } + s.zki.Ay3[iFee] = accCoord.PublicKey.Y + s.zki.Balance3[iFee] = accCoord.Balance + s.zki.EthAddr3[iFee] = common.EthAddrToBigInt(accCoord.EthAddr) + s.zki.Siblings3[iFee] = siblingsToZKInputFormat(pFee.Siblings) + + // add Coord Idx to ZKInputs.FeeTxsData + s.zki.FeeIdxs[iFee] = idx.BigInt() + } + iFee++ + } + if s.typ == TypeTxSelector { return nil, nil } @@ -178,7 +239,6 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs // once all txs processed (exitTree root frozen), for each Exit, // generate common.ExitInfo data var exitInfos []common.ExitInfo - // exitInfos := []common.ExitInfo{} for i := 0; i < nTx; i++ { if !exits[i].exit { continue @@ -245,9 +305,6 @@ func (s *StateDB) ProcessTxs(coordIdxs []common.Idx, l1usertxs, l1coordinatortxs // s.zki.ISInitStateRootFee = s.mt.Root().BigInt() - // TODO once the Node Config sets the Accounts where to send the Fees - // compute fees & update ZKInputs - // return ZKInputs as the BatchBuilder will return it to forge the Batch return &ProcessTxOutput{ ZKInputs: s.zki, @@ -606,10 +663,10 @@ func (s *StateDB) applyTransfer(coordIdxsMap map[common.TokenID]common.Idx, coll return err } - // increment nonce - accSender.Nonce++ - if !tx.IsL1 { + // increment nonce + accSender.Nonce++ + // compute fee and subtract it from the accSender fee, err := common.CalcFeeAmount(tx.Amount, *tx.Fee) if err != nil { @@ -618,19 +675,16 @@ func (s *StateDB) applyTransfer(coordIdxsMap map[common.TokenID]common.Idx, coll feeAndAmount := new(big.Int).Add(tx.Amount, fee) accSender.Balance = new(big.Int).Sub(accSender.Balance, feeAndAmount) - // send the fee to the Idx of the Coordinator for the TokenID accCoord, err := s.GetAccount(coordIdxsMap[accSender.TokenID]) if err != nil { log.Debugw("No coord Idx to receive fee", "tx", tx) } else { - accCoord.Balance = new(big.Int).Add(accCoord.Balance, fee) - _, err = s.UpdateAccount(coordIdxsMap[accSender.TokenID], accCoord) - if err != nil { - log.Error(err) - return err - } + // accumulate the fee for the Coord account + accumulated := s.accumulatedFees[accCoord.Idx] + accumulated.Add(accumulated, fee) + if s.typ == TypeSynchronizer { - collected := collectedFees[accSender.TokenID] + collected := collectedFees[accCoord.TokenID] collected.Add(collected, fee) } } @@ -753,6 +807,9 @@ func (s *StateDB) applyExit(coordIdxsMap map[common.TokenID]common.Idx, collecte } if !tx.IsL1 { + // increment nonce + acc.Nonce++ + // compute fee and subtract it from the accSender fee, err := common.CalcFeeAmount(tx.Amount, *tx.Fee) if err != nil { @@ -761,19 +818,16 @@ func (s *StateDB) applyExit(coordIdxsMap map[common.TokenID]common.Idx, collecte feeAndAmount := new(big.Int).Add(tx.Amount, fee) acc.Balance = new(big.Int).Sub(acc.Balance, feeAndAmount) - // send the fee to the Idx of the Coordinator for the TokenID accCoord, err := s.GetAccount(coordIdxsMap[acc.TokenID]) if err != nil { log.Debugw("No coord Idx to receive fee", "tx", tx) } else { - accCoord.Balance = new(big.Int).Add(accCoord.Balance, fee) - _, err = s.UpdateAccount(coordIdxsMap[acc.TokenID], accCoord) - if err != nil { - log.Error(err) - return nil, false, err - } + // accumulate the fee for the Coord account + accumulated := s.accumulatedFees[accCoord.Idx] + accumulated.Add(accumulated, fee) + if s.typ == TypeSynchronizer { - collected := collectedFees[acc.TokenID] + collected := collectedFees[accCoord.TokenID] collected.Add(collected, fee) } }