Merge pull request #216 from hermeznetwork/feature/batch-total-fee

Calulate total collected batch fee and fix merkleproof api format

api/dbtoapistructs.go

@@ -15,7 +15,6 @@ import (
 	"github.com/hermeznetwork/hermez-node/db/l2db"
 	"github.com/hermeznetwork/hermez-node/eth"
 	"github.com/iden3/go-iden3-crypto/babyjub"
-	"github.com/iden3/go-merkletree"
 )
 
 const exitIdx = "hez:EXIT:1"
@@ -171,26 +170,45 @@ func (e *exitsAPI) GetPagination() *db.Pagination {
 }
 func (e *exitsAPI) Len() int { return len(e.Exits) }
 
+type merkleProofAPI struct {
+	Root     string
+	Siblings []string
+	OldKey   string
+	OldValue string
+	IsOld0   bool
+	Key      string
+	Value    string
+	Fnc      int
+}
+
 type exitAPI struct {
-	ItemID                 int                             `json:"itemId"`
-	BatchNum               common.BatchNum                 `json:"batchNum"`
-	AccountIdx             string                          `json:"accountIndex"`
-	MerkleProof            *merkletree.CircomVerifierProof `json:"merkleProof"`
-	Balance                string                          `json:"balance"`
-	InstantWithdrawn       *int64                          `json:"instantWithdrawn"`
-	DelayedWithdrawRequest *int64                          `json:"delayedWithdrawRequest"`
-	DelayedWithdrawn       *int64                          `json:"delayedWithdrawn"`
-	Token                  tokenAPI                        `json:"token"`
+	ItemID                 int             `json:"itemId"`
+	BatchNum               common.BatchNum `json:"batchNum"`
+	AccountIdx             string          `json:"accountIndex"`
+	MerkleProof            merkleProofAPI  `json:"merkleProof"`
+	Balance                string          `json:"balance"`
+	InstantWithdrawn       *int64          `json:"instantWithdrawn"`
+	DelayedWithdrawRequest *int64          `json:"delayedWithdrawRequest"`
+	DelayedWithdrawn       *int64          `json:"delayedWithdrawn"`
+	Token                  tokenAPI        `json:"token"`
 }
 
 func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI {
 	apiExits := []exitAPI{}
 	for i := 0; i < len(dbExits); i++ {
-		apiExits = append(apiExits, exitAPI{
-			ItemID:                 dbExits[i].ItemID,
-			BatchNum:               dbExits[i].BatchNum,
-			AccountIdx:             idxToHez(dbExits[i].AccountIdx, dbExits[i].TokenSymbol),
-			MerkleProof:            dbExits[i].MerkleProof,
+		exit := exitAPI{
+			ItemID:     dbExits[i].ItemID,
+			BatchNum:   dbExits[i].BatchNum,
+			AccountIdx: idxToHez(dbExits[i].AccountIdx, dbExits[i].TokenSymbol),
+			MerkleProof: merkleProofAPI{
+				Root:     dbExits[i].MerkleProof.Root.String(),
+				OldKey:   dbExits[i].MerkleProof.OldKey.String(),
+				OldValue: dbExits[i].MerkleProof.OldValue.String(),
+				IsOld0:   dbExits[i].MerkleProof.IsOld0,
+				Key:      dbExits[i].MerkleProof.Key.String(),
+				Value:    dbExits[i].MerkleProof.Value.String(),
+				Fnc:      dbExits[i].MerkleProof.Fnc,
+			},
 			Balance:                dbExits[i].Balance.String(),
 			InstantWithdrawn:       dbExits[i].InstantWithdrawn,
 			DelayedWithdrawRequest: dbExits[i].DelayedWithdrawRequest,
@@ -205,7 +223,13 @@ func historyExitsToAPI(dbExits []historydb.HistoryExit) []exitAPI {
 				USD:         dbExits[i].TokenUSD,
 				USDUpdate:   dbExits[i].TokenUSDUpdate,
 			},
-		})
+		}
+		siblings := []string{}
+		for j := 0; j < len(dbExits[i].MerkleProof.Siblings); j++ {
+			siblings = append(siblings, dbExits[i].MerkleProof.Siblings[j].String())
+		}
+		exit.MerkleProof.Siblings = siblings
+		apiExits = append(apiExits, exit)
 	}
 	return apiExits
 }

api/swagger.yml

@@ -1968,33 +1968,25 @@ components:
           description: Existence proof of a leaf in a given Merkle Root. Encoded as hexadecimal string.
           properties:
               Root:
-                type: array
-                items: 
-                  type: integer
+                $ref: '#/components/schemas/BigInt'
               Siblings:
                 type: array
                 items:
-                  type: integer
+                  $ref: '#/components/schemas/BigInt'
               OldKey:
-                type: array
-                items:
-                  type: integer
+                $ref: '#/components/schemas/BigInt'
               OldValue:
-                type: array
-                items:
-                  type: integer
+                $ref: '#/components/schemas/BigInt'
               IsOld0:
                 type: boolean
               Key:
-                type: array
-                items:
-                  type: integer
+                $ref: '#/components/schemas/BigInt'
               Value:
-                type: array
-                items:
-                  type: integer
+                $ref: '#/components/schemas/BigInt'
               Fnc:
                 type: integer
+                maximum: 3
+                minimum: 0
           required:
             - Root
             - Siblings

common/batch.go

@@ -21,6 +21,7 @@ type Batch struct {
 	ExitRoot      Hash                 `meddler:"exit_root"`
 	ForgeL1TxsNum *int64               `meddler:"forge_l1_txs_num"` // optional, Only when the batch forges L1 txs. Identifier that corresponds to the group of L1 txs forged in the current batch.
 	SlotNum       SlotNum              `meddler:"slot_num"`         // Slot in which the batch is forged
+	TotalFeesUSD  *float64             `meddler:"total_fees_usd"`
 }
 
 // BatchNum identifies a batch

db/historydb/historydb.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"math"
 	"math/big"
 
 	ethCommon "github.com/ethereum/go-ethereum/common"
@@ -136,6 +137,44 @@ func (hdb *HistoryDB) GetLastBlock() (*common.Block, error) {
 // AddBatch insert a Batch into the DB
 func (hdb *HistoryDB) AddBatch(batch *common.Batch) error { return hdb.addBatch(hdb.db, batch) }
 func (hdb *HistoryDB) addBatch(d meddler.DB, batch *common.Batch) error {
+	// Calculate total collected fees in USD
+	// Get IDs of collected tokens for fees
+	tokenIDs := []common.TokenID{}
+	for id := range batch.CollectedFees {
+		tokenIDs = append(tokenIDs, id)
+	}
+	// Get USD value of the tokens
+	type tokenPrice struct {
+		ID       common.TokenID `meddler:"token_id"`
+		USD      *float64       `meddler:"usd"`
+		Decimals int            `meddler:"decimals"`
+	}
+	query, args, err := sqlx.In(
+		"SELECT token_id, usd, decimals FROM token WHERE token_id IN (?)",
+		tokenIDs,
+	)
+	if err != nil {
+		return err
+	}
+	query = hdb.db.Rebind(query)
+	var tokenPrices []*tokenPrice
+	if err := meddler.QueryAll(
+		hdb.db, &tokenPrices, query, args...,
+	); err != nil {
+		return err
+	}
+	// Calculate total collected
+	var total float64
+	for _, tokenPrice := range tokenPrices {
+		if tokenPrice.USD == nil {
+			continue
+		}
+		f := new(big.Float).SetInt(batch.CollectedFees[tokenPrice.ID])
+		amount, _ := f.Float64()
+		total += *tokenPrice.USD * (amount / math.Pow(10, float64(tokenPrice.Decimals))) //nolint decimals have to be ^10
+	}
+	batch.TotalFeesUSD = &total
+	// Insert to DB
 	return meddler.Insert(d, "batch", batch)
 }
 
@@ -144,22 +183,12 @@ func (hdb *HistoryDB) AddBatches(batches []common.Batch) error {
 	return hdb.addBatches(hdb.db, batches)
 }
 func (hdb *HistoryDB) addBatches(d meddler.DB, batches []common.Batch) error {
-	// TODO: Calculate and insert total_fees_usd
-	return db.BulkInsert(
-		d,
-		`INSERT INTO batch (
-			batch_num,
-			eth_block_num,
-			forger_addr,
-			fees_collected,
-			state_root,
-			num_accounts,
-			exit_root,
-			forge_l1_txs_num,
-			slot_num
-		) VALUES %s;`,
-		batches[:],
-	)
+	for i := 0; i < len(batches); i++ {
+		if err := hdb.addBatch(d, &batches[i]); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // GetBatches retrieve batches from the DB, given a range of batch numbers defined by from and to

db/historydb/historydb_test.go

@@ -1,6 +1,8 @@
 package historydb
 
 import (
+	"math"
+	"math/big"
 	"os"
 	"testing"
 
@@ -109,6 +111,34 @@ func TestBatches(t *testing.T) {
 	fetchedLastL1TxsNum, err = historyDB.GetLastL1TxsNum()
 	assert.NoError(t, err)
 	assert.Equal(t, *batches[nBatches-1].ForgeL1TxsNum, *fetchedLastL1TxsNum)
+	// Test total fee
+	// Generate fake tokens
+	const nTokens = 5
+	tokens := test.GenTokens(nTokens, blocks)
+	err = historyDB.AddTokens(tokens)
+	assert.NoError(t, err)
+	feeBatch := batches[0]
+	feeBatch.BatchNum = 9999
+	feeBatch.CollectedFees = make(map[common.TokenID]*big.Int)
+	var total float64
+	for i, token := range tokens {
+		value := 3.019237 * float64(i)
+		assert.NoError(t, historyDB.UpdateTokenValue(token.Symbol, value))
+		bigAmount := big.NewInt(345000000)
+		feeBatch.CollectedFees[token.TokenID] = bigAmount
+		f := new(big.Float).SetInt(bigAmount)
+		amount, _ := f.Float64()
+		total += value * (amount / math.Pow(10, float64(token.Decimals)))
+	}
+	err = historyDB.AddBatch(&feeBatch)
+	assert.NoError(t, err)
+	fetchedBatches, err = historyDB.GetBatches(feeBatch.BatchNum-1, feeBatch.BatchNum+1)
+	assert.NoError(t, err)
+	for _, fetchedBatch := range fetchedBatches {
+		if fetchedBatch.BatchNum == feeBatch.BatchNum {
+			assert.Equal(t, total, *fetchedBatch.TotalFeesUSD)
+		}
+	}
 }
 
 func TestBids(t *testing.T) {

test/historydb.go

@@ -37,7 +37,7 @@ func GenTokens(nTokens int, blocks []common.Block) []common.Token {
 			TokenID:     common.TokenID(i),
 			Name:        "NAME" + fmt.Sprint(i),
 			Symbol:      fmt.Sprint(i),
-			Decimals:    uint64(i),
+			Decimals:    uint64(i + 1),
 			EthBlockNum: blocks[i%len(blocks)].EthBlockNum,
 			EthAddr:     ethCommon.BigToAddress(big.NewInt(int64(i))),
 		}