Merge pull request #647 from hermeznetwork/fix/apistatecollectedfees

Fix API /state collected fees unmarshal

api/batch_test.go

@@ -7,10 +7,12 @@ import (
 	"time"
 
 	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/apitypes"
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db/historydb"
 	"github.com/mitchellh/copystructure"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type testBatch struct {
@@ -20,7 +22,7 @@ type testBatch struct {
 	EthBlockHash  ethCommon.Hash            `json:"ethereumBlockHash"`
 	Timestamp     time.Time                 `json:"timestamp"`
 	ForgerAddr    ethCommon.Address         `json:"forgerAddr"`
-	CollectedFees map[common.TokenID]string `json:"collectedFees"`
+	CollectedFees apitypes.CollectedFeesAPI `json:"collectedFees"`
 	TotalFeesUSD  *float64                  `json:"historicTotalCollectedFeesUSD"`
 	StateRoot     string                    `json:"stateRoot"`
 	NumAccounts   int                       `json:"numAccounts"`
@@ -73,9 +75,9 @@ func genTestBatches(
 		if !found {
 			panic("block not found")
 		}
-		collectedFees := make(map[common.TokenID]string)
+		collectedFees := apitypes.CollectedFeesAPI(make(map[common.TokenID]apitypes.BigIntStr))
 		for k, v := range cBatches[i].CollectedFees {
-			collectedFees[k] = v.String()
+			collectedFees[k] = *apitypes.NewBigIntStr(v)
 		}
 		forgedTxs := 0
 		for _, tx := range txs {
@@ -132,7 +134,7 @@ func TestGetBatches(t *testing.T) {
 	limit := 3
 	path := fmt.Sprintf("%s?limit=%d", endpoint, limit)
 	err := doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	assertBatches(t, tc.batches, fetchedBatches)
 
 	// minBatchNum
@@ -141,7 +143,7 @@ func TestGetBatches(t *testing.T) {
 	minBatchNum := tc.batches[len(tc.batches)/2].BatchNum
 	path = fmt.Sprintf("%s?minBatchNum=%d&limit=%d", endpoint, minBatchNum, limit)
 	err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	minBatchNumBatches := []testBatch{}
 	for i := 0; i < len(tc.batches); i++ {
 		if tc.batches[i].BatchNum > minBatchNum {
@@ -156,7 +158,7 @@ func TestGetBatches(t *testing.T) {
 	maxBatchNum := tc.batches[len(tc.batches)/2].BatchNum
 	path = fmt.Sprintf("%s?maxBatchNum=%d&limit=%d", endpoint, maxBatchNum, limit)
 	err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	maxBatchNumBatches := []testBatch{}
 	for i := 0; i < len(tc.batches); i++ {
 		if tc.batches[i].BatchNum < maxBatchNum {
@@ -171,7 +173,7 @@ func TestGetBatches(t *testing.T) {
 	slotNum := tc.batches[len(tc.batches)/2].SlotNum
 	path = fmt.Sprintf("%s?slotNum=%d&limit=%d", endpoint, slotNum, limit)
 	err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	slotNumBatches := []testBatch{}
 	for i := 0; i < len(tc.batches); i++ {
 		if tc.batches[i].SlotNum == slotNum {
@@ -186,7 +188,7 @@ func TestGetBatches(t *testing.T) {
 	forgerAddr := tc.batches[len(tc.batches)/2].ForgerAddr
 	path = fmt.Sprintf("%s?forgerAddr=%s&limit=%d", endpoint, forgerAddr.String(), limit)
 	err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	forgerAddrBatches := []testBatch{}
 	for i := 0; i < len(tc.batches); i++ {
 		if tc.batches[i].ForgerAddr == forgerAddr {
@@ -200,7 +202,7 @@ func TestGetBatches(t *testing.T) {
 	limit = 6
 	path = fmt.Sprintf("%s?limit=%d", endpoint, limit)
 	err = doGoodReqPaginated(path, historydb.OrderDesc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	flippedBatches := []testBatch{}
 	for i := len(tc.batches) - 1; i >= 0; i-- {
 		flippedBatches = append(flippedBatches, tc.batches[i])
@@ -214,7 +216,7 @@ func TestGetBatches(t *testing.T) {
 	minBatchNum = tc.batches[len(tc.batches)/4].BatchNum
 	path = fmt.Sprintf("%s?minBatchNum=%d&maxBatchNum=%d&limit=%d", endpoint, minBatchNum, maxBatchNum, limit)
 	err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	minMaxBatchNumBatches := []testBatch{}
 	for i := 0; i < len(tc.batches); i++ {
 		if tc.batches[i].BatchNum < maxBatchNum && tc.batches[i].BatchNum > minBatchNum {
@@ -227,25 +229,25 @@ func TestGetBatches(t *testing.T) {
 	fetchedBatches = []testBatch{}
 	path = fmt.Sprintf("%s?slotNum=%d&minBatchNum=%d", endpoint, 1, 25)
 	err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	assertBatches(t, []testBatch{}, fetchedBatches)
 
 	// 400
 	// Invalid minBatchNum
 	path = fmt.Sprintf("%s?minBatchNum=%d", endpoint, -2)
 	err = doBadReq("GET", path, nil, 400)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	// Invalid forgerAddr
 	path = fmt.Sprintf("%s?forgerAddr=%s", endpoint, "0xG0000001")
 	err = doBadReq("GET", path, nil, 400)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 }
 
 func TestGetBatch(t *testing.T) {
 	endpoint := apiURL + "batches/"
 	for _, batch := range tc.batches {
 		fetchedBatch := testBatch{}
-		assert.NoError(
+		require.NoError(
 			t, doGoodReq(
 				"GET",
 				endpoint+strconv.Itoa(int(batch.BatchNum)),
@@ -255,16 +257,16 @@ func TestGetBatch(t *testing.T) {
 		assertBatch(t, batch, fetchedBatch)
 	}
 	// 400
-	assert.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400))
+	require.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400))
 	// 404
-	assert.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404))
+	require.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404))
 }
 
 func TestGetFullBatch(t *testing.T) {
 	endpoint := apiURL + "full-batches/"
 	for _, fullBatch := range tc.fullBatches {
 		fetchedFullBatch := testFullBatch{}
-		assert.NoError(
+		require.NoError(
 			t, doGoodReq(
 				"GET",
 				endpoint+strconv.Itoa(int(fullBatch.Batch.BatchNum)),
@@ -275,9 +277,9 @@ func TestGetFullBatch(t *testing.T) {
 		assertTxs(t, fullBatch.Txs, fetchedFullBatch.Txs)
 	}
 	// 400
-	assert.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400))
+	require.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400))
 	// 404
-	assert.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404))
+	require.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404))
 }
 
 func assertBatches(t *testing.T, expected, actual []testBatch) {

apitypes/apitypes.go

@@ -4,7 +4,6 @@ import (
 	"database/sql/driver"
 	"encoding/base64"
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
@@ -73,22 +72,16 @@ func (s *StrBigInt) UnmarshalText(text []byte) error {
 	return nil
 }
 
-// CollectedFees is used to retrieve common.batch.CollectedFee from the DB
+// CollectedFeesAPI is send common.batch.CollectedFee through the API
-type CollectedFees map[common.TokenID]BigIntStr
+type CollectedFeesAPI map[common.TokenID]BigIntStr
 
-// UnmarshalJSON unmarshals a json representation of map[common.TokenID]*big.Int
+// NewCollectedFeesAPI creates a new CollectedFeesAPI from a *big.Int map
-func (c *CollectedFees) UnmarshalJSON(text []byte) error {
+func NewCollectedFeesAPI(m map[common.TokenID]*big.Int) CollectedFeesAPI {
-	bigIntMap := make(map[common.TokenID]*big.Int)
+	c := CollectedFeesAPI(make(map[common.TokenID]BigIntStr))
-	if err := json.Unmarshal(text, &bigIntMap); err != nil {
+	for k, v := range m {
-		return tracerr.Wrap(err)
+		c[k] = *NewBigIntStr(v)
 	}
-	*c = CollectedFees(make(map[common.TokenID]BigIntStr))
+	return c
-	for k, v := range bigIntMap {
-		bStr := NewBigIntStr(v)
-		(CollectedFees(*c)[k]) = *bStr
-	}
-	// *c = CollectedFees(bStrMap)
-	return nil
 }
 
 // HezEthAddr is used to scan/value Ethereum Address directly into strings that follow the Ethereum address hez format (^hez:0x[a-fA-F0-9]{40}$) from/to sql DBs.

db/historydb/apiqueries.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/apitypes"
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db"
 	"github.com/hermeznetwork/tracerr"
@@ -45,7 +46,7 @@ func (hdb *HistoryDB) GetBatchInternalAPI(batchNum common.BatchNum) (*BatchAPI,
 
 func (hdb *HistoryDB) getBatchAPI(d meddler.DB, batchNum common.BatchNum) (*BatchAPI, error) {
 	batch := &BatchAPI{}
-	return batch, tracerr.Wrap(meddler.QueryRow(
+	if err := meddler.QueryRow(
 		d, batch,
 		`SELECT batch.item_id, batch.batch_num, batch.eth_block_num,
 		batch.forger_addr, batch.fees_collected, batch.total_fees_usd, batch.state_root,
@@ -54,7 +55,11 @@ func (hdb *HistoryDB) getBatchAPI(d meddler.DB, batchNum common.BatchNum) (*Batc
 	    COALESCE ((SELECT COUNT(*) FROM tx WHERE batch_num = batch.batch_num), 0) AS forged_txs
 	    FROM batch INNER JOIN block ON batch.eth_block_num = block.eth_block_num
 	 	WHERE batch_num = $1;`, batchNum,
-	))
+	); err != nil {
+		return nil, tracerr.Wrap(err)
+	}
+	batch.CollectedFeesAPI = apitypes.NewCollectedFeesAPI(batch.CollectedFeesDB)
+	return batch, nil
 }
 
 // GetBatchesAPI return the batches applying the given filters
@@ -155,6 +160,9 @@ func (hdb *HistoryDB) GetBatchesAPI(
 	if len(batches) == 0 {
 		return batches, 0, nil
 	}
+	for i := range batches {
+		batches[i].CollectedFeesAPI = apitypes.NewCollectedFeesAPI(batches[i].CollectedFeesDB)
+	}
 	return batches, batches[0].TotalItems - uint64(len(batches)), nil
 }
 

db/historydb/historydb_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/apitypes"
 	"github.com/hermeznetwork/hermez-node/common"
 	dbUtils "github.com/hermeznetwork/hermez-node/db"
 	"github.com/hermeznetwork/hermez-node/log"
@@ -1479,6 +1480,10 @@ func TestNodeInfo(t *testing.T) {
 	require.NoError(t, err)
 
 	// Test parameters
+	var f64 float64 = 1.2
+	var i64 int64 = 8888
+	addr := ethCommon.HexToAddress("0x1234")
+	hash := ethCommon.HexToHash("0x5678")
 	stateAPI := &StateAPI{
 		NodePublicConfig: NodePublicConfig{
 			ForgeDelay: 3.1,
@@ -1486,6 +1491,56 @@ func TestNodeInfo(t *testing.T) {
 		Network: NetworkAPI{
 			LastEthBlock:  12,
 			LastSyncBlock: 34,
+			LastBatch: &BatchAPI{
+				ItemID:       123,
+				BatchNum:     456,
+				EthBlockNum:  789,
+				EthBlockHash: hash,
+				Timestamp:    time.Now(),
+				ForgerAddr:   addr,
+				// CollectedFeesDB: map[common.TokenID]*big.Int{
+				// 	0: big.NewInt(11111),
+				// 	1: big.NewInt(21111),
+				// 	2: big.NewInt(31111),
+				// },
+				CollectedFeesAPI: apitypes.CollectedFeesAPI(map[common.TokenID]apitypes.BigIntStr{
+					0: apitypes.BigIntStr("11111"),
+					1: apitypes.BigIntStr("21111"),
+					2: apitypes.BigIntStr("31111"),
+				}),
+				TotalFeesUSD:  &f64,
+				StateRoot:     apitypes.BigIntStr("1234"),
+				NumAccounts:   11,
+				ExitRoot:      apitypes.BigIntStr("5678"),
+				ForgeL1TxsNum: &i64,
+				SlotNum:       44,
+				ForgedTxs:     23,
+				TotalItems:    0,
+				FirstItem:     0,
+				LastItem:      0,
+			},
+			CurrentSlot: 22,
+			NextForgers: []NextForgerAPI{
+				{
+					Coordinator: CoordinatorAPI{
+						ItemID:      111,
+						Bidder:      addr,
+						Forger:      addr,
+						EthBlockNum: 566,
+						URL:         "asd",
+						TotalItems:  0,
+						FirstItem:   0,
+						LastItem:    0,
+					},
+					Period: Period{
+						SlotNum:       33,
+						FromBlock:     55,
+						ToBlock:       66,
+						FromTimestamp: time.Now(),
+						ToTimestamp:   time.Now(),
+					},
+				},
+			},
 		},
 		Metrics: MetricsAPI{
 			TransactionsPerBatch: 1.1,
@@ -1518,5 +1573,14 @@ func TestNodeInfo(t *testing.T) {
 
 	dbStateAPI, err := historyDB.getStateAPI(historyDB.dbRead)
 	require.NoError(t, err)
+	assert.Equal(t, stateAPI.Network.LastBatch.Timestamp.Unix(),
+		dbStateAPI.Network.LastBatch.Timestamp.Unix())
+	dbStateAPI.Network.LastBatch.Timestamp = stateAPI.Network.LastBatch.Timestamp
+	assert.Equal(t, stateAPI.Network.NextForgers[0].Period.FromTimestamp.Unix(),
+		dbStateAPI.Network.NextForgers[0].Period.FromTimestamp.Unix())
+	dbStateAPI.Network.NextForgers[0].Period.FromTimestamp = stateAPI.Network.NextForgers[0].Period.FromTimestamp
+	assert.Equal(t, stateAPI.Network.NextForgers[0].Period.ToTimestamp.Unix(),
+		dbStateAPI.Network.NextForgers[0].Period.ToTimestamp.Unix())
+	dbStateAPI.Network.NextForgers[0].Period.ToTimestamp = stateAPI.Network.NextForgers[0].Period.ToTimestamp
 	assert.Equal(t, stateAPI, dbStateAPI)
 }

db/historydb/nodeinfo.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/apitypes"
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/tracerr"
 	"github.com/russross/meddler"
@@ -124,6 +125,10 @@ func (hdb *HistoryDB) getStateAPI(d meddler.DB) (*StateAPI, error) {
 
 // SetStateInternalAPI sets the StateAPI
 func (hdb *HistoryDB) SetStateInternalAPI(stateAPI *StateAPI) error {
+	if stateAPI.Network.LastBatch != nil {
+		stateAPI.Network.LastBatch.CollectedFeesAPI =
+			apitypes.NewCollectedFeesAPI(stateAPI.Network.LastBatch.CollectedFeesDB)
+	}
 	_stateAPI := struct {
 		StateAPI *StateAPI `meddler:"state,json"`
 	}{stateAPI}

db/historydb/views.go

@@ -289,23 +289,24 @@ func (account AccountAPI) MarshalJSON() ([]byte, error) {
 // BatchAPI is a representation of a batch with additional information
 // required by the API, and extracted by joining block table
 type BatchAPI struct {
-	ItemID        uint64                 `json:"itemId" meddler:"item_id"`
+	ItemID           uint64                      `json:"itemId" meddler:"item_id"`
-	BatchNum      common.BatchNum        `json:"batchNum" meddler:"batch_num"`
+	BatchNum         common.BatchNum             `json:"batchNum" meddler:"batch_num"`
-	EthBlockNum   int64                  `json:"ethereumBlockNum" meddler:"eth_block_num"`
+	EthBlockNum      int64                       `json:"ethereumBlockNum" meddler:"eth_block_num"`
-	EthBlockHash  ethCommon.Hash         `json:"ethereumBlockHash" meddler:"hash"`
+	EthBlockHash     ethCommon.Hash              `json:"ethereumBlockHash" meddler:"hash"`
-	Timestamp     time.Time              `json:"timestamp" meddler:"timestamp,utctime"`
+	Timestamp        time.Time                   `json:"timestamp" meddler:"timestamp,utctime"`
-	ForgerAddr    ethCommon.Address      `json:"forgerAddr" meddler:"forger_addr"`
+	ForgerAddr       ethCommon.Address           `json:"forgerAddr" meddler:"forger_addr"`
-	CollectedFees apitypes.CollectedFees `json:"collectedFees" meddler:"fees_collected,json"`
+	CollectedFeesDB  map[common.TokenID]*big.Int `json:"-" meddler:"fees_collected,json"`
-	TotalFeesUSD  *float64               `json:"historicTotalCollectedFeesUSD" meddler:"total_fees_usd"`
+	CollectedFeesAPI apitypes.CollectedFeesAPI   `json:"collectedFees" meddler:"-"`
-	StateRoot     apitypes.BigIntStr     `json:"stateRoot" meddler:"state_root"`
+	TotalFeesUSD     *float64                    `json:"historicTotalCollectedFeesUSD" meddler:"total_fees_usd"`
-	NumAccounts   int                    `json:"numAccounts" meddler:"num_accounts"`
+	StateRoot        apitypes.BigIntStr          `json:"stateRoot" meddler:"state_root"`
-	ExitRoot      apitypes.BigIntStr     `json:"exitRoot" meddler:"exit_root"`
+	NumAccounts      int                         `json:"numAccounts" meddler:"num_accounts"`
-	ForgeL1TxsNum *int64                 `json:"forgeL1TransactionsNum" meddler:"forge_l1_txs_num"`
+	ExitRoot         apitypes.BigIntStr          `json:"exitRoot" meddler:"exit_root"`
-	SlotNum       int64                  `json:"slotNum" meddler:"slot_num"`
+	ForgeL1TxsNum    *int64                      `json:"forgeL1TransactionsNum" meddler:"forge_l1_txs_num"`
-	ForgedTxs     int                    `json:"forgedTransactions" meddler:"forged_txs"`
+	SlotNum          int64                       `json:"slotNum" meddler:"slot_num"`
-	TotalItems    uint64                 `json:"-" meddler:"total_items"`
+	ForgedTxs        int                         `json:"forgedTransactions" meddler:"forged_txs"`
-	FirstItem     uint64                 `json:"-" meddler:"first_item"`
+	TotalItems       uint64                      `json:"-" meddler:"total_items"`
-	LastItem      uint64                 `json:"-" meddler:"last_item"`
+	FirstItem        uint64                      `json:"-" meddler:"first_item"`
+	LastItem         uint64                      `json:"-" meddler:"last_item"`
 }
 
 // MetricsAPI define metrics of the network