From 009d0c5be1bd7e0b86da643f8edd7375c7d46876 Mon Sep 17 00:00:00 2001 From: Eduard S Date: Wed, 17 Mar 2021 14:56:58 +0100 Subject: [PATCH] Fix API /state collected fees unmarshal --- api/batch_test.go | 40 +++++++++++---------- apitypes/apitypes.go | 23 +++++------- db/historydb/apiqueries.go | 12 +++++-- db/historydb/historydb_test.go | 64 ++++++++++++++++++++++++++++++++++ db/historydb/nodeinfo.go | 5 +++ db/historydb/views.go | 35 ++++++++++--------- 6 files changed, 126 insertions(+), 53 deletions(-) diff --git a/api/batch_test.go b/api/batch_test.go index 23a91fa..5f74a70 100644 --- a/api/batch_test.go +++ b/api/batch_test.go @@ -7,10 +7,12 @@ import ( "time" ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/apitypes" "github.com/hermeznetwork/hermez-node/common" "github.com/hermeznetwork/hermez-node/db/historydb" "github.com/mitchellh/copystructure" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type testBatch struct { @@ -20,7 +22,7 @@ type testBatch struct { EthBlockHash ethCommon.Hash `json:"ethereumBlockHash"` Timestamp time.Time `json:"timestamp"` ForgerAddr ethCommon.Address `json:"forgerAddr"` - CollectedFees map[common.TokenID]string `json:"collectedFees"` + CollectedFees apitypes.CollectedFeesAPI `json:"collectedFees"` TotalFeesUSD *float64 `json:"historicTotalCollectedFeesUSD"` StateRoot string `json:"stateRoot"` NumAccounts int `json:"numAccounts"` @@ -73,9 +75,9 @@ func genTestBatches( if !found { panic("block not found") } - collectedFees := make(map[common.TokenID]string) + collectedFees := apitypes.CollectedFeesAPI(make(map[common.TokenID]apitypes.BigIntStr)) for k, v := range cBatches[i].CollectedFees { - collectedFees[k] = v.String() + collectedFees[k] = *apitypes.NewBigIntStr(v) } forgedTxs := 0 for _, tx := range txs { @@ -132,7 +134,7 @@ func TestGetBatches(t *testing.T) { limit := 3 path := fmt.Sprintf("%s?limit=%d", endpoint, limit) err := doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) assertBatches(t, tc.batches, fetchedBatches) // minBatchNum @@ -141,7 +143,7 @@ func TestGetBatches(t *testing.T) { minBatchNum := tc.batches[len(tc.batches)/2].BatchNum path = fmt.Sprintf("%s?minBatchNum=%d&limit=%d", endpoint, minBatchNum, limit) err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) minBatchNumBatches := []testBatch{} for i := 0; i < len(tc.batches); i++ { if tc.batches[i].BatchNum > minBatchNum { @@ -156,7 +158,7 @@ func TestGetBatches(t *testing.T) { maxBatchNum := tc.batches[len(tc.batches)/2].BatchNum path = fmt.Sprintf("%s?maxBatchNum=%d&limit=%d", endpoint, maxBatchNum, limit) err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) maxBatchNumBatches := []testBatch{} for i := 0; i < len(tc.batches); i++ { if tc.batches[i].BatchNum < maxBatchNum { @@ -171,7 +173,7 @@ func TestGetBatches(t *testing.T) { slotNum := tc.batches[len(tc.batches)/2].SlotNum path = fmt.Sprintf("%s?slotNum=%d&limit=%d", endpoint, slotNum, limit) err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) slotNumBatches := []testBatch{} for i := 0; i < len(tc.batches); i++ { if tc.batches[i].SlotNum == slotNum { @@ -186,7 +188,7 @@ func TestGetBatches(t *testing.T) { forgerAddr := tc.batches[len(tc.batches)/2].ForgerAddr path = fmt.Sprintf("%s?forgerAddr=%s&limit=%d", endpoint, forgerAddr.String(), limit) err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) forgerAddrBatches := []testBatch{} for i := 0; i < len(tc.batches); i++ { if tc.batches[i].ForgerAddr == forgerAddr { @@ -200,7 +202,7 @@ func TestGetBatches(t *testing.T) { limit = 6 path = fmt.Sprintf("%s?limit=%d", endpoint, limit) err = doGoodReqPaginated(path, historydb.OrderDesc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) flippedBatches := []testBatch{} for i := len(tc.batches) - 1; i >= 0; i-- { flippedBatches = append(flippedBatches, tc.batches[i]) @@ -214,7 +216,7 @@ func TestGetBatches(t *testing.T) { minBatchNum = tc.batches[len(tc.batches)/4].BatchNum path = fmt.Sprintf("%s?minBatchNum=%d&maxBatchNum=%d&limit=%d", endpoint, minBatchNum, maxBatchNum, limit) err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) minMaxBatchNumBatches := []testBatch{} for i := 0; i < len(tc.batches); i++ { if tc.batches[i].BatchNum < maxBatchNum && tc.batches[i].BatchNum > minBatchNum { @@ -227,25 +229,25 @@ func TestGetBatches(t *testing.T) { fetchedBatches = []testBatch{} path = fmt.Sprintf("%s?slotNum=%d&minBatchNum=%d", endpoint, 1, 25) err = doGoodReqPaginated(path, historydb.OrderAsc, &testBatchesResponse{}, appendIter) - assert.NoError(t, err) + require.NoError(t, err) assertBatches(t, []testBatch{}, fetchedBatches) // 400 // Invalid minBatchNum path = fmt.Sprintf("%s?minBatchNum=%d", endpoint, -2) err = doBadReq("GET", path, nil, 400) - assert.NoError(t, err) + require.NoError(t, err) // Invalid forgerAddr path = fmt.Sprintf("%s?forgerAddr=%s", endpoint, "0xG0000001") err = doBadReq("GET", path, nil, 400) - assert.NoError(t, err) + require.NoError(t, err) } func TestGetBatch(t *testing.T) { endpoint := apiURL + "batches/" for _, batch := range tc.batches { fetchedBatch := testBatch{} - assert.NoError( + require.NoError( t, doGoodReq( "GET", endpoint+strconv.Itoa(int(batch.BatchNum)), @@ -255,16 +257,16 @@ func TestGetBatch(t *testing.T) { assertBatch(t, batch, fetchedBatch) } // 400 - assert.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400)) + require.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400)) // 404 - assert.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404)) + require.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404)) } func TestGetFullBatch(t *testing.T) { endpoint := apiURL + "full-batches/" for _, fullBatch := range tc.fullBatches { fetchedFullBatch := testFullBatch{} - assert.NoError( + require.NoError( t, doGoodReq( "GET", endpoint+strconv.Itoa(int(fullBatch.Batch.BatchNum)), @@ -275,9 +277,9 @@ func TestGetFullBatch(t *testing.T) { assertTxs(t, fullBatch.Txs, fetchedFullBatch.Txs) } // 400 - assert.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400)) + require.NoError(t, doBadReq("GET", endpoint+"foo", nil, 400)) // 404 - assert.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404)) + require.NoError(t, doBadReq("GET", endpoint+"99999", nil, 404)) } func assertBatches(t *testing.T, expected, actual []testBatch) { diff --git a/apitypes/apitypes.go b/apitypes/apitypes.go index 7bf04ae..274f363 100644 --- a/apitypes/apitypes.go +++ b/apitypes/apitypes.go @@ -4,7 +4,6 @@ import ( "database/sql/driver" "encoding/base64" "encoding/hex" - "encoding/json" "errors" "fmt" "math/big" @@ -73,22 +72,16 @@ func (s *StrBigInt) UnmarshalText(text []byte) error { return nil } -// CollectedFees is used to retrieve common.batch.CollectedFee from the DB -type CollectedFees map[common.TokenID]BigIntStr +// CollectedFeesAPI is send common.batch.CollectedFee through the API +type CollectedFeesAPI map[common.TokenID]BigIntStr -// UnmarshalJSON unmarshals a json representation of map[common.TokenID]*big.Int -func (c *CollectedFees) UnmarshalJSON(text []byte) error { - bigIntMap := make(map[common.TokenID]*big.Int) - if err := json.Unmarshal(text, &bigIntMap); err != nil { - return tracerr.Wrap(err) - } - *c = CollectedFees(make(map[common.TokenID]BigIntStr)) - for k, v := range bigIntMap { - bStr := NewBigIntStr(v) - (CollectedFees(*c)[k]) = *bStr +// NewCollectedFeesAPI creates a new CollectedFeesAPI from a *big.Int map +func NewCollectedFeesAPI(m map[common.TokenID]*big.Int) CollectedFeesAPI { + c := CollectedFeesAPI(make(map[common.TokenID]BigIntStr)) + for k, v := range m { + c[k] = *NewBigIntStr(v) } - // *c = CollectedFees(bStrMap) - return nil + return c } // HezEthAddr is used to scan/value Ethereum Address directly into strings that follow the Ethereum address hez format (^hez:0x[a-fA-F0-9]{40}$) from/to sql DBs. diff --git a/db/historydb/apiqueries.go b/db/historydb/apiqueries.go index 6e1bc04..913fb5d 100644 --- a/db/historydb/apiqueries.go +++ b/db/historydb/apiqueries.go @@ -8,6 +8,7 @@ import ( "time" ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/apitypes" "github.com/hermeznetwork/hermez-node/common" "github.com/hermeznetwork/hermez-node/db" "github.com/hermeznetwork/tracerr" @@ -45,7 +46,7 @@ func (hdb *HistoryDB) GetBatchInternalAPI(batchNum common.BatchNum) (*BatchAPI, func (hdb *HistoryDB) getBatchAPI(d meddler.DB, batchNum common.BatchNum) (*BatchAPI, error) { batch := &BatchAPI{} - return batch, tracerr.Wrap(meddler.QueryRow( + if err := meddler.QueryRow( d, batch, `SELECT batch.item_id, batch.batch_num, batch.eth_block_num, batch.forger_addr, batch.fees_collected, batch.total_fees_usd, batch.state_root, @@ -54,7 +55,11 @@ func (hdb *HistoryDB) getBatchAPI(d meddler.DB, batchNum common.BatchNum) (*Batc COALESCE ((SELECT COUNT(*) FROM tx WHERE batch_num = batch.batch_num), 0) AS forged_txs FROM batch INNER JOIN block ON batch.eth_block_num = block.eth_block_num WHERE batch_num = $1;`, batchNum, - )) + ); err != nil { + return nil, tracerr.Wrap(err) + } + batch.CollectedFeesAPI = apitypes.NewCollectedFeesAPI(batch.CollectedFeesDB) + return batch, nil } // GetBatchesAPI return the batches applying the given filters @@ -155,6 +160,9 @@ func (hdb *HistoryDB) GetBatchesAPI( if len(batches) == 0 { return batches, 0, nil } + for i := range batches { + batches[i].CollectedFeesAPI = apitypes.NewCollectedFeesAPI(batches[i].CollectedFeesDB) + } return batches, batches[0].TotalItems - uint64(len(batches)), nil } diff --git a/db/historydb/historydb_test.go b/db/historydb/historydb_test.go index de4384a..daa7b4e 100644 --- a/db/historydb/historydb_test.go +++ b/db/historydb/historydb_test.go @@ -11,6 +11,7 @@ import ( "time" ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/apitypes" "github.com/hermeznetwork/hermez-node/common" dbUtils "github.com/hermeznetwork/hermez-node/db" "github.com/hermeznetwork/hermez-node/log" @@ -1479,6 +1480,10 @@ func TestNodeInfo(t *testing.T) { require.NoError(t, err) // Test parameters + var f64 float64 = 1.2 + var i64 int64 = 8888 + addr := ethCommon.HexToAddress("0x1234") + hash := ethCommon.HexToHash("0x5678") stateAPI := &StateAPI{ NodePublicConfig: NodePublicConfig{ ForgeDelay: 3.1, @@ -1486,6 +1491,56 @@ func TestNodeInfo(t *testing.T) { Network: NetworkAPI{ LastEthBlock: 12, LastSyncBlock: 34, + LastBatch: &BatchAPI{ + ItemID: 123, + BatchNum: 456, + EthBlockNum: 789, + EthBlockHash: hash, + Timestamp: time.Now(), + ForgerAddr: addr, + // CollectedFeesDB: map[common.TokenID]*big.Int{ + // 0: big.NewInt(11111), + // 1: big.NewInt(21111), + // 2: big.NewInt(31111), + // }, + CollectedFeesAPI: apitypes.CollectedFeesAPI(map[common.TokenID]apitypes.BigIntStr{ + 0: apitypes.BigIntStr("11111"), + 1: apitypes.BigIntStr("21111"), + 2: apitypes.BigIntStr("31111"), + }), + TotalFeesUSD: &f64, + StateRoot: apitypes.BigIntStr("1234"), + NumAccounts: 11, + ExitRoot: apitypes.BigIntStr("5678"), + ForgeL1TxsNum: &i64, + SlotNum: 44, + ForgedTxs: 23, + TotalItems: 0, + FirstItem: 0, + LastItem: 0, + }, + CurrentSlot: 22, + NextForgers: []NextForgerAPI{ + { + Coordinator: CoordinatorAPI{ + ItemID: 111, + Bidder: addr, + Forger: addr, + EthBlockNum: 566, + URL: "asd", + TotalItems: 0, + FirstItem: 0, + LastItem: 0, + }, + Period: Period{ + SlotNum: 33, + FromBlock: 55, + ToBlock: 66, + FromTimestamp: time.Now(), + ToTimestamp: time.Now(), + }, + }, + }, }, Metrics: MetricsAPI{ TransactionsPerBatch: 1.1, @@ -1518,5 +1573,14 @@ func TestNodeInfo(t *testing.T) { dbStateAPI, err := historyDB.getStateAPI(historyDB.dbRead) require.NoError(t, err) + assert.Equal(t, stateAPI.Network.LastBatch.Timestamp.Unix(), + dbStateAPI.Network.LastBatch.Timestamp.Unix()) + dbStateAPI.Network.LastBatch.Timestamp = stateAPI.Network.LastBatch.Timestamp + assert.Equal(t, stateAPI.Network.NextForgers[0].Period.FromTimestamp.Unix(), + dbStateAPI.Network.NextForgers[0].Period.FromTimestamp.Unix()) + dbStateAPI.Network.NextForgers[0].Period.FromTimestamp = stateAPI.Network.NextForgers[0].Period.FromTimestamp + assert.Equal(t, stateAPI.Network.NextForgers[0].Period.ToTimestamp.Unix(), + dbStateAPI.Network.NextForgers[0].Period.ToTimestamp.Unix()) + dbStateAPI.Network.NextForgers[0].Period.ToTimestamp = stateAPI.Network.NextForgers[0].Period.ToTimestamp assert.Equal(t, stateAPI, dbStateAPI) } diff --git a/db/historydb/nodeinfo.go b/db/historydb/nodeinfo.go index c78b938..6447695 100644 --- a/db/historydb/nodeinfo.go +++ b/db/historydb/nodeinfo.go @@ -4,6 +4,7 @@ import ( "time" ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/hermeznetwork/hermez-node/apitypes" "github.com/hermeznetwork/hermez-node/common" "github.com/hermeznetwork/tracerr" "github.com/russross/meddler" @@ -124,6 +125,10 @@ func (hdb *HistoryDB) getStateAPI(d meddler.DB) (*StateAPI, error) { // SetStateInternalAPI sets the StateAPI func (hdb *HistoryDB) SetStateInternalAPI(stateAPI *StateAPI) error { + if stateAPI.Network.LastBatch != nil { + stateAPI.Network.LastBatch.CollectedFeesAPI = + apitypes.NewCollectedFeesAPI(stateAPI.Network.LastBatch.CollectedFeesDB) + } _stateAPI := struct { StateAPI *StateAPI `meddler:"state,json"` }{stateAPI} diff --git a/db/historydb/views.go b/db/historydb/views.go index 43756ef..f053a3d 100644 --- a/db/historydb/views.go +++ b/db/historydb/views.go @@ -289,23 +289,24 @@ func (account AccountAPI) MarshalJSON() ([]byte, error) { // BatchAPI is a representation of a batch with additional information // required by the API, and extracted by joining block table type BatchAPI struct { - ItemID uint64 `json:"itemId" meddler:"item_id"` - BatchNum common.BatchNum `json:"batchNum" meddler:"batch_num"` - EthBlockNum int64 `json:"ethereumBlockNum" meddler:"eth_block_num"` - EthBlockHash ethCommon.Hash `json:"ethereumBlockHash" meddler:"hash"` - Timestamp time.Time `json:"timestamp" meddler:"timestamp,utctime"` - ForgerAddr ethCommon.Address `json:"forgerAddr" meddler:"forger_addr"` - CollectedFees apitypes.CollectedFees `json:"collectedFees" meddler:"fees_collected,json"` - TotalFeesUSD *float64 `json:"historicTotalCollectedFeesUSD" meddler:"total_fees_usd"` - StateRoot apitypes.BigIntStr `json:"stateRoot" meddler:"state_root"` - NumAccounts int `json:"numAccounts" meddler:"num_accounts"` - ExitRoot apitypes.BigIntStr `json:"exitRoot" meddler:"exit_root"` - ForgeL1TxsNum *int64 `json:"forgeL1TransactionsNum" meddler:"forge_l1_txs_num"` - SlotNum int64 `json:"slotNum" meddler:"slot_num"` - ForgedTxs int `json:"forgedTransactions" meddler:"forged_txs"` - TotalItems uint64 `json:"-" meddler:"total_items"` - FirstItem uint64 `json:"-" meddler:"first_item"` - LastItem uint64 `json:"-" meddler:"last_item"` + ItemID uint64 `json:"itemId" meddler:"item_id"` + BatchNum common.BatchNum `json:"batchNum" meddler:"batch_num"` + EthBlockNum int64 `json:"ethereumBlockNum" meddler:"eth_block_num"` + EthBlockHash ethCommon.Hash `json:"ethereumBlockHash" meddler:"hash"` + Timestamp time.Time `json:"timestamp" meddler:"timestamp,utctime"` + ForgerAddr ethCommon.Address `json:"forgerAddr" meddler:"forger_addr"` + CollectedFeesDB map[common.TokenID]*big.Int `json:"-" meddler:"fees_collected,json"` + CollectedFeesAPI apitypes.CollectedFeesAPI `json:"collectedFees" meddler:"-"` + TotalFeesUSD *float64 `json:"historicTotalCollectedFeesUSD" meddler:"total_fees_usd"` + StateRoot apitypes.BigIntStr `json:"stateRoot" meddler:"state_root"` + NumAccounts int `json:"numAccounts" meddler:"num_accounts"` + ExitRoot apitypes.BigIntStr `json:"exitRoot" meddler:"exit_root"` + ForgeL1TxsNum *int64 `json:"forgeL1TransactionsNum" meddler:"forge_l1_txs_num"` + SlotNum int64 `json:"slotNum" meddler:"slot_num"` + ForgedTxs int `json:"forgedTransactions" meddler:"forged_txs"` + TotalItems uint64 `json:"-" meddler:"total_items"` + FirstItem uint64 `json:"-" meddler:"first_item"` + LastItem uint64 `json:"-" meddler:"last_item"` } // MetricsAPI define metrics of the network