Add filter to include unforged history txs in api

fix the invalid goarch build

.gitignore

@@ -1 +1,2 @@
-bin/
+bin/
+dist/

.goreleaser.yml

@@ -1,6 +1,7 @@
 before:
   hooks:
     - go mod download
+    - make migration-pack
 
 builds:
   - main: ./cli/node/main.go
@@ -9,10 +10,8 @@ builds:
     goos:
       - linux
       - darwin
-      - windows
-    hooks:
-      pre: make migration-pack
-      post: make migration-clean
+    goarch:
+      - amd64
 
 archives:
   - replacements:

api/api_test.go

@@ -161,6 +161,8 @@ var SetBlockchain = `
 	> block
 	> batch
 	> block
+	ForceTransfer(0) D-B: 77777700000000000
+	> block
 `
 
 type testCommon struct {
@@ -362,6 +364,12 @@ func TestMain(m *testing.M) {
 			commonL1Txs = append(commonL1Txs, batch.L1CoordinatorTxs...)
 		}
 	}
+	// Add unforged L1 tx
+	unforgedTx := blocksData[len(blocksData)-1].Rollup.L1UserTxs[0]
+	if unforgedTx.BatchNum != nil {
+		panic("Unforged tx batch num should be nil")
+	}
+	commonL1Txs = append(commonL1Txs, unforgedTx)
 
 	// Generate Coordinators and add them to HistoryDB
 	const nCoords = 10

api/batch.go

@@ -109,7 +109,7 @@ func (a *API) getFullBatch(c *gin.Context) {
 	// Fetch txs forged in the batch from historyDB
 	maxTxsPerBatch := uint(2048) //nolint:gomnd
 	txs, _, err := a.h.GetTxsAPI(
-		nil, nil, nil, nil, batchNum, nil, nil, &maxTxsPerBatch, historydb.OrderAsc,
+		nil, nil, nil, nil, batchNum, nil, nil, nil, &maxTxsPerBatch, historydb.OrderAsc,
 	)
 	if err != nil && tracerr.Unwrap(err) != sql.ErrNoRows {
 		retSQLErr(err, c)

api/stateapiupdater/stateapiupdater.go

@@ -2,6 +2,7 @@ package stateapiupdater
 
 import (
 	"database/sql"
+	"fmt"
 	"sync"
 
 	"github.com/hermeznetwork/hermez-node/common"
@@ -23,7 +24,7 @@ type Updater struct {
 
 // RecommendedFeePolicy describes how the recommended fee is calculated
 type RecommendedFeePolicy struct {
-	PolicyType  RecommendedFeePolicyType
+	PolicyType  RecommendedFeePolicyType `validate:"required"`
 	StaticValue float64
 }
 
@@ -31,9 +32,9 @@ type RecommendedFeePolicy struct {
 type RecommendedFeePolicyType string
 
 const (
-	// Always give the same StaticValue as recommended fee
+	// RecommendedFeePolicyTypeStatic always give the same StaticValue as recommended fee
 	RecommendedFeePolicyTypeStatic RecommendedFeePolicyType = "Static"
-	// Set the recommended fee using the average fee of the last hour
+	// RecommendedFeePolicyTypeAvgLastHour set the recommended fee using the average fee of the last hour
 	RecommendedFeePolicyTypeAvgLastHour RecommendedFeePolicyType = "AvgLastHour"
 )
 
@@ -55,7 +56,7 @@ func (rfp *RecommendedFeePolicy) valid() bool {
 func NewUpdater(hdb *historydb.HistoryDB, config *historydb.NodeConfig, vars *common.SCVariables,
 	consts *historydb.Constants, rfp *RecommendedFeePolicy) (*Updater, error) {
 	if ok := rfp.valid(); !ok {
-		return nil, tracerr.New("Invalid recommende fee policy")
+		return nil, tracerr.Wrap(fmt.Errorf("Invalid recommended fee policy: %v", rfp.PolicyType))
 	}
 	u := Updater{
 		hdb:    hdb,

api/swagger.yml

@@ -59,21 +59,17 @@ externalDocs:
   description: Find out more about Hermez network.
   url: 'https://hermez.io'
 servers:
-  - description: Hosted mock up, returns fake data useful for development
-    url: https://apimock.hermez.network
-  - description: Localhost mock up, returns fake data useful for development
-    url: http://localhost:4010
-  - description: Testnet (Rinkeby) server
-    url: https://api.testnet.hermez.io
-  - description: Mainnet (Ethereum) server, use it carefully, specially if attempting to send transactions. You could lose money!
-    url: https://api.hermez.io
+  - description: Hosted mock up
+    url: https://apimock.hermez.network/v1
+  - description: Localhost mock Up
+    url: http://localhost:4010/v1
 tags:
   - name: Coordinator
     description: Endpoints used by the nodes running in coordinator mode. They are used to interact with the network.
   - name: Explorer
     description: Endpoints used by the nodes running in explorer mode. They are used to get information of the netwrok.
 paths:
-  '/v1/account-creation-authorization':
+  '/account-creation-authorization':
     post:
       tags:
         - Coordinator
@@ -103,7 +99,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/account-creation-authorization/{hezEthereumAddress}':
+  '/account-creation-authorization/{hezEthereumAddress}':
     get:
       tags:
         - Coordinator
@@ -143,7 +139,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/accounts':
+  '/accounts':
     get:
       tags:
         - Explorer
@@ -214,7 +210,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/accounts/{accountIndex}':
+  '/accounts/{accountIndex}':
     get:
       tags:
         - Explorer
@@ -253,7 +249,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/exits':
+  '/exits':
     get:
       tags:
         - Explorer
@@ -340,7 +336,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/exits/{batchNum}/{accountIndex}':
+  '/exits/{batchNum}/{accountIndex}':
     get:
       tags:
         - Explorer
@@ -385,7 +381,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/transactions-pool':
+  '/transactions-pool':
     post:
       tags:
         - Coordinator
@@ -419,7 +415,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/transactions-pool/{id}':
+  '/transactions-pool/{id}':
     get:
       tags:
         - Coordinator
@@ -462,7 +458,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/transactions-history':
+  '/transactions-history':
     get:
       tags:
         - Explorer
@@ -533,6 +529,15 @@ paths:
             type: integer
             minimum: 1
             maximum: 2049
+        - name: includePendingL1s
+          in: query
+          required: false
+          description: |
+            If set to true L1 transactions that have been added to the smart contract queue but haven't been forged yet are returned.
+            Warning: the correctness of the order is not guaranteed when using this filter, as the unforged transactions may change their position
+            once they are forged. 
+          schema:
+            type: boolean
       responses:
         '200':
           description: Successful operation.
@@ -552,7 +557,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/transactions-history/{id}':
+  '/transactions-history/{id}':
     get:
       tags:
         - Explorer
@@ -592,7 +597,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/batches':
+  '/batches':
     get:
       tags:
         - Explorer
@@ -668,7 +673,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'                
-  '/v1/batches/{batchNum}':
+  '/batches/{batchNum}':
     get:
       tags:
         - Explorer
@@ -708,7 +713,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/full-batches/{batchNum}':
+  '/full-batches/{batchNum}':
     get:
       tags:
         - Explorer
@@ -749,7 +754,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/slots':
+  '/slots':
     get:
       tags:
         - Explorer
@@ -825,7 +830,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/slots/{slotNum}':
+  '/slots/{slotNum}':
     get:
       tags:
         - Explorer
@@ -865,7 +870,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/bids':
+  '/bids':
     get:
       tags:
         - Explorer
@@ -929,7 +934,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/state':
+  '/state':
     get:
       tags:
         - Explorer
@@ -955,7 +960,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/config':
+  '/config':
     get:
       tags:
         - Explorer
@@ -975,7 +980,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/tokens':
+  '/tokens':
     get:
       tags:
         - Explorer
@@ -1048,7 +1053,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/tokens/{id}':
+  '/tokens/{id}':
     get:
       tags:
         - Explorer
@@ -1087,7 +1092,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error500'
-  '/v1/coordinators':
+  '/coordinators':
     get:
       tags:
         - Explorer

api/txshistory.go

@@ -26,6 +26,14 @@ func (a *API) getHistoryTxs(c *gin.Context) {
 		retBadReq(err, c)
 		return
 	}
+	// IncludePendingL1s
+	includePendingL1s := new(bool)
+	*includePendingL1s = false
+	includePendingL1s, err = parseQueryBool("includePendingL1s", includePendingL1s, c)
+	if err != nil {
+		retBadReq(err, c)
+		return
+	}
 	// Pagination
 	fromItem, order, limit, err := parsePagination(c)
 	if err != nil {
@@ -35,7 +43,7 @@ func (a *API) getHistoryTxs(c *gin.Context) {
 
 	// Fetch txs from historyDB
 	txs, pendingItems, err := a.h.GetTxsAPI(
-		addr, bjj, tokenID, idx, batchNum, txType, fromItem, limit, order,
+		addr, bjj, tokenID, idx, batchNum, txType, includePendingL1s, fromItem, limit, order,
 	)
 	if err != nil {
 		retSQLErr(err, c)

api/txshistory_test.go

@@ -71,7 +71,7 @@ func (t txsSort) Less(i, j int) bool {
 	}
 	// i is forged
 	if jsf.BatchNum == nil {
-		return false // j is not forged
+		return true // j is not forged
 	}
 	// Both are forged
 	if *isf.BatchNum == *jsf.BatchNum {
@@ -111,7 +111,7 @@ func genTestTxs(
 ) []testTx {
 	txs := []testTx{}
 	// common.L1Tx ==> testTx
-	for _, l1 := range l1s {
+	for i, l1 := range l1s {
 		token := getTokenByID(l1.TokenID, tokens)
 		// l1.FromEthAddr and l1.FromBJJ can't be nil
 		fromEthAddr := string(apitypes.NewHezEthAddr(l1.FromEthAddr))
@@ -137,15 +137,26 @@ func genTestTxs(
 			},
 			Token: token,
 		}
+
 		// set BatchNum for user txs
 		if tx.L1Info.ToForgeL1TxsNum != nil {
-			// WARNING: this is an asumption, and the test input data can brake it easily
+			// WARNING: this works just because the way "common" txs are generated using til
+			// any change on the test set could break this
 			bn := common.BatchNum(*tx.L1Info.ToForgeL1TxsNum + 2)
 			tx.BatchNum = &bn
 		}
 		// If FromIdx is not nil
 		idxStr := idxToHez(l1.EffectiveFromIdx, token.Symbol)
 		tx.FromIdx = &idxStr
+		if i == len(l1s)-1 {
+			// Last tx of the L1 set is supposed to be unforged as per the til set.
+			// Unforged txs have some special propperties
+			tx.L1Info.DepositAmountSuccess = false
+			tx.L1Info.AmountSuccess = false
+			tx.BatchNum = nil
+			idxStrUnforged := idxToHez(l1.FromIdx, token.Symbol)
+			tx.FromIdx = &idxStrUnforged
+		}
 		// If tx has a normal ToIdx (>255), set FromEthAddr and FromBJJ
 		if l1.ToIdx >= common.UserThreshold {
 			// find account
@@ -261,12 +272,26 @@ func TestGetHistoryTxs(t *testing.T) {
 			fetchedTxs = append(fetchedTxs, tmp.(testTx))
 		}
 	}
-	// Get all (no filters)
+	// Get all (no filters, excluding unforged txs)
 	limit := 20
 	path := fmt.Sprintf("%s?limit=%d", endpoint, limit)
 	err := doGoodReqPaginated(path, historydb.OrderAsc, &testTxsResponse{}, appendIter)
 	assert.NoError(t, err)
+	forgedTxs := []testTx{}
+	for i := 0; i < len(tc.txs); i++ {
+		if tc.txs[i].BatchNum != nil {
+			forgedTxs = append(forgedTxs, tc.txs[i])
+		}
+	}
+	assertTxs(t, forgedTxs, fetchedTxs)
+
+	// Get all, including unforged txs
+	fetchedTxs = []testTx{}
+	path = fmt.Sprintf("%s?limit=%d&includePendingL1s=true", endpoint, limit)
+	err = doGoodReqPaginated(path, historydb.OrderAsc, &testTxsResponse{}, appendIter)
+	assert.NoError(t, err)
 	assertTxs(t, tc.txs, fetchedTxs)
+
 	// Get by ethAddr
 	account := tc.accounts[2]
 	fetchedTxs = []testTx{}
@@ -285,7 +310,7 @@ func TestGetHistoryTxs(t *testing.T) {
 			(tx.FromEthAddr != nil && *tx.FromEthAddr == string(account.EthAddr)) ||
 			(tx.ToEthAddr != nil && *tx.ToEthAddr == string(account.EthAddr)) ||
 			(tx.FromBJJ != nil && *tx.FromBJJ == string(account.PublicKey)) ||
-			(tx.ToBJJ != nil && *tx.ToBJJ == string(account.PublicKey)) {
+			(tx.ToBJJ != nil && *tx.ToBJJ == string(account.PublicKey)) && tx.BatchNum != nil {
 			accountTxs = append(accountTxs, tx)
 		}
 	}
@@ -312,7 +337,7 @@ func TestGetHistoryTxs(t *testing.T) {
 	assert.NoError(t, err)
 	tokenIDTxs := []testTx{}
 	for i := 0; i < len(tc.txs); i++ {
-		if tc.txs[i].Token.TokenID == tokenID {
+		if tc.txs[i].BatchNum != nil && tc.txs[i].Token.TokenID == tokenID {
 			tokenIDTxs = append(tokenIDTxs, tc.txs[i])
 		}
 	}
@@ -331,6 +356,9 @@ func TestGetHistoryTxs(t *testing.T) {
 	assert.NoError(t, err)
 	idxTxs := []testTx{}
 	for i := 0; i < len(tc.txs); i++ {
+		if tc.txs[i].BatchNum == nil {
+			continue
+		}
 		var fromIdx *common.Idx
 		if tc.txs[i].FromIdx != nil {
 			fromIdx, err = stringToIdx(*tc.txs[i].FromIdx, "")
@@ -388,7 +416,7 @@ func TestGetHistoryTxs(t *testing.T) {
 		assert.NoError(t, err)
 		txTypeTxs := []testTx{}
 		for i := 0; i < len(tc.txs); i++ {
-			if tc.txs[i].Type == txType {
+			if tc.txs[i].Type == txType && tc.txs[i].BatchNum != nil {
 				txTypeTxs = append(txTypeTxs, tc.txs[i])
 			}
 		}
@@ -420,7 +448,9 @@ func TestGetHistoryTxs(t *testing.T) {
 	assert.NoError(t, err)
 	flipedTxs := []testTx{}
 	for i := 0; i < len(tc.txs); i++ {
-		flipedTxs = append(flipedTxs, tc.txs[len(tc.txs)-1-i])
+		if tc.txs[len(tc.txs)-1-i].BatchNum != nil {
+			flipedTxs = append(flipedTxs, tc.txs[len(tc.txs)-1-i])
+		}
 	}
 	assertTxs(t, flipedTxs, fetchedTxs)
 	// Empty array

cli/node/main.go

@@ -365,7 +365,6 @@ func getConfig(c *cli.Context) (*Config, error) {
 		}
 	case modeCoord:
 		cfg.mode = node.ModeCoordinator
-		fmt.Println("LOADING CFG")
 		cfg.node, err = config.LoadNode(nodeCfgPath, true)
 		if err != nil {
 			return nil, tracerr.Wrap(err)

db/historydb/apiqueries.go

@@ -457,6 +457,7 @@ func (hdb *HistoryDB) GetTxAPI(txID common.TxID) (*TxAPI, error) {
 func (hdb *HistoryDB) GetTxsAPI(
 	ethAddr *ethCommon.Address, bjj *babyjub.PublicKeyComp,
 	tokenID *common.TokenID, idx *common.Idx, batchNum *uint, txType *common.TxType,
+	includePendingL1s *bool,
 	fromItem, limit *uint, order string,
 ) ([]TxAPI, uint64, error) {
 	// Warning: amount_success and deposit_amount_success have true as default for
@@ -554,12 +555,16 @@ func (hdb *HistoryDB) GetTxsAPI(
 		args = append(args, fromItem)
 		nextIsAnd = true
 	}
-	if nextIsAnd {
-		queryStr += "AND "
-	} else {
-		queryStr += "WHERE "
+
+	// Include pending L1 txs? (deafault false)
+	if includePendingL1s == nil || (includePendingL1s != nil && !*includePendingL1s) {
+		if nextIsAnd {
+			queryStr += "AND "
+		} else {
+			queryStr += "WHERE "
+		}
+		queryStr += "tx.batch_num IS NOT NULL "
 	}
-	queryStr += "tx.batch_num IS NOT NULL "
 
 	// pagination
 	queryStr += "ORDER BY tx.item_id "