Add package level documentation for db, historydb, l2db and api/...

Add configuration option to choose recommended fee strategy, and add …

api/api.go

@@ -1,3 +1,27 @@
+/*
+Package api implements the public interface of the hermez-node using a HTTP REST API.
+There are two subsets of endpoints:
+- coordinatorEndpoints: used to receive L2 transactions and account creation authorizations. Targeted for wallets.
+- explorerEndpoints: used to provide all sorts of information about the network. Targeted for explorers and similar services.
+
+About the configuration of the API:
+- The API is supposed to be launched using the cli found at the package cli/node, and configured through the configuration file.
+- The mentioned configuration file allows exposing any combination of the endpoint subsets.
+- Although the API can run in a "standalone" manner using the serveapi command, it won't work properly
+unless another process acting as a coord or sync is filling the HistoryDB.
+
+Design principles and considerations:
+- In order to decouple the API process from the rest of the node, all the communication between this package and the rest of
+the system is done through the SQL database. As a matter of fact, the only public function of the package is the constructor NewAPI.
+All the information needed for the API to work should be obtained through the configuration file of the cli or the database.
+- The format of the requests / responses doesn't match directly with the common types, and for this reason, the package api/apitypes is used
+to facilitate the format conversion. Most of the time, this is done directly at the db level.
+- The API endpoints are fully documented using OpenAPI aka Swagger. All the endpoints are tested against the spec to ensure consistency
+between implementation and specification. To get a sense of which endpoints exist and how they work, it's strongly recommended to check this specification.
+The specification can be found at api/swagger.yml.
+- In general, all the API endpoints produce queries to the SQL database in order to retrieve / insert the requested information. The most notable exceptions to this are
+the /config endpoint, which returns a static object generated at construction time, and the /state, which also is retrieved from the database, but it's generated by API/stateapiupdater package.
+*/
 package api
 
 import (
@@ -27,7 +51,6 @@ func NewAPI(
 	l2db *l2db.L2DB,
 ) (*API, error) {
 	// Check input
-	// TODO: is stateDB only needed for explorer endpoints or for both?
 	if coordinatorEndpoints && l2db == nil {
 		return nil, tracerr.Wrap(errors.New("cannot serve Coordinator endpoints without L2DB"))
 	}
@@ -54,7 +77,7 @@ func NewAPI(
 
 	// Add coordinator endpoints
 	if coordinatorEndpoints {
-		// Account
+		// Account creation authorization
 		v1.POST("/account-creation-authorization", a.postAccountCreationAuth)
 		v1.GET("/account-creation-authorization/:hezEthereumAddress", a.getAccountCreationAuth)
 		// Transaction
@@ -72,17 +95,23 @@ func NewAPI(
 		// Transaction
 		v1.GET("/transactions-history", a.getHistoryTxs)
 		v1.GET("/transactions-history/:id", a.getHistoryTx)
-		// Status
+		// Batches
 		v1.GET("/batches", a.getBatches)
 		v1.GET("/batches/:batchNum", a.getBatch)
 		v1.GET("/full-batches/:batchNum", a.getFullBatch)
+		// Slots
 		v1.GET("/slots", a.getSlots)
 		v1.GET("/slots/:slotNum", a.getSlot)
+		// Bids
 		v1.GET("/bids", a.getBids)
+		// State
 		v1.GET("/state", a.getState)
+		// Config
 		v1.GET("/config", a.getConfig)
+		// Tokens
 		v1.GET("/tokens", a.getTokens)
 		v1.GET("/tokens/:id", a.getToken)
+		// Coordinators
 		v1.GET("/coordinators", a.getCoordinators)
 	}
 

api/api_test.go

@@ -522,11 +522,16 @@ func TestMain(m *testing.M) {
 		WithdrawalDelay: uint64(3000),
 	}
 
-	stateAPIUpdater = stateapiupdater.NewUpdater(hdb, nodeConfig, &common.SCVariables{
+	stateAPIUpdater, err = stateapiupdater.NewUpdater(hdb, nodeConfig, &common.SCVariables{
 		Rollup:   rollupVars,
 		Auction:  auctionVars,
 		WDelayer: wdelayerVars,
-	}, constants)
+	}, constants, &stateapiupdater.RecommendedFeePolicy{
+		PolicyType: stateapiupdater.RecommendedFeePolicyTypeAvgLastHour,
+	})
+	if err != nil {
+		panic(err)
+	}
 
 	// Generate test data, as expected to be received/sended from/to the API
 	testCoords := genTestCoordinators(commonCoords)

api/apitypes/apitypes.go

@@ -1,3 +1,11 @@
+/*
+Package apitypes is used to map the common types used across the node with the format expected by the API.
+
+This is done using different strategies:
+- Marshallers: they get triggered when the API marshals the response structs into JSONs
+- Scanners/Valuers: they get triggered when a struct is sent/received to/from the SQL database
+- Adhoc functions: when the already mentioned strategies are not suitable, functions are added to the structs to facilitate the conversions
+*/
 package apitypes
 
 import (

api/stateapiupdater/stateapiupdater.go

@@ -1,11 +1,20 @@
+/*
+Package stateapiupdater is responsible for generating and storing the object response of the GET /state endpoint exposed through the api package.
+This object is extensively defined at the OpenAPI spec located at api/swagger.yml.
+
+Deployment considerations: in a setup where multiple processes are used (dedicated api process, separated coord / sync, ...), only one process should care
+of using this package.
+*/
 package stateapiupdater
 
 import (
 	"database/sql"
+	"fmt"
 	"sync"
 
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/db/historydb"
+	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/tracerr"
 )
 
@@ -17,11 +26,45 @@ type Updater struct {
 	vars   common.SCVariablesPtr
 	consts historydb.Constants
 	rw     sync.RWMutex
+	rfp    *RecommendedFeePolicy
+}
+
+// RecommendedFeePolicy describes how the recommended fee is calculated
+type RecommendedFeePolicy struct {
+	PolicyType  RecommendedFeePolicyType `validate:"required"`
+	StaticValue float64
+}
+
+// RecommendedFeePolicyType describes the different available recommended fee strategies
+type RecommendedFeePolicyType string
+
+const (
+	// RecommendedFeePolicyTypeStatic always give the same StaticValue as recommended fee
+	RecommendedFeePolicyTypeStatic RecommendedFeePolicyType = "Static"
+	// RecommendedFeePolicyTypeAvgLastHour set the recommended fee using the average fee of the last hour
+	RecommendedFeePolicyTypeAvgLastHour RecommendedFeePolicyType = "AvgLastHour"
+)
+
+func (rfp *RecommendedFeePolicy) valid() bool {
+	switch rfp.PolicyType {
+	case RecommendedFeePolicyTypeStatic:
+		if rfp.StaticValue == 0 {
+			log.Warn("RcommendedFee is set to 0 USD, and the policy is static")
+		}
+		return true
+	case RecommendedFeePolicyTypeAvgLastHour:
+		return true
+	default:
+		return false
+	}
 }
 
 // NewUpdater creates a new Updater
 func NewUpdater(hdb *historydb.HistoryDB, config *historydb.NodeConfig, vars *common.SCVariables,
-	consts *historydb.Constants) *Updater {
+	consts *historydb.Constants, rfp *RecommendedFeePolicy) (*Updater, error) {
+	if ok := rfp.valid(); !ok {
+		return nil, tracerr.Wrap(fmt.Errorf("Invalid recommended fee policy: %v", rfp.PolicyType))
+	}
 	u := Updater{
 		hdb:    hdb,
 		config: *config,
@@ -31,9 +74,10 @@ func NewUpdater(hdb *historydb.HistoryDB, config *historydb.NodeConfig, vars *co
 				ForgeDelay: config.ForgeDelay,
 			},
 		},
+		rfp: rfp,
 	}
 	u.SetSCVars(vars.AsPtr())
-	return &u
+	return &u, nil
 }
 
 // Store the State in the HistoryDB
@@ -65,13 +109,27 @@ func (u *Updater) SetSCVars(vars *common.SCVariablesPtr) {
 
 // UpdateRecommendedFee update Status.RecommendedFee information
 func (u *Updater) UpdateRecommendedFee() error {
-	recommendedFee, err := u.hdb.GetRecommendedFee(u.config.MinFeeUSD, u.config.MaxFeeUSD)
-	if err != nil {
-		return tracerr.Wrap(err)
+	switch u.rfp.PolicyType {
+	case RecommendedFeePolicyTypeStatic:
+		u.rw.Lock()
+		u.state.RecommendedFee = common.RecommendedFee{
+			ExistingAccount:        u.rfp.StaticValue,
+			CreatesAccount:         u.rfp.StaticValue,
+			CreatesAccountInternal: u.rfp.StaticValue,
+		}
+		u.rw.Unlock()
+	case RecommendedFeePolicyTypeAvgLastHour:
+		recommendedFee, err := u.hdb.GetRecommendedFee(u.config.MinFeeUSD, u.config.MaxFeeUSD)
+		if err != nil {
+			return tracerr.Wrap(err)
+		}
+		u.rw.Lock()
+		u.state.RecommendedFee = *recommendedFee
+		u.rw.Unlock()
+	default:
+		return tracerr.New("Invalid recommende fee policy")
 	}
-	u.rw.Lock()
-	u.state.RecommendedFee = *recommendedFee
-	u.rw.Unlock()
+
 	return nil
 }
 

cli/node/cfg.buidler.toml

@@ -145,3 +145,11 @@ Coordinator = true
 BatchPath = "/tmp/iden3-test/hermez/batchesdebug"
 LightScrypt = true
 # RollupVerifierIndex = 0
+
+[RecommendedFeePolicy]
+# Strategy used to calculate the recommended fee that the API will expose.
+# Available options:
+# - Static: always return the same value (StaticValue) in USD
+# - AvgLastHour: calculate using the average fee of the forged transactions during the last hour
+PolicyType = "Static"
+StaticValue = 0.99

config/config.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	ethCommon "github.com/ethereum/go-ethereum/common"
+	"github.com/hermeznetwork/hermez-node/api/stateapiupdater"
 	"github.com/hermeznetwork/hermez-node/common"
 	"github.com/hermeznetwork/hermez-node/priceupdater"
 	"github.com/hermeznetwork/tracerr"
@@ -347,8 +348,9 @@ type Node struct {
 		// can wait to stablish a SQL connection
 		SQLConnectionTimeout Duration
 	} `validate:"required"`
-	Debug       NodeDebug   `validate:"required"`
-	Coordinator Coordinator `validate:"-"`
+	RecommendedFeePolicy stateapiupdater.RecommendedFeePolicy `validate:"required"`
+	Debug                NodeDebug                            `validate:"required"`
+	Coordinator          Coordinator                          `validate:"-"`
 }
 
 // APIServer is the api server configuration parameters

db/historydb/historydb.go

@@ -1,3 +1,24 @@
+/*
+Package historydb is responsible for storing and retrieving the historic data of the Hermez network.
+It's mostly but not exclusively used by the API and the synchronizer.
+
+Apart from the logic defined in this package, it's important to notice that there are some triggers defined in the
+migration files that have to be taken into consideration to understanding the results of some queries. This is especially true
+for reorgs: all the data is directly or indirectly related to a block, this makes handling reorgs as easy as deleting the
+reorged blocks from the block table, and all related items will be dropped in cascade. This is not the only case, in general
+functions defined in this package that get affected somehow by the SQL level defined logic has a special mention on the function description.
+
+Some of the database tooling used in this package such as meddler and migration tools is explained in the db package.
+
+This package is spitted in different files following these ideas:
+- historydb.go: constructor and functions used by packages other than the api.
+- apiqueries.go: functions used by the API, the queries implemented in this functions use a semaphore
+to restrict the maximum concurrent connections to the database.
+- views.go: structs used to retrieve/store data from/to the database. When possible, the common structs are used, however
+most of the time there is no 1:1 relation between the struct fields and the tables of the schema, especially when joining tables.
+In some cases, some of the structs defined in this file also include custom Marshallers to easily match the expected api formats.
+- nodeinfo.go: used to handle the interfaces and structs that allow communication across running in different machines/process but sharing the same database.
+*/
 package historydb
 
 import (

db/l2db/l2db.go

@@ -1,3 +1,20 @@
+/*
+Package l2db is responsible for storing and retrieving the data received by the coordinator through the api.
+Note that this data will be different for each coordinator in the network, as this represents the L2 information.
+
+The data managed by this package is fundamentally PoolL2Tx and AccountCreationAuth. All this data come from
+the API sent by clients and is used by the txselector to decide which transactions are selected to forge a batch.
+
+Some of the database tooling used in this package such as meddler and migration tools is explained in the db package.
+
+This package is spitted in different files following these ideas:
+- l2db.go: constructor and functions used by packages other than the api.
+- apiqueries.go: functions used by the API, the queries implemented in this functions use a semaphore
+to restrict the maximum concurrent connections to the database.
+- views.go: structs used to retrieve/store data from/to the database. When possible, the common structs are used, however
+most of the time there is no 1:1 relation between the struct fields and the tables of the schema, especially when joining tables.
+In some cases, some of the structs defined in this file also include custom Marshallers to easily match the expected api formats.
+*/
 package l2db
 
 import (

db/utils.go

@@ -1,3 +1,10 @@
+/*
+Package db have some common utilities shared by db/l2db and db/historydb, the most relevant ones are:
+- SQL connection utilities
+- Managing the SQL schema: this is done using migration files placed under db/migrations. The files are executed by
+order of the file name.
+- Custom meddlers: used to easily transform struct <==> table
+*/
 package db
 
 import (

node/node.go

@@ -288,7 +288,16 @@ func NewNode(mode Mode, cfg *config.Node) (*Node, error) {
 		return nil, tracerr.Wrap(err)
 	}
 
-	stateAPIUpdater := stateapiupdater.NewUpdater(historyDB, &hdbNodeCfg, initSCVars, &hdbConsts)
+	stateAPIUpdater, err := stateapiupdater.NewUpdater(
+		historyDB,
+		&hdbNodeCfg,
+		initSCVars,
+		&hdbConsts,
+		&cfg.RecommendedFeePolicy,
+	)
+	if err != nil {
+		return nil, tracerr.Wrap(err)
+	}
 
 	var coord *coordinator.Coordinator
 	if mode == ModeCoordinator {