Merge pull request #563 from hermeznetwork/feature/metrics0

Add Prometheus initial setup

README.md

@@ -15,7 +15,7 @@ start PostgreSQL with docker easily this way (where `yourpasswordhere` should
 be your password):
 
 ```
-POSTGRES_PASS=yourpasswordhere sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=hermez -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$POSTGRES_PASS" -d postgres
+POSTGRES_PASS=yourpasswordhere; sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=hermez -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$POSTGRES_PASS" -d postgres
 ```
 
 Afterwards, run the tests with the password as env var:

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/miguelmota/go-ethereum-hdwallet v0.0.0-20200123000308-a60dcd172b4c
 	github.com/mitchellh/copystructure v1.0.0
 	github.com/mitchellh/mapstructure v1.3.0
+	github.com/prometheus/client_golang v1.3.0
 	github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
 	github.com/russross/meddler v1.0.0
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -68,6 +68,7 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
@@ -444,6 +445,7 @@ github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
@@ -544,23 +546,27 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
 github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=

synchronizer/metrics.go

@@ -0,0 +1,44 @@
+package synchronizer
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	metricReorgsCount = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "sync_reorgs",
+			Help: "",
+		},
+	)
+	metricSyncedLastBlockNum = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "sync_synced_last_block_num",
+			Help: "",
+		},
+	)
+	metricEthLastBlockNum = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "sync_eth_last_block_num",
+			Help: "",
+		},
+	)
+	metricSyncedLastBatchNum = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "sync_synced_last_batch_num",
+			Help: "",
+		},
+	)
+	metricEthLastBatchNum = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "sync_eth_last_batch_num",
+			Help: "",
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(metricReorgsCount)
+	prometheus.MustRegister(metricSyncedLastBlockNum)
+	prometheus.MustRegister(metricEthLastBlockNum)
+	prometheus.MustRegister(metricSyncedLastBatchNum)
+	prometheus.MustRegister(metricEthLastBatchNum)
+}

synchronizer/synchronizer.go

@@ -662,12 +662,16 @@ func (s *Synchronizer) Sync2(ctx context.Context,
 	}
 
 	for _, batchData := range rollupData.Batches {
+		metricSyncedLastBatchNum.Set(float64(batchData.Batch.BatchNum))
+		metricEthLastBatchNum.Set(float64(s.stats.Eth.LastBatchNum))
 		log.Debugw("Synced batch",
 			"syncLastBatch", batchData.Batch.BatchNum,
 			"syncBatchesPerc", s.stats.batchesPerc(batchData.Batch.BatchNum),
 			"ethLastBatch", s.stats.Eth.LastBatchNum,
 		)
 	}
+	metricSyncedLastBlockNum.Set(float64(s.stats.Sync.LastBlock.Num))
+	metricEthLastBlockNum.Set(float64(s.stats.Eth.LastBlock.Num))
 	log.Debugw("Synced block",
 		"syncLastBlockNum", s.stats.Sync.LastBlock.Num,
 		"syncBlocksPerc", s.stats.blocksPerc(),

test/debugapi/debugapi.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hermeznetwork/hermez-node/log"
 	"github.com/hermeznetwork/hermez-node/synchronizer"
 	"github.com/hermeznetwork/tracerr"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 func handleNoRoute(c *gin.Context) {
@@ -107,6 +108,8 @@ func (a *DebugAPI) Run(ctx context.Context) error {
 	api.Use(cors.Default())
 	debugAPI := api.Group("/debug")
 
+	debugAPI.GET("/metrics", gin.WrapH(promhttp.Handler()))
+
 	debugAPI.GET("sdb/batchnum", a.handleCurrentBatch)
 	debugAPI.GET("sdb/mtroot", a.handleMTRoot)
 	// Accounts returned by these endpoints will always have BatchNum = 0,

txselector/metrics.go

@@ -0,0 +1,53 @@
+package txselector
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	metricGetL2TxSelection = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "txsel_get_l2_txselecton_total",
+			Help: "",
+		},
+	)
+	metricGetL1L2TxSelection = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "txsel_get_l1_l2_txselecton_total",
+			Help: "",
+		},
+	)
+
+	metricSelectedL1CoordinatorTxs = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "txsel_selected_l1_coordinator_txs",
+			Help: "",
+		},
+	)
+	metricSelectedL1UserTxs = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "txsel_selected_l1_user_txs",
+			Help: "",
+		},
+	)
+	metricSelectedL2Txs = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "txsel_selected_l2_txs",
+			Help: "",
+		},
+	)
+	metricDiscardedL2Txs = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "txsel_discarded_l2_txs",
+			Help: "",
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(metricGetL2TxSelection)
+	prometheus.MustRegister(metricGetL1L2TxSelection)
+
+	prometheus.MustRegister(metricSelectedL1CoordinatorTxs)
+	prometheus.MustRegister(metricSelectedL1UserTxs)
+	prometheus.MustRegister(metricSelectedL2Txs)
+	prometheus.MustRegister(metricDiscardedL2Txs)
+}

txselector/txselector.go

@@ -133,9 +133,11 @@ func (txsel *TxSelector) coordAccountForTokenID(l1CoordinatorTxs []common.L1Tx,
 // included in the next batch.
 func (txsel *TxSelector) GetL2TxSelection(selectionConfig *SelectionConfig) ([]common.Idx,
 	[][]byte, []common.L1Tx, []common.PoolL2Tx, []common.PoolL2Tx, error) {
-	coordIdxs, accCreationAuths, _, l1CoordinatorTxs, l2Txs, discardedL2Txs, err :=
-		txsel.GetL1L2TxSelection(selectionConfig, []common.L1Tx{})
-	return coordIdxs, accCreationAuths, l1CoordinatorTxs, l2Txs, discardedL2Txs, tracerr.Wrap(err)
+	metricGetL2TxSelection.Inc()
+	coordIdxs, accCreationAuths, _, l1CoordinatorTxs, l2Txs,
+		discardedL2Txs, err := txsel.getL1L2TxSelection(selectionConfig, []common.L1Tx{})
+	return coordIdxs, accCreationAuths, l1CoordinatorTxs, l2Txs,
+		discardedL2Txs, tracerr.Wrap(err)
 }
 
 // GetL1L2TxSelection returns the selection of L1 + L2 txs.
@@ -147,6 +149,16 @@ func (txsel *TxSelector) GetL2TxSelection(selectionConfig *SelectionConfig) ([]c
 // creation exists. The L1UserTxs, L1CoordinatorTxs, PoolL2Txs that will be
 // included in the next batch.
 func (txsel *TxSelector) GetL1L2TxSelection(selectionConfig *SelectionConfig,
+	l1UserTxs []common.L1Tx) ([]common.Idx, [][]byte, []common.L1Tx,
+	[]common.L1Tx, []common.PoolL2Tx, []common.PoolL2Tx, error) {
+	metricGetL1L2TxSelection.Inc()
+	coordIdxs, accCreationAuths, l1UserTxs, l1CoordinatorTxs, l2Txs,
+		discardedL2Txs, err := txsel.getL1L2TxSelection(selectionConfig, l1UserTxs)
+	return coordIdxs, accCreationAuths, l1UserTxs, l1CoordinatorTxs, l2Txs,
+		discardedL2Txs, tracerr.Wrap(err)
+}
+
+func (txsel *TxSelector) getL1L2TxSelection(selectionConfig *SelectionConfig,
 	l1UserTxs []common.L1Tx) ([]common.Idx, [][]byte, []common.L1Tx,
 	[]common.L1Tx, []common.PoolL2Tx, []common.PoolL2Tx, error) {
 	// WIP.0: the TxSelector is not optimized and will need a redesign. The
@@ -452,6 +464,11 @@ func (txsel *TxSelector) GetL1L2TxSelection(selectionConfig *SelectionConfig,
 		return nil, nil, nil, nil, nil, nil, tracerr.Wrap(err)
 	}
 
+	metricSelectedL1CoordinatorTxs.Set(float64(len(l1CoordinatorTxs)))
+	metricSelectedL1UserTxs.Set(float64(len(l1UserTxs)))
+	metricSelectedL2Txs.Set(float64(len(finalL2Txs)))
+	metricDiscardedL2Txs.Set(float64(len(discardedL2Txs)))
+
 	return coordIdxs, accAuths, l1UserTxs, l1CoordinatorTxs, finalL2Txs, discardedL2Txs, nil
 }