diff --git a/.gitignore b/.gitignore index e69de29..6dd29b7 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +bin/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9544cc3 --- /dev/null +++ b/Makefile @@ -0,0 +1,135 @@ +#! /usr/bin/make -f + +# Project variables. +PACKAGE := github.com/hermeznetwork/hermez-node +VERSION := $(shell git describe --tags --always) +BUILD := $(shell git rev-parse --short HEAD) +BUILD_DATE := $(shell date +%Y-%m-%dT%H:%M:%S%z) +PROJECT_NAME := $(shell basename "$(PWD)") + +# Go related variables. +GO_FILES ?= $$(find . -name '*.go' | grep -v vendor) +GOBASE := $(shell pwd) +GOBIN := $(GOBASE)/bin +GOPKG := $(.) +GOENVVARS := GOBIN=$(GOBIN) +GOCMD := $(GOBASE)/cli/node +GOPROOF := $(GOBASE)/test/proofserver/cli +GOBINARY := node + +# Project configs. +MODE ?= sync +CONFIG ?= $(GOBASE)/cli/node/cfg.buidler.toml +POSTGRES_PASS ?= yourpasswordhere + +# Use linker flags to provide version/build settings. +LDFLAGS=-ldflags "-X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -X=main.Date=$(BUILD_DATE)" + +# PID file will keep the process id of the server. +PID_PROOF_MOCK := /tmp/.$(PROJECT_NAME).proof.pid + +# Make is verbose in Linux. Make it silent. +MAKEFLAGS += --silent + +.PHONY: help +help: Makefile + @echo + @echo " Choose a command run in "$(PROJECT_NAME)":" + @echo + @sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /' + @echo + +## test: Run the application check and all tests. +test: govet gocilint test-unit + +## test-unit: Run all unit tests. +test-unit: + @echo " > Running unit tests" + $(GOENVVARS) go test -race -p 1 -failfast -timeout 300s -v ./... + +## test-api-server: Run the API server using the Go tests. +test-api-server: + @echo " > Running unit tests" + $(GOENVVARS) FAKE_SERVER=yes go test -timeout 0 ./api -p 1 -count 1 -v + +## gofmt: Run `go fmt` for all go files. +gofmt: + @echo " > Format all go files" + $(GOENVVARS) gofmt -w ${GO_FILES} + +## govet: Run go vet. +govet: + @echo " > Running go vet" + $(GOENVVARS) go vet ./... + +## golint: Run default golint. +golint: + @echo " > Running golint" + $(GOENVVARS) golint -set_exit_status ./... + +## gocilint: Run Golang CI Lint. +gocilint: + @echo " > Running Golang CI Lint" + $-golangci-lint run --timeout=5m -E whitespace -E gosec -E gci -E misspell -E gomnd -E gofmt -E goimports -E golint --exclude-use-default=false --max-same-issues 0 + +## exec: Run given command. e.g; make exec run="go test ./..." +exec: + GOBIN=$(GOBIN) $(run) + +## clean: Clean build files. Runs `go clean` internally. +clean: + @-rm $(GOBIN)/ 2> /dev/null + @echo " > Cleaning build cache" + $(GOENVVARS) go clean + +## build: Build the project. +build: install + @echo " > Building Hermez binary..." + @bash -c "$(MAKE) migration-pack" + $(GOENVVARS) go build $(LDFLAGS) -o $(GOBIN)/$(GOBINARY) $(GOCMD) + @bash -c "$(MAKE) migration-clean" + +## install: Install missing dependencies. Runs `go get` internally. e.g; make install get=github.com/foo/bar +install: + @echo " > Checking if there is any missing dependencies..." + $(GOENVVARS) go get $(GOCMD)/... $(get) + +## run: Run Hermez node. +run: + @bash -c "$(MAKE) clean build" + @echo " > Running $(PROJECT_NAME)" + @$(GOBIN)/$(GOBINARY) --mode $(MODE) --cfg $(CONFIG) run + +## run-proof-mock: Run proof server mock API. +run-proof-mock: stop-proof-mock + @echo " > Running Proof Server Mock" + $(GOENVVARS) go build -o $(GOBIN)/proof $(GOPROOF) + @$(GOBIN)/proof 2>&1 & echo $$! > $(PID_PROOF_MOCK) + @cat $(PID_PROOF_MOCK) | sed "/^/s/^/ \> Proof Server Mock PID: /" + +## stop-proof-mock: Stop proof server mock API. +stop-proof-mock: + @-touch $(PID_PROOF_MOCK) + @-kill -s INT `cat $(PID_PROOF_MOCK)` 2> /dev/null || true + @-rm $(PID_PROOF_MOCK) $(GOBIN)/proof 2> /dev/null || true + +## migration-pack: Pack the database migrations into the binary. +migration-pack: + @echo " > Packing the migrations..." + @cd /tmp && go get -u github.com/gobuffalo/packr/v2/packr2 && cd - + @cd $(GOBASE)/db && packr2 && cd - + +## migration-clean: Clean the database migrations pack. +migration-clean: + @echo " > Cleaning the migrations..." + @cd $(GOBASE)/db && packr2 clean && cd - + +## run-database-container: Run the Postgres container +run-database-container: + @echo " > Running the postgreSQL DB..." + @-docker run --rm --name hermez-db -p 5432:5432 -e POSTGRES_DB=hermez -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$(POSTGRES_PASS)" -d postgres + +## stop-database-container: Stop the Postgres container +stop-database-container: + @echo " > Stopping the postgreSQL DB..." + @-docker stop hermez-db diff --git a/README.md b/README.md index 3873531..12b0a21 100644 --- a/README.md +++ b/README.md @@ -8,42 +8,75 @@ Go implementation of the Hermez node. The `hermez-node` has been tested with go version 1.14 +### Build + +Build the binary and check the current version: + +```shell +$ make build +$ bin/node version +``` + +### Run + +First you must edit the default/template config file into [cli/node/cfg.buidler.toml](cli/node/cfg.buidler.toml), +there are more information about the config file into [cli/node/README.md](cli/node/README.md) + +After setting the config, you can build and run the Hermez Node as a synchronizer: + +```shell +$ make run +``` + +Or build and run as a coordinator, and also passing the config file from other location: + +```shell +$ MODE=sync CONFIG=cli/node/cfg.buidler.toml make run +``` + +To check the useful make commands: + +```shell +$ make help +``` + ### Unit testing Running the unit tests requires a connection to a PostgreSQL database. You can -start PostgreSQL with docker easily this way (where `yourpasswordhere` should +run PostgreSQL with docker easily this way (where `yourpasswordhere` should be your password): -``` -POSTGRES_PASS=yourpasswordhere; sudo docker run --rm --name hermez-db-test -p 5432:5432 -e POSTGRES_DB=hermez -e POSTGRES_USER=hermez -e POSTGRES_PASSWORD="$POSTGRES_PASS" -d postgres +```shell +$ POSTGRES_PASS="yourpasswordhere" make run-database-container ``` -Afterwards, run the tests with the password as env var: +Afterward, run the tests with the password as env var: -``` -POSTGRES_PASS=yourpasswordhere go test -p 1 ./... +```shell +$ POSTGRES_PASS="yourpasswordhere" make test ``` -NOTE: `-p 1` forces execution of package test in serial. Otherwise they may be -executed in paralel and the test may find unexpected entries in the SQL databse +NOTE: `-p 1` forces execution of package test in serial. Otherwise, they may be +executed in parallel, and the test may find unexpected entries in the SQL database because it's shared among all tests. -There is an extra temporary option that allows you to run the API server using -the Go tests. This will be removed once the API can be properly initialized, -with data from the synchronizer and so on. To use this, run: +There is an extra temporary option that allows you to run the API server using the +Go tests. It will be removed once the API can be properly initialized with data +from the synchronizer. To use this, run: -``` -FAKE_SERVER=yes POSTGRES_PASS=yourpasswordhere go test -timeout 0 ./api -p 1 -count 1 -v` +```shell +$ POSTGRES_PASS="yourpasswordhere" make test-api-server ``` ### Lint All Pull Requests need to pass the configured linter. -To run the linter locally, first install [golangci-lint](https://golangci-lint.run). Afterwards you can check the lints with this command: +To run the linter locally, first, install [golangci-lint](https://golangci-lint.run). +Afterward, you can check the lints with this command: -``` -golangci-lint run --timeout=5m -E whitespace -E gosec -E gci -E misspell -E gomnd -E gofmt -E goimports -E golint --exclude-use-default=false --max-same-issues 0 +```shell +$ make gocilint ``` ## Usage @@ -54,13 +87,13 @@ See [cli/node/README.md](cli/node/README.md) ### Proof Server -The node in mode coordinator requires a proof server (a server that is capable -of calculating proofs from the zkInputs). For testing purposes there is a mock -proof server cli at `test/proofserver/cli`. +The node in mode coordinator requires a proof server (a server capable of +calculating proofs from the zkInputs). There is a mock proof server CLI +at `test/proofserver/cli` for testing purposes. Usage of `test/proofserver/cli`: -``` +```shell USAGE: go run ./test/proofserver/cli OPTIONS @@ -71,11 +104,19 @@ OPTIONS: proving time duration (default 2s) ``` +Also, the Makefile commands can be used to run and stop the proof server +in the background: + +```shell +$ make run-proof-mock +$ make stop-proof-mock +``` + ### `/tmp` as tmpfs For every processed batch, the node builds a temporary exit tree in a key-value DB stored in `/tmp`. It is highly recommended that `/tmp` is mounted as a RAM -file system in production to avoid unecessary reads an writes to disk. This +file system in production to avoid unnecessary reads a writes to disk. This can be done by mounting `/tmp` as tmpfs; for example, by having this line in `/etc/fstab`: ``` diff --git a/cli/node/README.md b/cli/node/README.md index 073ac82..82756c7 100644 --- a/cli/node/README.md +++ b/cli/node/README.md @@ -8,7 +8,7 @@ The `hermez-node` has been tested with go version 1.14 ## Usage -``` +```shell NAME: hermez-node - A new cli application @@ -16,18 +16,18 @@ USAGE: node [global options] command [command options] [arguments...] VERSION: - 0.1.0-alpha + v0.1.0-6-gd8a50c5 COMMANDS: + version Show the application version importkey Import ethereum private key genbjj Generate a new BabyJubJub key - wipesql Wipe the SQL DB (HistoryDB and L2DB), leaving the DB in a clean state + wipesql Wipe the SQL DB (HistoryDB and L2DB) and the StateDBs, leaving the DB in a clean state run Run the hermez-node in the indicated mode + discard Discard blocks up to a specified block number help, h Shows a list of commands or help for one command GLOBAL OPTIONS: - --mode MODE Set node MODE (can be "sync" or "coord") - --cfg FILE Node configuration FILE --help, -h show help (default: false) --version, -v print the version (default: false) ``` @@ -75,7 +75,7 @@ when running the coordinator in sync mode Building the node requires using the packr utility to bundle the database migrations inside the resulting binary. Install the packr utility with: -``` +```shell cd /tmp && go get -u github.com/gobuffalo/packr/v2/packr2 && cd - ``` @@ -83,7 +83,7 @@ Make sure your `$PATH` contains `$GOPATH/bin`, otherwise the packr utility will not be found. Now build the node executable: -``` +```shell cd ../../db && packr2 && cd - go build . cd ../../db && packr2 clean && cd - @@ -98,35 +98,40 @@ run the following examples by replacing `./node` with `go run .` and executing them in the `cli/node` directory to build from source and run at the same time. Run the node in mode synchronizer: -``` -./node --mode sync --cfg cfg.buidler.toml run +```shell +./node run --mode sync --cfg cfg.buidler.toml ``` Run the node in mode coordinator: -``` -./node --mode coord --cfg cfg.buidler.toml run +```shell +./node run --mode coord --cfg cfg.buidler.toml ``` Import an ethereum private key into the keystore: -``` -./node --mode coord --cfg cfg.buidler.toml importkey --privatekey 0x618b35096c477aab18b11a752be619f0023a539bb02dd6c813477a6211916cde +```shell +./node importkey --mode coord --cfg cfg.buidler.toml --privatekey 0x618b35096c477aab18b11a752be619f0023a539bb02dd6c813477a6211916cde ``` Generate a new BabyJubJub key pair: +```shell +./node genbjj ``` -./node --mode coord --cfg cfg.buidler.toml genbjj + +Check the binary version: +```shell +./node version ``` Wipe the entier SQL database (this will destroy all synchronized and pool data): -``` -./node --mode coord --cfg cfg.buidler.toml wipesql +```shell +./node wipesql --mode coord --cfg cfg.buidler.toml ``` Discard all synchronized blocks and associated state up to a given block number. This command is useful in case the synchronizer reaches an invalid state and you want to roll back a few blocks and try again (maybe with some fixes in the code). -``` -./node --mode coord --cfg cfg.buidler.toml discard --block 8061330 +```shell +./node discard --mode coord --cfg cfg.buidler.toml --block 8061330 ```