env setup (#6)

.envrc

@@ -0,0 +1 @@
+use nix

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -0,0 +1,82 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Outdated Builds
+        uses: styfle/cancel-workflow-action@0.9.1
+        with:
+          all_but_latest: true
+          access_token: ${{ github.token }}
+
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: stable
+          override: true
+          default: true
+          components: rustfmt, clippy
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: false
+          default: false
+          components: rustfmt
+
+      - uses: Swatinem/rust-cache@v1
+        name: Enable Rust Caching
+
+      - name: Format Check
+        run: cargo +nightly fmt -- --check
+
+      - name: Clippy
+        uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ github.token }}
+
+      - name: Audit
+        uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ github.token }}
+
+      - name: Check Bench
+        run: cargo bench --no-run
+
+      - name: Check Ignored Tests
+        run: cargo test --no-run -- --ignored
+
+      - name: Test
+        run: bash ./scripts/run_tests.sh      
+
+      - name: Generate Documentation
+        run: |
+          cargo doc --no-deps --lib --release
+          cp -R target/doc public
+          echo '<meta http-equiv="refresh" content="0; url=jf_plonk">' > public/index.html
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.ref == 'refs/heads/main' }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./public
+          cname: hyperplonk.docs.espressosys.com

Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+members = [
+  "hyperplonk"
+]

hyperplonk/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "hyperplonk"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

hyperplonk/src/lib.rs

@@ -0,0 +1,8 @@
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn it_works() {
+        let result = 2 + 2;
+        assert_eq!(result, 4);
+    }
+}

nix/grcov/default.nix

@@ -0,0 +1,28 @@
+{ lib, rustToolchain, rustPlatform, fetchFromGitHub }:
+
+rustPlatform.buildRustPackage rec {
+  pname = "grcov";
+  version = "v0.8.2";
+
+  # See https://nixos.org/manual/nixpkgs/stable/#using-community-rust-overlays
+  nativeBuildInputs = [
+    rustToolchain
+  ];
+
+  doCheck = false;
+
+  src = fetchFromGitHub {
+    owner = "mozilla";
+    repo = pname;
+    rev = version;
+    sha256 = "t1Gj5u4MmXPbQ5jmO9Sstn7aXJ6Ge+AnsmmG2GiAGKE=";
+  };
+
+  cargoSha256 = "DRAUeDzNUMg0AGrqU1TdrqBZJw4A2o3YJB0MdwwzefQ=";
+
+  meta = with lib; {
+    description = "grcov collects and aggregates code coverage information for multiple source files.";
+    homepage = "https://github.com/mozilla/grcov";
+    license = licenses.mpl20;
+  };
+}

nix/nightly.nix

@@ -0,0 +1,25 @@
+let
+  basePkgs = import ./nixpkgs.nix { };
+
+  rust_overlay = with basePkgs; import (fetchFromGitHub
+    (lib.importJSON ./oxalica_rust_overlay.json));
+
+  pkgs = import ./nixpkgs.nix { overlays = [ rust_overlay ]; };
+
+  nightlyToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal);
+  grcov = with pkgs; callPackage ./grcov { rustToolchain = nightlyToolchain; };
+in
+with pkgs;
+
+mkShell {
+  buildInputs = [
+    nightlyToolchain
+    grcov
+  ] ++ lib.optionals stdenv.isDarwin [
+    darwin.apple_sdk.frameworks.Security
+  ];
+
+  shellHook = ''
+    export RUST_BACKTRACE=full
+  '';
+}

nix/nixpkgs.json

@@ -0,0 +1,4 @@
+{
+  "url": "https://github.com/nixos/nixpkgs/archive/db8ab32efd3a4ad59044848d889480954e458f25.tar.gz",
+  "sha256": "1i7ayivjm3rx62qq263jjj55m0nzhn4b99wax25kw6a8zhhwcwjb"
+}

nix/nixpkgs.nix

@@ -0,0 +1,10 @@
+# Behaves like `<nixpkgs>` but pinned. Like `<nixpkgs>`, requires attrset for opt overlays.
+attrs:
+let
+  hostpkgs = import <nixpkgs> {};
+  pinnedNixpkgs = hostpkgs.lib.importJSON ./nixpkgs.json;
+  nixpkgs = builtins.fetchTarball {
+    url = pinnedNixpkgs.url;
+    sha256 = pinnedNixpkgs.sha256;
+  };
+in import nixpkgs attrs

nix/oxalica_rust_overlay.json

@@ -0,0 +1,7 @@
+{
+    "owner": "oxalica",
+    "repo": "rust-overlay",
+    "rev": "9d7c777625640b70a4d211f62711fa316bca7176",
+    "sha256": "025bw59nl12jqf4nrvbn0a8xn03aj9bz54nvf1rb25zl2l1nkrnd",
+    "fetchSubmodules": true
+}

nix/pre-commit.nix

@@ -0,0 +1,34 @@
+{ pkgs, ... }:
+
+let
+  nix-pre-commit-hooks = import (pkgs.fetchFromGitHub {
+    owner = "cachix";
+    repo = "pre-commit-hooks.nix";
+    rev = "ff9c0b459ddc4b79c06e19d44251daa8e9cd1746";
+    sha256 = "jlsQb2y6A5dB1R0wVPLOfDGM0wLyfYqEJNzMtXuzCXw=";
+  });
+in
+nix-pre-commit-hooks.run {
+  src = ./.;
+  hooks = {
+    check-format = {
+      enable = true;
+      files = "\\.rs$";
+      entry = "cargo fmt -- --check";
+    };
+    doctest = {
+      enable = true;
+      entry = "cargo test --doc";
+      files = "\\.rs$";
+      pass_filenames = false;
+    };
+    # The hook "clippy" that ships with nix-precommit-hooks is outdated.
+    cargo-clippy = {
+      enable = true;
+      description = "Lint Rust code.";
+      entry = "cargo-clippy";
+      files = "\\.rs$";
+      pass_filenames = false;
+    };
+  };
+}

nix/update-nix

@@ -0,0 +1,27 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p curl jq nix
+#
+#   Updates nixpkgs.json to the latest or chosen nixpkgs revision
+#
+#   Usage: ./update-nix
+#          ./update-nix $rev
+#          ./update-nix $owner $rev
+#
+#   Arguments default to owner=nixos and rev=master and refer
+#   to the github owner of a nixpkgs fork and a git revision.
+#
+set -exo pipefail
+
+owner="nixos"
+
+if [ ! -z "$2" ]; then
+    owner="$1"
+    rev="$2"
+else
+    rev="${1:-master}"
+fi
+
+resolved_rev=$(curl "https://api.github.com/repos/${owner}/nixpkgs/commits?sha=${rev}" | jq -r 'first.sha')
+url="https://github.com/${owner}/nixpkgs/archive/${resolved_rev}.tar.gz"
+digest=$(nix-prefetch-url --unpack "$url")
+echo "{\"url\": \"${url}\", \"sha256\": \"${digest}\"}" | jq '.' > nix/nixpkgs.json

nix/update-rust-overlay

@@ -0,0 +1,5 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p nix-prefetch-github
+set -exo pipefail
+
+nix-prefetch-github oxalica rust-overlay | tee nix/oxalica_rust_overlay.json

nix/vagrant/.gitignore

@@ -0,0 +1,2 @@
+hyperplonk/
+.vagrant

nix/vagrant/README.md

@@ -0,0 +1,66 @@
+# Test nix-shell in vagrant VMs
+
+Set up a vagrant guest VM, and test the dev environment inside the guest.
+
+- Only tested on nixos host with _libvirt_ virtualization provider.
+- Assumes that the host has an SSH agent. The agent is used for SSH auth inside
+  the guest.
+- Upon creation (`vagrant up`) a copy of this local repo is rsynced to the
+  `/hyperplonk` directory in the guest. The tests are run against these files. To
+  see changes made to the code on the host run `vagrant reload` to re-sync the
+  source code from host to guest.
+
+## Available vagrant boxes
+The following boxes are available:
+
+  - `ubuntu`: `ubuntu20.04` + `nix`
+  - `ubuntu_rustup`: `ubuntu20.04` + `nix` + `rustup`
+
+More OSes/VMs can be added in the `Vagrantfile`.
+
+Append name of box after vagrant command to apply to a single box only
+
+    vagrant up ubuntu_rustup
+    vagrant ssh ubuntu_rustup
+
+## Usage
+Enable `libvrtd` on your host:
+[ubuntu](https://ubuntu.com/server/docs/virtualization-libvirt),
+[nixos](https://nixos.wiki/wiki/Libvirt).
+
+Make sure we are in the `libvirtd` group.
+
+Install `libvirt` vagrant plugin (not needed on nixos):
+
+    vagrant plugin install vagrant-libvirt
+
+Activate nix-shell in this directory (or ensure vagrant is installed):
+
+    nix-shell
+
+Start vm:
+
+    vagrant up ubuntu
+
+There is a password prompt to add the insecure vagrant key to the agent. One can
+supply an empty password once or cancel the prompt each time one runs `vagrant
+ssh`.
+
+Run formatter, linter, tests inside a nix-shell environment inside the `ubuntu`
+guest:
+
+    vagrant ssh ubuntu -- -t /vagrant/test-nix-shell-guest
+
+This runs the `test-nix-shell-guest` script in this directory inside the vagrant
+guest.
+
+Clean up with
+
+    vagrant destroy ubuntu
+
+## Notes
+
+- After editing the Vagrantfile, `vagrant reload` will apply the changes.
+- When making substantial changes or changing names of vagrant boxes I usually
+  have more luck with running `vagrant destroy` with the previous `Vagrantfile`
+  and then `vagrant up` again with the new `Vagrantfile`.

nix/vagrant/Vagrantfile

@@ -0,0 +1,36 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+
+  config.vm.define "ubuntu" do |ubuntu|
+    ubuntu.vm.box = "generic/ubuntu2004"
+    $script = <<~SCRIPT
+      set -euxo pipefail
+      curl -L https://nixos.org/nix/install | sh
+    SCRIPT
+    ubuntu.vm.provision "shell", inline: $script, privileged: false
+  end
+
+  config.vm.define "ubuntu_rustup" do |ubuntu|
+    ubuntu.vm.box = "generic/ubuntu2004"
+    $script = <<~SCRIPT
+      set -euxo pipefail
+      curl -L https://nixos.org/nix/install | sh
+      curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      source $HOME/.cargo/env
+      rustup default stable-2021-06-17
+    SCRIPT
+    ubuntu.vm.provision "shell", inline: $script, privileged: false
+  end
+
+  config.ssh.forward_agent = true
+  config.vm.synced_folder  ".", "/vagrant", disabled: false
+  config.vm.synced_folder  "../..", "/hyperplonk", disabled: false, rsync__exclude: [".git/", "target"]
+
+
+  config.vm.provider "libvirt" do |v|
+    v.cpus = 4
+  end
+
+end

nix/vagrant/shell.nix

@@ -0,0 +1,7 @@
+with import ../nixpkgs.nix { };
+
+mkShell {
+  buildInputs = [
+    vagrant
+  ];
+}

nix/vagrant/test-nix-shell-guest

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# vagrant "ssh + command" does not source, adding -- -t does not help
+. $HOME/.nix-profile/etc/profile.d/nix.sh
+if [ -f $HOME/.carg/env ]; then
+    source $HOME/.cargo/env
+fi
+
+ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
+ssh -T git@gitlab.com
+
+cd /hyperplonk
+
+nix-shell --run "cargo-clippy"
+nix-shell --run "cargo fmt -- --check"
+nix-shell --run "cargo test --doc"
+nix-shell --run "cargo test --release"
+
+echo "Ok!"

rustfmt.toml

@@ -0,0 +1,9 @@
+reorder_imports = true
+wrap_comments = true
+normalize_comments = true
+use_try_shorthand = true
+match_block_trailing_comma = true
+use_field_init_shorthand = true
+edition = "2018"
+condense_wildcard_suffixes = true
+imports_granularity = "Crate"

scripts/run_benchmarks.m4

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# m4_ignore(
+echo "This is just a script template, not the script (yet) - pass it to 'argbash' to fix this." >&2
+exit 11  #)Created by argbash-init v2.10.0
+# ARG_OPTIONAL_BOOLEAN([asm])
+# ARG_OPTIONAL_BOOLEAN([multi_threads])
+# ARG_HELP([<Hyperplonk benchmarks>])
+# ARGBASH_GO
+
+# [ <-- needed because of Argbash
+
+if [ "$_arg_multi_threads" = on ]
+then
+  echo "Multi-threads: ON"
+  # Do nothing
+else
+  echo "Multi-threads: OFF"
+  export RAYON_NUM_THREADS=1
+fi
+
+if [ "$_arg_asm" = on ]
+then
+  echo "Asm feature: ON"
+  export RUSTFLAGS="-C target-feature=+bmi2,+adx"
+else
+  echo "Asm feature: OFF"
+  # Do nothing
+fi
+
+# Run the benchmark binary
+cargo +nightly bench
+
+
+# ^^^  TERMINATE YOUR CODE BEFORE THE BOTTOM ARGBASH MARKER  ^^^
+
+# ] <-- needed because of Argbash

scripts/run_benchmarks.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+# Created by argbash-init v2.10.0
+# ARG_OPTIONAL_BOOLEAN([asm])
+# ARG_OPTIONAL_BOOLEAN([multi_threads])
+# ARG_HELP([<Hyperplonk benchmarks>])
+# ARGBASH_GO()
+# needed because of Argbash --> m4_ignore([
+### START OF CODE GENERATED BY Argbash v2.10.0 one line above ###
+# Argbash is a bash code generator used to get arguments parsing right.
+# Argbash is FREE SOFTWARE, see https://argbash.io for more info
+
+
+die()
+{
+	local _ret="${2:-1}"
+	test "${_PRINT_HELP:-no}" = yes && print_help >&2
+	echo "$1" >&2
+	exit "${_ret}"
+}
+
+
+begins_with_short_option()
+{
+	local first_option all_short_options='h'
+	first_option="${1:0:1}"
+	test "$all_short_options" = "${all_short_options/$first_option/}" && return 1 || return 0
+}
+
+# THE DEFAULTS INITIALIZATION - OPTIONALS
+_arg_asm="off"
+_arg_multi_threads="off"
+
+
+print_help()
+{
+	printf '%s\n' "<Hyperplonk benchmarks>"
+	printf 'Usage: %s [--(no-)asm] [--(no-)multi_threads] [-h|--help]\n' "$0"
+	printf '\t%s\n' "-h, --help: Prints help"
+}
+
+
+parse_commandline()
+{
+	while test $# -gt 0
+	do
+		_key="$1"
+		case "$_key" in
+			--no-asm|--asm)
+				_arg_asm="on"
+				test "${1:0:5}" = "--no-" && _arg_asm="off"
+				;;
+			--no-multi_threads|--multi_threads)
+				_arg_multi_threads="on"
+				test "${1:0:5}" = "--no-" && _arg_multi_threads="off"
+				;;
+			-h|--help)
+				print_help
+				exit 0
+				;;
+			-h*)
+				print_help
+				exit 0
+				;;
+			*)
+				_PRINT_HELP=yes die "FATAL ERROR: Got an unexpected argument '$1'" 1
+				;;
+		esac
+		shift
+	done
+}
+
+parse_commandline "$@"
+
+# OTHER STUFF GENERATED BY Argbash
+
+### END OF CODE GENERATED BY Argbash (sortof) ### ])
+# [ <-- needed because of Argbash
+
+cargo clean
+
+if [ "$_arg_multi_threads" = on ]
+then
+  echo "Multi-threads: ON"
+  # Do nothing
+else
+  echo "Multi-threads: OFF"
+  export RAYON_NUM_THREADS=1
+fi
+
+if [ "$_arg_asm" = on ]
+then
+  echo "Asm feature: ON"
+  export RUSTFLAGS="-C target-feature=+bmi2,+adx"
+else
+  echo "Asm feature: OFF"
+  # Do nothing
+fi
+
+# Run the benchmark binary
+cargo bench
+
+
+# ^^^  TERMINATE YOUR CODE BEFORE THE BOTTOM ARGBASH MARKER  ^^^
+
+# ] <-- needed because of Argbash

scripts/run_tests.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# We want the code to panic if there is an integer overflow
+export RUSTFLAGS="-C overflow-checks=on"
+
+cargo test --release -- -Zunstable-options --report-time
+

scripts/test_coverage.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env nix-shell
+#!nix-shell ../nix/nightly.nix -i bash
+set -o xtrace
+IGNORED_FILES="--ignore **/errors.rs\
+               --ignore **/src/bin/*\
+               --ignore transactions/src/parameters.rs\
+               --ignore transactions/src/bench_utils/*\
+              "
+export CARGO_INCREMENTAL=0
+export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=3 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests"
+export RUSTDOCFLAGS=""
+rm -vf ./target/**/*.gcda
+cargo build
+cargo test --lib
+grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing  $IGNORED_FILES -o ./target/debug/coverage/
+echo "Coverage report available at target/debug/coverage/index.html."

shell.nix

@@ -0,0 +1,45 @@
+let
+  basePkgs = import ./nix/nixpkgs.nix { };
+
+  rust_overlay = with basePkgs; import (fetchFromGitHub
+    (lib.importJSON ./nix/oxalica_rust_overlay.json));
+
+  pkgs = import ./nix/nixpkgs.nix { overlays = [ rust_overlay ]; };
+
+  nightlyToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal.override {
+    extensions = [ "rustfmt" ];
+  });
+
+  stableToolchain = pkgs.rust-bin.stable."1.56.1".minimal.override {
+    extensions = [ "clippy" "llvm-tools-preview" "rust-src" ];
+  };
+
+  pre-commit-check = pkgs.callPackage ./nix/pre-commit.nix { };
+in
+with pkgs;
+
+mkShell {
+  buildInputs = [
+    argbash
+    openssl
+    pkgconfig
+    git
+
+    stableToolchain
+    nightlyToolchain
+
+  ] ++ lib.optionals stdenv.isDarwin [
+    darwin.apple_sdk.frameworks.Security
+  ];
+
+  shellHook = ''
+    export RUST_BACKTRACE=full
+    export PATH="$PATH:$(pwd)/target/debug:$(pwd)/target/release"
+
+    # Ensure `cargo fmt` uses `rustfmt` from nightly.
+    export RUSTFMT="${nightlyToolchain}/bin/rustfmt"
+
+    # install pre-commit hooks
+    ${pre-commit-check.shellHook}
+  '';
+}