diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..1d953f4 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use nix diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9f0a052 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..bef97eb --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,82 @@ +name: Build + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '0 0 * * 1' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Cancel Outdated Builds + uses: styfle/cancel-workflow-action@0.9.1 + with: + all_but_latest: true + access_token: ${{ github.token }} + + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: stable + override: true + default: true + components: rustfmt, clippy + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: false + default: false + components: rustfmt + + - uses: Swatinem/rust-cache@v1 + name: Enable Rust Caching + + - name: Format Check + run: cargo +nightly fmt -- --check + + - name: Clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ github.token }} + + - name: Audit + uses: actions-rs/audit-check@v1 + with: + token: ${{ github.token }} + + - name: Check Bench + run: cargo bench --no-run + + - name: Check Ignored Tests + run: cargo test --no-run -- --ignored + + - name: Test + run: bash ./scripts/run_tests.sh + + - name: Generate Documentation + run: | + cargo doc --no-deps --lib --release + cp -R target/doc public + echo '' > public/index.html + + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.ref == 'refs/heads/main' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./public + cname: hyperplonk.docs.espressosys.com diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..6c26082 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,4 @@ +[workspace] +members = [ + "hyperplonk" +] diff --git a/hyperplonk/Cargo.toml b/hyperplonk/Cargo.toml new file mode 100644 index 0000000..55988d4 --- /dev/null +++ b/hyperplonk/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hyperplonk" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/hyperplonk/src/lib.rs b/hyperplonk/src/lib.rs new file mode 100644 index 0000000..1b4a90c --- /dev/null +++ b/hyperplonk/src/lib.rs @@ -0,0 +1,8 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + let result = 2 + 2; + assert_eq!(result, 4); + } +} diff --git a/nix/grcov/default.nix b/nix/grcov/default.nix new file mode 100644 index 0000000..51cb4f6 --- /dev/null +++ b/nix/grcov/default.nix @@ -0,0 +1,28 @@ +{ lib, rustToolchain, rustPlatform, fetchFromGitHub }: + +rustPlatform.buildRustPackage rec { + pname = "grcov"; + version = "v0.8.2"; + + # See https://nixos.org/manual/nixpkgs/stable/#using-community-rust-overlays + nativeBuildInputs = [ + rustToolchain + ]; + + doCheck = false; + + src = fetchFromGitHub { + owner = "mozilla"; + repo = pname; + rev = version; + sha256 = "t1Gj5u4MmXPbQ5jmO9Sstn7aXJ6Ge+AnsmmG2GiAGKE="; + }; + + cargoSha256 = "DRAUeDzNUMg0AGrqU1TdrqBZJw4A2o3YJB0MdwwzefQ="; + + meta = with lib; { + description = "grcov collects and aggregates code coverage information for multiple source files."; + homepage = "https://github.com/mozilla/grcov"; + license = licenses.mpl20; + }; +} diff --git a/nix/nightly.nix b/nix/nightly.nix new file mode 100644 index 0000000..a054b8e --- /dev/null +++ b/nix/nightly.nix @@ -0,0 +1,25 @@ +let + basePkgs = import ./nixpkgs.nix { }; + + rust_overlay = with basePkgs; import (fetchFromGitHub + (lib.importJSON ./oxalica_rust_overlay.json)); + + pkgs = import ./nixpkgs.nix { overlays = [ rust_overlay ]; }; + + nightlyToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal); + grcov = with pkgs; callPackage ./grcov { rustToolchain = nightlyToolchain; }; +in +with pkgs; + +mkShell { + buildInputs = [ + nightlyToolchain + grcov + ] ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.Security + ]; + + shellHook = '' + export RUST_BACKTRACE=full + ''; +} diff --git a/nix/nixpkgs.json b/nix/nixpkgs.json new file mode 100644 index 0000000..cc5cf51 --- /dev/null +++ b/nix/nixpkgs.json @@ -0,0 +1,4 @@ +{ + "url": "https://github.com/nixos/nixpkgs/archive/db8ab32efd3a4ad59044848d889480954e458f25.tar.gz", + "sha256": "1i7ayivjm3rx62qq263jjj55m0nzhn4b99wax25kw6a8zhhwcwjb" +} diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix new file mode 100644 index 0000000..ddfbdc4 --- /dev/null +++ b/nix/nixpkgs.nix @@ -0,0 +1,10 @@ +# Behaves like `` but pinned. Like ``, requires attrset for opt overlays. +attrs: +let + hostpkgs = import {}; + pinnedNixpkgs = hostpkgs.lib.importJSON ./nixpkgs.json; + nixpkgs = builtins.fetchTarball { + url = pinnedNixpkgs.url; + sha256 = pinnedNixpkgs.sha256; + }; +in import nixpkgs attrs diff --git a/nix/oxalica_rust_overlay.json b/nix/oxalica_rust_overlay.json new file mode 100644 index 0000000..98b0237 --- /dev/null +++ b/nix/oxalica_rust_overlay.json @@ -0,0 +1,7 @@ +{ + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "9d7c777625640b70a4d211f62711fa316bca7176", + "sha256": "025bw59nl12jqf4nrvbn0a8xn03aj9bz54nvf1rb25zl2l1nkrnd", + "fetchSubmodules": true +} diff --git a/nix/pre-commit.nix b/nix/pre-commit.nix new file mode 100644 index 0000000..8d3552b --- /dev/null +++ b/nix/pre-commit.nix @@ -0,0 +1,34 @@ +{ pkgs, ... }: + +let + nix-pre-commit-hooks = import (pkgs.fetchFromGitHub { + owner = "cachix"; + repo = "pre-commit-hooks.nix"; + rev = "ff9c0b459ddc4b79c06e19d44251daa8e9cd1746"; + sha256 = "jlsQb2y6A5dB1R0wVPLOfDGM0wLyfYqEJNzMtXuzCXw="; + }); +in +nix-pre-commit-hooks.run { + src = ./.; + hooks = { + check-format = { + enable = true; + files = "\\.rs$"; + entry = "cargo fmt -- --check"; + }; + doctest = { + enable = true; + entry = "cargo test --doc"; + files = "\\.rs$"; + pass_filenames = false; + }; + # The hook "clippy" that ships with nix-precommit-hooks is outdated. + cargo-clippy = { + enable = true; + description = "Lint Rust code."; + entry = "cargo-clippy"; + files = "\\.rs$"; + pass_filenames = false; + }; + }; +} diff --git a/nix/update-nix b/nix/update-nix new file mode 100755 index 0000000..e25d6d8 --- /dev/null +++ b/nix/update-nix @@ -0,0 +1,27 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p curl jq nix +# +# Updates nixpkgs.json to the latest or chosen nixpkgs revision +# +# Usage: ./update-nix +# ./update-nix $rev +# ./update-nix $owner $rev +# +# Arguments default to owner=nixos and rev=master and refer +# to the github owner of a nixpkgs fork and a git revision. +# +set -exo pipefail + +owner="nixos" + +if [ ! -z "$2" ]; then + owner="$1" + rev="$2" +else + rev="${1:-master}" +fi + +resolved_rev=$(curl "https://api.github.com/repos/${owner}/nixpkgs/commits?sha=${rev}" | jq -r 'first.sha') +url="https://github.com/${owner}/nixpkgs/archive/${resolved_rev}.tar.gz" +digest=$(nix-prefetch-url --unpack "$url") +echo "{\"url\": \"${url}\", \"sha256\": \"${digest}\"}" | jq '.' > nix/nixpkgs.json diff --git a/nix/update-rust-overlay b/nix/update-rust-overlay new file mode 100755 index 0000000..df2349c --- /dev/null +++ b/nix/update-rust-overlay @@ -0,0 +1,5 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p nix-prefetch-github +set -exo pipefail + +nix-prefetch-github oxalica rust-overlay | tee nix/oxalica_rust_overlay.json diff --git a/nix/vagrant/.gitignore b/nix/vagrant/.gitignore new file mode 100644 index 0000000..51f8e48 --- /dev/null +++ b/nix/vagrant/.gitignore @@ -0,0 +1,2 @@ +hyperplonk/ +.vagrant diff --git a/nix/vagrant/README.md b/nix/vagrant/README.md new file mode 100644 index 0000000..96255fb --- /dev/null +++ b/nix/vagrant/README.md @@ -0,0 +1,66 @@ +# Test nix-shell in vagrant VMs + +Set up a vagrant guest VM, and test the dev environment inside the guest. + +- Only tested on nixos host with _libvirt_ virtualization provider. +- Assumes that the host has an SSH agent. The agent is used for SSH auth inside + the guest. +- Upon creation (`vagrant up`) a copy of this local repo is rsynced to the + `/hyperplonk` directory in the guest. The tests are run against these files. To + see changes made to the code on the host run `vagrant reload` to re-sync the + source code from host to guest. + +## Available vagrant boxes +The following boxes are available: + + - `ubuntu`: `ubuntu20.04` + `nix` + - `ubuntu_rustup`: `ubuntu20.04` + `nix` + `rustup` + +More OSes/VMs can be added in the `Vagrantfile`. + +Append name of box after vagrant command to apply to a single box only + + vagrant up ubuntu_rustup + vagrant ssh ubuntu_rustup + +## Usage +Enable `libvrtd` on your host: +[ubuntu](https://ubuntu.com/server/docs/virtualization-libvirt), +[nixos](https://nixos.wiki/wiki/Libvirt). + +Make sure we are in the `libvirtd` group. + +Install `libvirt` vagrant plugin (not needed on nixos): + + vagrant plugin install vagrant-libvirt + +Activate nix-shell in this directory (or ensure vagrant is installed): + + nix-shell + +Start vm: + + vagrant up ubuntu + +There is a password prompt to add the insecure vagrant key to the agent. One can +supply an empty password once or cancel the prompt each time one runs `vagrant +ssh`. + +Run formatter, linter, tests inside a nix-shell environment inside the `ubuntu` +guest: + + vagrant ssh ubuntu -- -t /vagrant/test-nix-shell-guest + +This runs the `test-nix-shell-guest` script in this directory inside the vagrant +guest. + +Clean up with + + vagrant destroy ubuntu + +## Notes + +- After editing the Vagrantfile, `vagrant reload` will apply the changes. +- When making substantial changes or changing names of vagrant boxes I usually + have more luck with running `vagrant destroy` with the previous `Vagrantfile` + and then `vagrant up` again with the new `Vagrantfile`. diff --git a/nix/vagrant/Vagrantfile b/nix/vagrant/Vagrantfile new file mode 100644 index 0000000..6fab702 --- /dev/null +++ b/nix/vagrant/Vagrantfile @@ -0,0 +1,36 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + + config.vm.define "ubuntu" do |ubuntu| + ubuntu.vm.box = "generic/ubuntu2004" + $script = <<~SCRIPT + set -euxo pipefail + curl -L https://nixos.org/nix/install | sh + SCRIPT + ubuntu.vm.provision "shell", inline: $script, privileged: false + end + + config.vm.define "ubuntu_rustup" do |ubuntu| + ubuntu.vm.box = "generic/ubuntu2004" + $script = <<~SCRIPT + set -euxo pipefail + curl -L https://nixos.org/nix/install | sh + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source $HOME/.cargo/env + rustup default stable-2021-06-17 + SCRIPT + ubuntu.vm.provision "shell", inline: $script, privileged: false + end + + config.ssh.forward_agent = true + config.vm.synced_folder ".", "/vagrant", disabled: false + config.vm.synced_folder "../..", "/hyperplonk", disabled: false, rsync__exclude: [".git/", "target"] + + + config.vm.provider "libvirt" do |v| + v.cpus = 4 + end + +end diff --git a/nix/vagrant/shell.nix b/nix/vagrant/shell.nix new file mode 100644 index 0000000..0a6b6cb --- /dev/null +++ b/nix/vagrant/shell.nix @@ -0,0 +1,7 @@ +with import ../nixpkgs.nix { }; + +mkShell { + buildInputs = [ + vagrant + ]; +} diff --git a/nix/vagrant/test-nix-shell-guest b/nix/vagrant/test-nix-shell-guest new file mode 100755 index 0000000..81b9e50 --- /dev/null +++ b/nix/vagrant/test-nix-shell-guest @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# vagrant "ssh + command" does not source, adding -- -t does not help +. $HOME/.nix-profile/etc/profile.d/nix.sh +if [ -f $HOME/.carg/env ]; then + source $HOME/.cargo/env +fi + +ssh-keyscan gitlab.com >> ~/.ssh/known_hosts +ssh -T git@gitlab.com + +cd /hyperplonk + +nix-shell --run "cargo-clippy" +nix-shell --run "cargo fmt -- --check" +nix-shell --run "cargo test --doc" +nix-shell --run "cargo test --release" + +echo "Ok!" diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..b288fc8 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,9 @@ +reorder_imports = true +wrap_comments = true +normalize_comments = true +use_try_shorthand = true +match_block_trailing_comma = true +use_field_init_shorthand = true +edition = "2018" +condense_wildcard_suffixes = true +imports_granularity = "Crate" diff --git a/scripts/run_benchmarks.m4 b/scripts/run_benchmarks.m4 new file mode 100755 index 0000000..26df9f8 --- /dev/null +++ b/scripts/run_benchmarks.m4 @@ -0,0 +1,37 @@ +#!/bin/bash + +# m4_ignore( +echo "This is just a script template, not the script (yet) - pass it to 'argbash' to fix this." >&2 +exit 11 #)Created by argbash-init v2.10.0 +# ARG_OPTIONAL_BOOLEAN([asm]) +# ARG_OPTIONAL_BOOLEAN([multi_threads]) +# ARG_HELP([]) +# ARGBASH_GO + +# [ <-- needed because of Argbash + +if [ "$_arg_multi_threads" = on ] +then + echo "Multi-threads: ON" + # Do nothing +else + echo "Multi-threads: OFF" + export RAYON_NUM_THREADS=1 +fi + +if [ "$_arg_asm" = on ] +then + echo "Asm feature: ON" + export RUSTFLAGS="-C target-feature=+bmi2,+adx" +else + echo "Asm feature: OFF" + # Do nothing +fi + +# Run the benchmark binary +cargo +nightly bench + + +# ^^^ TERMINATE YOUR CODE BEFORE THE BOTTOM ARGBASH MARKER ^^^ + +# ] <-- needed because of Argbash diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh new file mode 100755 index 0000000..72d554a --- /dev/null +++ b/scripts/run_benchmarks.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Created by argbash-init v2.10.0 +# ARG_OPTIONAL_BOOLEAN([asm]) +# ARG_OPTIONAL_BOOLEAN([multi_threads]) +# ARG_HELP([]) +# ARGBASH_GO() +# needed because of Argbash --> m4_ignore([ +### START OF CODE GENERATED BY Argbash v2.10.0 one line above ### +# Argbash is a bash code generator used to get arguments parsing right. +# Argbash is FREE SOFTWARE, see https://argbash.io for more info + + +die() +{ + local _ret="${2:-1}" + test "${_PRINT_HELP:-no}" = yes && print_help >&2 + echo "$1" >&2 + exit "${_ret}" +} + + +begins_with_short_option() +{ + local first_option all_short_options='h' + first_option="${1:0:1}" + test "$all_short_options" = "${all_short_options/$first_option/}" && return 1 || return 0 +} + +# THE DEFAULTS INITIALIZATION - OPTIONALS +_arg_asm="off" +_arg_multi_threads="off" + + +print_help() +{ + printf '%s\n' "" + printf 'Usage: %s [--(no-)asm] [--(no-)multi_threads] [-h|--help]\n' "$0" + printf '\t%s\n' "-h, --help: Prints help" +} + + +parse_commandline() +{ + while test $# -gt 0 + do + _key="$1" + case "$_key" in + --no-asm|--asm) + _arg_asm="on" + test "${1:0:5}" = "--no-" && _arg_asm="off" + ;; + --no-multi_threads|--multi_threads) + _arg_multi_threads="on" + test "${1:0:5}" = "--no-" && _arg_multi_threads="off" + ;; + -h|--help) + print_help + exit 0 + ;; + -h*) + print_help + exit 0 + ;; + *) + _PRINT_HELP=yes die "FATAL ERROR: Got an unexpected argument '$1'" 1 + ;; + esac + shift + done +} + +parse_commandline "$@" + +# OTHER STUFF GENERATED BY Argbash + +### END OF CODE GENERATED BY Argbash (sortof) ### ]) +# [ <-- needed because of Argbash + +cargo clean + +if [ "$_arg_multi_threads" = on ] +then + echo "Multi-threads: ON" + # Do nothing +else + echo "Multi-threads: OFF" + export RAYON_NUM_THREADS=1 +fi + +if [ "$_arg_asm" = on ] +then + echo "Asm feature: ON" + export RUSTFLAGS="-C target-feature=+bmi2,+adx" +else + echo "Asm feature: OFF" + # Do nothing +fi + +# Run the benchmark binary +cargo bench + + +# ^^^ TERMINATE YOUR CODE BEFORE THE BOTTOM ARGBASH MARKER ^^^ + +# ] <-- needed because of Argbash diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh new file mode 100755 index 0000000..5fab546 --- /dev/null +++ b/scripts/run_tests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# We want the code to panic if there is an integer overflow +export RUSTFLAGS="-C overflow-checks=on" + +cargo test --release -- -Zunstable-options --report-time + diff --git a/scripts/test_coverage.sh b/scripts/test_coverage.sh new file mode 100755 index 0000000..64be660 --- /dev/null +++ b/scripts/test_coverage.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env nix-shell +#!nix-shell ../nix/nightly.nix -i bash +set -o xtrace +IGNORED_FILES="--ignore **/errors.rs\ + --ignore **/src/bin/*\ + --ignore transactions/src/parameters.rs\ + --ignore transactions/src/bench_utils/*\ + " +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=3 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests" +export RUSTDOCFLAGS="" +rm -vf ./target/**/*.gcda +cargo build +cargo test --lib +grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing $IGNORED_FILES -o ./target/debug/coverage/ +echo "Coverage report available at target/debug/coverage/index.html." diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..993c4be --- /dev/null +++ b/shell.nix @@ -0,0 +1,45 @@ +let + basePkgs = import ./nix/nixpkgs.nix { }; + + rust_overlay = with basePkgs; import (fetchFromGitHub + (lib.importJSON ./nix/oxalica_rust_overlay.json)); + + pkgs = import ./nix/nixpkgs.nix { overlays = [ rust_overlay ]; }; + + nightlyToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal.override { + extensions = [ "rustfmt" ]; + }); + + stableToolchain = pkgs.rust-bin.stable."1.56.1".minimal.override { + extensions = [ "clippy" "llvm-tools-preview" "rust-src" ]; + }; + + pre-commit-check = pkgs.callPackage ./nix/pre-commit.nix { }; +in +with pkgs; + +mkShell { + buildInputs = [ + argbash + openssl + pkgconfig + git + + stableToolchain + nightlyToolchain + + ] ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.Security + ]; + + shellHook = '' + export RUST_BACKTRACE=full + export PATH="$PATH:$(pwd)/target/debug:$(pwd)/target/release" + + # Ensure `cargo fmt` uses `rustfmt` from nightly. + export RUSTFMT="${nightlyToolchain}/bin/rustfmt" + + # install pre-commit hooks + ${pre-commit-check.shellHook} + ''; +}