arnaucube
arnaucube/hyperplonkPublic Self-hosted repository
env setup (#6)

zhenfei zhenfei.zhang@hotmail.com
Wed, 20 Apr 2022 17:10:25 -0400
commit

9d4d1784557a5e9ef9730a967128ecab14a6eda9

parent

5b2265b633bd08623c388e927bbfbba53ec4706a

A .envrc

@@ -0,0 +1,1 @@

+use nix
A .github/dependabot.yml

@@ -0,0 +1,11 @@

+version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "daily"
A .github/workflows/build.yml

@@ -0,0 +1,82 @@

+name: Build + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '0 0 * * 1' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Cancel Outdated Builds + uses: styfle/cancel-workflow-action@0.9.1 + with: + all_but_latest: true + access_token: ${{ github.token }} + + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: stable + override: true + default: true + components: rustfmt, clippy + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: false + default: false + components: rustfmt + + - uses: Swatinem/rust-cache@v1 + name: Enable Rust Caching + + - name: Format Check + run: cargo +nightly fmt -- --check + + - name: Clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ github.token }} + + - name: Audit + uses: actions-rs/audit-check@v1 + with: + token: ${{ github.token }} + + - name: Check Bench + run: cargo bench --no-run + + - name: Check Ignored Tests + run: cargo test --no-run -- --ignored + + - name: Test + run: bash ./scripts/run_tests.sh + + - name: Generate Documentation + run: | + cargo doc --no-deps --lib --release + cp -R target/doc public + echo '<meta http-equiv="refresh" content="0; url=jf_plonk">' > public/index.html + + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.ref == 'refs/heads/main' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./public + cname: hyperplonk.docs.espressosys.com
A Cargo.toml

@@ -0,0 +1,4 @@

+[workspace] +members = [ + "hyperplonk" +]
A hyperplonk/Cargo.toml

@@ -0,0 +1,8 @@

+[package] +name = "hyperplonk" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies]
A hyperplonk/src/lib.rs

@@ -0,0 +1,8 @@

+#[cfg(test)] +mod tests { + #[test] + fn it_works() { + let result = 2 + 2; + assert_eq!(result, 4); + } +}
A nix/grcov/default.nix

@@ -0,0 +1,28 @@

+{ lib, rustToolchain, rustPlatform, fetchFromGitHub }: + +rustPlatform.buildRustPackage rec { + pname = "grcov"; + version = "v0.8.2"; + + # See https://nixos.org/manual/nixpkgs/stable/#using-community-rust-overlays + nativeBuildInputs = [ + rustToolchain + ]; + + doCheck = false; + + src = fetchFromGitHub { + owner = "mozilla"; + repo = pname; + rev = version; + sha256 = "t1Gj5u4MmXPbQ5jmO9Sstn7aXJ6Ge+AnsmmG2GiAGKE="; + }; + + cargoSha256 = "DRAUeDzNUMg0AGrqU1TdrqBZJw4A2o3YJB0MdwwzefQ="; + + meta = with lib; { + description = "grcov collects and aggregates code coverage information for multiple source files."; + homepage = "https://github.com/mozilla/grcov"; + license = licenses.mpl20; + }; +}
A nix/nightly.nix

@@ -0,0 +1,25 @@

+let + basePkgs = import ./nixpkgs.nix { }; + + rust_overlay = with basePkgs; import (fetchFromGitHub + (lib.importJSON ./oxalica_rust_overlay.json)); + + pkgs = import ./nixpkgs.nix { overlays = [ rust_overlay ]; }; + + nightlyToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal); + grcov = with pkgs; callPackage ./grcov { rustToolchain = nightlyToolchain; }; +in +with pkgs; + +mkShell { + buildInputs = [ + nightlyToolchain + grcov + ] ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.Security + ]; + + shellHook = '' + export RUST_BACKTRACE=full + ''; +}
A nix/nixpkgs.json

@@ -0,0 +1,4 @@

+{ + "url": "https://github.com/nixos/nixpkgs/archive/db8ab32efd3a4ad59044848d889480954e458f25.tar.gz", + "sha256": "1i7ayivjm3rx62qq263jjj55m0nzhn4b99wax25kw6a8zhhwcwjb" +}
A nix/nixpkgs.nix

@@ -0,0 +1,10 @@

+# Behaves like `<nixpkgs>` but pinned. Like `<nixpkgs>`, requires attrset for opt overlays. +attrs: +let + hostpkgs = import <nixpkgs> {}; + pinnedNixpkgs = hostpkgs.lib.importJSON ./nixpkgs.json; + nixpkgs = builtins.fetchTarball { + url = pinnedNixpkgs.url; + sha256 = pinnedNixpkgs.sha256; + }; +in import nixpkgs attrs
A nix/oxalica_rust_overlay.json

@@ -0,0 +1,7 @@

+{ + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "9d7c777625640b70a4d211f62711fa316bca7176", + "sha256": "025bw59nl12jqf4nrvbn0a8xn03aj9bz54nvf1rb25zl2l1nkrnd", + "fetchSubmodules": true +}
A nix/pre-commit.nix

@@ -0,0 +1,34 @@

+{ pkgs, ... }: + +let + nix-pre-commit-hooks = import (pkgs.fetchFromGitHub { + owner = "cachix"; + repo = "pre-commit-hooks.nix"; + rev = "ff9c0b459ddc4b79c06e19d44251daa8e9cd1746"; + sha256 = "jlsQb2y6A5dB1R0wVPLOfDGM0wLyfYqEJNzMtXuzCXw="; + }); +in +nix-pre-commit-hooks.run { + src = ./.; + hooks = { + check-format = { + enable = true; + files = "\\.rs$"; + entry = "cargo fmt -- --check"; + }; + doctest = { + enable = true; + entry = "cargo test --doc"; + files = "\\.rs$"; + pass_filenames = false; + }; + # The hook "clippy" that ships with nix-precommit-hooks is outdated. + cargo-clippy = { + enable = true; + description = "Lint Rust code."; + entry = "cargo-clippy"; + files = "\\.rs$"; + pass_filenames = false; + }; + }; +}
A nix/update-nix

@@ -0,0 +1,27 @@

+#!/usr/bin/env nix-shell +#! nix-shell -i bash -p curl jq nix +# +# Updates nixpkgs.json to the latest or chosen nixpkgs revision +# +# Usage: ./update-nix +# ./update-nix $rev +# ./update-nix $owner $rev +# +# Arguments default to owner=nixos and rev=master and refer +# to the github owner of a nixpkgs fork and a git revision. +# +set -exo pipefail + +owner="nixos" + +if [ ! -z "$2" ]; then + owner="$1" + rev="$2" +else + rev="${1:-master}" +fi + +resolved_rev=$(curl "https://api.github.com/repos/${owner}/nixpkgs/commits?sha=${rev}" | jq -r 'first.sha') +url="https://github.com/${owner}/nixpkgs/archive/${resolved_rev}.tar.gz" +digest=$(nix-prefetch-url --unpack "$url") +echo "{\"url\": \"${url}\", \"sha256\": \"${digest}\"}" | jq '.' > nix/nixpkgs.json
A nix/update-rust-overlay

@@ -0,0 +1,5 @@

+#!/usr/bin/env nix-shell +#! nix-shell -i bash -p nix-prefetch-github +set -exo pipefail + +nix-prefetch-github oxalica rust-overlay | tee nix/oxalica_rust_overlay.json
A nix/vagrant/.gitignore

@@ -0,0 +1,2 @@

+hyperplonk/ +.vagrant
A nix/vagrant/README.md

@@ -0,0 +1,66 @@

+# Test nix-shell in vagrant VMs + +Set up a vagrant guest VM, and test the dev environment inside the guest. + +- Only tested on nixos host with _libvirt_ virtualization provider. +- Assumes that the host has an SSH agent. The agent is used for SSH auth inside + the guest. +- Upon creation (`vagrant up`) a copy of this local repo is rsynced to the + `/hyperplonk` directory in the guest. The tests are run against these files. To + see changes made to the code on the host run `vagrant reload` to re-sync the + source code from host to guest. + +## Available vagrant boxes +The following boxes are available: + + - `ubuntu`: `ubuntu20.04` + `nix` + - `ubuntu_rustup`: `ubuntu20.04` + `nix` + `rustup` + +More OSes/VMs can be added in the `Vagrantfile`. + +Append name of box after vagrant command to apply to a single box only + + vagrant up ubuntu_rustup + vagrant ssh ubuntu_rustup + +## Usage +Enable `libvrtd` on your host: +[ubuntu](https://ubuntu.com/server/docs/virtualization-libvirt), +[nixos](https://nixos.wiki/wiki/Libvirt). + +Make sure we are in the `libvirtd` group. + +Install `libvirt` vagrant plugin (not needed on nixos): + + vagrant plugin install vagrant-libvirt + +Activate nix-shell in this directory (or ensure vagrant is installed): + + nix-shell + +Start vm: + + vagrant up ubuntu + +There is a password prompt to add the insecure vagrant key to the agent. One can +supply an empty password once or cancel the prompt each time one runs `vagrant +ssh`. + +Run formatter, linter, tests inside a nix-shell environment inside the `ubuntu` +guest: + + vagrant ssh ubuntu -- -t /vagrant/test-nix-shell-guest + +This runs the `test-nix-shell-guest` script in this directory inside the vagrant +guest. + +Clean up with + + vagrant destroy ubuntu + +## Notes + +- After editing the Vagrantfile, `vagrant reload` will apply the changes. +- When making substantial changes or changing names of vagrant boxes I usually + have more luck with running `vagrant destroy` with the previous `Vagrantfile` + and then `vagrant up` again with the new `Vagrantfile`.
A nix/vagrant/Vagrantfile

@@ -0,0 +1,36 @@

+# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + + config.vm.define "ubuntu" do |ubuntu| + ubuntu.vm.box = "generic/ubuntu2004" + $script = <<~SCRIPT + set -euxo pipefail + curl -L https://nixos.org/nix/install | sh + SCRIPT + ubuntu.vm.provision "shell", inline: $script, privileged: false + end + + config.vm.define "ubuntu_rustup" do |ubuntu| + ubuntu.vm.box = "generic/ubuntu2004" + $script = <<~SCRIPT + set -euxo pipefail + curl -L https://nixos.org/nix/install | sh + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source $HOME/.cargo/env + rustup default stable-2021-06-17 + SCRIPT + ubuntu.vm.provision "shell", inline: $script, privileged: false + end + + config.ssh.forward_agent = true + config.vm.synced_folder ".", "/vagrant", disabled: false + config.vm.synced_folder "../..", "/hyperplonk", disabled: false, rsync__exclude: [".git/", "target"] + + + config.vm.provider "libvirt" do |v| + v.cpus = 4 + end + +end
A nix/vagrant/shell.nix

@@ -0,0 +1,7 @@

+with import ../nixpkgs.nix { }; + +mkShell { + buildInputs = [ + vagrant + ]; +}
A nix/vagrant/test-nix-shell-guest

@@ -0,0 +1,20 @@

+#!/usr/bin/env bash +set -euxo pipefail + +# vagrant "ssh + command" does not source, adding -- -t does not help +. $HOME/.nix-profile/etc/profile.d/nix.sh +if [ -f $HOME/.carg/env ]; then + source $HOME/.cargo/env +fi + +ssh-keyscan gitlab.com >> ~/.ssh/known_hosts +ssh -T git@gitlab.com + +cd /hyperplonk + +nix-shell --run "cargo-clippy" +nix-shell --run "cargo fmt -- --check" +nix-shell --run "cargo test --doc" +nix-shell --run "cargo test --release" + +echo "Ok!"
A rustfmt.toml

@@ -0,0 +1,9 @@

+reorder_imports = true +wrap_comments = true +normalize_comments = true +use_try_shorthand = true +match_block_trailing_comma = true +use_field_init_shorthand = true +edition = "2018" +condense_wildcard_suffixes = true +imports_granularity = "Crate"
A scripts/run_benchmarks.m4

@@ -0,0 +1,37 @@

+#!/bin/bash + +# m4_ignore( +echo "This is just a script template, not the script (yet) - pass it to 'argbash' to fix this." >&2 +exit 11 #)Created by argbash-init v2.10.0 +# ARG_OPTIONAL_BOOLEAN([asm]) +# ARG_OPTIONAL_BOOLEAN([multi_threads]) +# ARG_HELP([<Hyperplonk benchmarks>]) +# ARGBASH_GO + +# [ <-- needed because of Argbash + +if [ "$_arg_multi_threads" = on ] +then + echo "Multi-threads: ON" + # Do nothing +else + echo "Multi-threads: OFF" + export RAYON_NUM_THREADS=1 +fi + +if [ "$_arg_asm" = on ] +then + echo "Asm feature: ON" + export RUSTFLAGS="-C target-feature=+bmi2,+adx" +else + echo "Asm feature: OFF" + # Do nothing +fi + +# Run the benchmark binary +cargo +nightly bench + + +# ^^^ TERMINATE YOUR CODE BEFORE THE BOTTOM ARGBASH MARKER ^^^ + +# ] <-- needed because of Argbash
A scripts/run_benchmarks.sh

@@ -0,0 +1,106 @@

+#!/usr/bin/env bash + +# Created by argbash-init v2.10.0 +# ARG_OPTIONAL_BOOLEAN([asm]) +# ARG_OPTIONAL_BOOLEAN([multi_threads]) +# ARG_HELP([<Hyperplonk benchmarks>]) +# ARGBASH_GO() +# needed because of Argbash --> m4_ignore([ +### START OF CODE GENERATED BY Argbash v2.10.0 one line above ### +# Argbash is a bash code generator used to get arguments parsing right. +# Argbash is FREE SOFTWARE, see https://argbash.io for more info + + +die() +{ + local _ret="${2:-1}" + test "${_PRINT_HELP:-no}" = yes && print_help >&2 + echo "$1" >&2 + exit "${_ret}" +} + + +begins_with_short_option() +{ + local first_option all_short_options='h' + first_option="${1:0:1}" + test "$all_short_options" = "${all_short_options/$first_option/}" && return 1 || return 0 +} + +# THE DEFAULTS INITIALIZATION - OPTIONALS +_arg_asm="off" +_arg_multi_threads="off" + + +print_help() +{ + printf '%s\n' "<Hyperplonk benchmarks>" + printf 'Usage: %s [--(no-)asm] [--(no-)multi_threads] [-h|--help]\n' "$0" + printf '\t%s\n' "-h, --help: Prints help" +} + + +parse_commandline() +{ + while test $# -gt 0 + do + _key="$1" + case "$_key" in + --no-asm|--asm) + _arg_asm="on" + test "${1:0:5}" = "--no-" && _arg_asm="off" + ;; + --no-multi_threads|--multi_threads) + _arg_multi_threads="on" + test "${1:0:5}" = "--no-" && _arg_multi_threads="off" + ;; + -h|--help) + print_help + exit 0 + ;; + -h*) + print_help + exit 0 + ;; + *) + _PRINT_HELP=yes die "FATAL ERROR: Got an unexpected argument '$1'" 1 + ;; + esac + shift + done +} + +parse_commandline "$@" + +# OTHER STUFF GENERATED BY Argbash + +### END OF CODE GENERATED BY Argbash (sortof) ### ]) +# [ <-- needed because of Argbash + +cargo clean + +if [ "$_arg_multi_threads" = on ] +then + echo "Multi-threads: ON" + # Do nothing +else + echo "Multi-threads: OFF" + export RAYON_NUM_THREADS=1 +fi + +if [ "$_arg_asm" = on ] +then + echo "Asm feature: ON" + export RUSTFLAGS="-C target-feature=+bmi2,+adx" +else + echo "Asm feature: OFF" + # Do nothing +fi + +# Run the benchmark binary +cargo bench + + +# ^^^ TERMINATE YOUR CODE BEFORE THE BOTTOM ARGBASH MARKER ^^^ + +# ] <-- needed because of Argbash
A scripts/run_tests.sh

@@ -0,0 +1,7 @@

+#!/usr/bin/env bash + +# We want the code to panic if there is an integer overflow +export RUSTFLAGS="-C overflow-checks=on" + +cargo test --release -- -Zunstable-options --report-time +
A scripts/test_coverage.sh

@@ -0,0 +1,16 @@

+#!/usr/bin/env nix-shell +#!nix-shell ../nix/nightly.nix -i bash +set -o xtrace +IGNORED_FILES="--ignore **/errors.rs\ + --ignore **/src/bin/*\ + --ignore transactions/src/parameters.rs\ + --ignore transactions/src/bench_utils/*\ + " +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=3 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests" +export RUSTDOCFLAGS="" +rm -vf ./target/**/*.gcda +cargo build +cargo test --lib +grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing $IGNORED_FILES -o ./target/debug/coverage/ +echo "Coverage report available at target/debug/coverage/index.html."
A shell.nix

@@ -0,0 +1,45 @@

+let + basePkgs = import ./nix/nixpkgs.nix { }; + + rust_overlay = with basePkgs; import (fetchFromGitHub + (lib.importJSON ./nix/oxalica_rust_overlay.json)); + + pkgs = import ./nix/nixpkgs.nix { overlays = [ rust_overlay ]; }; + + nightlyToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal.override { + extensions = [ "rustfmt" ]; + }); + + stableToolchain = pkgs.rust-bin.stable."1.56.1".minimal.override { + extensions = [ "clippy" "llvm-tools-preview" "rust-src" ]; + }; + + pre-commit-check = pkgs.callPackage ./nix/pre-commit.nix { }; +in +with pkgs; + +mkShell { + buildInputs = [ + argbash + openssl + pkgconfig + git + + stableToolchain + nightlyToolchain + + ] ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.Security + ]; + + shellHook = '' + export RUST_BACKTRACE=full + export PATH="$PATH:$(pwd)/target/debug:$(pwd)/target/release" + + # Ensure `cargo fmt` uses `rustfmt` from nightly. + export RUSTFMT="${nightlyToolchain}/bin/rustfmt" + + # install pre-commit hooks + ${pre-commit-check.shellHook} + ''; +}