From 5ec9c2c57656981ced94b94ae964bddf0a539430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez?= <37264926+CPerezz@users.noreply.github.com> Date: Sat, 17 Aug 2024 09:06:52 +0200 Subject: [PATCH] Enable WASM-compat and monitor it in the CI (#142) * fix: Use `target_pointer_size` conditional compilation There are some parts of the code where is needed to de/serialze `usize`s. These, have sizes that vary depending on the target achitecture the code is compiled for. Hence, this adapts the de/serialization to the specific pointer size for which the crate is being compiled to. * change: Support WASM-compatibility and polish Cargo.toml In order to support Wasm-compat and to simplify and improve `Cargo.toml` readability, the follwing changes have been made: - All the deps that can use `parallel` feature, do so. As `rayon` supports non-threaded targets with a fallback option. See: https://docs.rs/rayon-core/1.12.1/rayon_core/index.html#global-fallback-when-threading-is-unsupported - `ark-grumpking` has been brought to `0.5.0-alpha.0` as `0.4.0` appears to not be in `crates.io` anymore. See: https://crates.io/crates/ark-grumpkin/versions - By default, the crate uses `"ark-circom/default"` which selects the `wasmer/sys` feature such that it knows where wasmer is suposed to be run`. - Added a `wasm` feature which forces `ark-circom/wasm` to be used instead. Which internally selects the `wasmer/js` backend to be used such that in-browser execution is possible. - Added `getrandom` with `js` feature as dependency when `wasm32-unknown-unknown` target is selected such that compilation of the crate for testing or simply building is possible. Notice that with `wasi` and other wasm targets, this is not the case as they're automatically supported. For more info, please check: https://docs.rs/getrandom/latest/getrandom/#webassembly-support * feat: Support WASM-compatibility tests in CI Add support for both testing the build of `sonobe/folding-schemes` for WASM-targets and also, it's build as a dependency for a WASM-crate. This includes a build job for the three main supported rust-WASM targets and the same but for a thrid crate creted on-the-fly which uses `sonobe/folding-schemes` as a dependency. * chore: Add README docs about WASM-compat & feats * ci: don't run WASM-compat job if PR is draft * chore: depend on `arnaucube/circom-compat` fork. Since https://github.com/arnaucube/circom-compat/pull/2 was merged, we can already switch to it as we were depending before. * chore: minimal build/test instructions * fix: CI typos * fix: Update CI to use correct feature sets * fix: `ark-grumpkin` versioning issues As mentioned in https://github.com/privacy-scaling-explorations/sonobe/issues/146 there's a big issue that involves some dependencies of the crate. As a temporary fix, this forces the workspace to rely on a "non-existing" version of `ark-grumpkin` which is immediately patched at workspace-level for a custom version that @arnaucube owns with some cherry-picked commits. While this allows the CI to pass and crate to build, a better solution is needed. * fix: Clippy CI avoiding --all-targets * fix: use `wasm` feat only with folding-schemes --- .github/scripts/wasm-target-test-build.sh | 29 ++++++ .github/workflows/ci.yml | 92 +++++++++++++------ README.md | 31 +++++++ folding-schemes/Cargo.toml | 47 +++++----- folding-schemes/src/folding/hypernova/mod.rs | 18 +++- .../src/folding/nova/decider_eth.rs | 2 +- folding-schemes/src/folding/nova/mod.rs | 19 +++- 7 files changed, 181 insertions(+), 57 deletions(-) create mode 100644 .github/scripts/wasm-target-test-build.sh diff --git a/.github/scripts/wasm-target-test-build.sh b/.github/scripts/wasm-target-test-build.sh new file mode 100644 index 0000000..e86ef83 --- /dev/null +++ b/.github/scripts/wasm-target-test-build.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +GIT_ROOT=$(pwd) + +cd /tmp + +# create test project +cargo new foobar +cd foobar + +# set rust-toolchain same as "sonobe" +cp "${GIT_ROOT}/rust-toolchain" . + +# add wasm32-* targets +rustup target add wasm32-unknown-unknown wasm32-wasi + +# add dependencies +cargo add --path "${GIT_ROOT}/folding-schemes" --features wasm, parallel +cargo add getrandom --features js --target wasm32-unknown-unknown + +# test build for wasm32-* targets +cargo build --release --target wasm32-unknown-unknown +cargo build --release --target wasm32-wasi +# Emscripten would require to fetch the `emcc` tooling. Hence we don't build the lib as a dep for it. +# cargo build --release --target wasm32-unknown-emscripten + +# delete test project +cd ../ +rm -rf foobar \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fcf108..b7ef464 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,6 @@ env: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true - jobs: test: @@ -41,17 +40,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + feature_set: [basic] include: - - feature: default + - feature_set: basic + features: --features default,light-test steps: - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 - uses: noir-lang/noirup@v0.1.3 with: - toolchain: nightly - - uses: actions-rs/toolchain@v1 - # use the more efficient nextest - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 + toolchain: nightly - name: Download Circom run: | mkdir -p $HOME/bin @@ -66,16 +64,47 @@ jobs: run: ./folding-schemes/src/frontend/circom/test_folder/compile.sh - name: Execute compile.sh to generate .json from noir run: ./folding-schemes/src/frontend/noir/test_folder/compile.sh - - name: Build - # This build will be reused by nextest, - # and also checks (--all-targets) that benches don't bit-rot - run: cargo build --release --all-targets --no-default-features --features "light-test,${{ matrix.feature }}" - - name: Test - run: | - cargo nextest run --profile ci --release --workspace --no-default-features --features "light-test,${{ matrix.feature }}" - - name: Doctests # nextest does not support doc tests + - name: Run tests + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --workspace --no-default-features ${{ matrix.features }} + - name: Run Doc-tests + uses: actions-rs/cargo@v1 + with: + command: test + args: --doc + + build: + if: github.event.pull_request.draft == false + name: Build target ${{ matrix.target }} + runs-on: ubuntu-latest + strategy: + matrix: + target: + - wasm32-unknown-unknown + - wasm32-wasi + # Ignoring until clear usage is required + # - wasm32-unknown-emscripten + + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + override: false + default: true + - name: Add target + run: rustup target add ${{ matrix.target }} + - name: Wasm-compat build + uses: actions-rs/cargo@v1 + with: + command: build + args: -p folding-schemes --no-default-features --target ${{ matrix.target }} --features "wasm, parallel" + - name: Run wasm-compat script run: | - cargo test --doc + chmod +x .github/scripts/wasm-target-test-build.sh + .github/scripts/wasm-target-test-build.sh + shell: bash examples: if: github.event.pull_request.draft == false @@ -86,7 +115,7 @@ jobs: - uses: actions-rs/toolchain@v1 - uses: noir-lang/noirup@v0.1.3 with: - toolchain: nightly + toolchain: nightly - name: Download Circom run: | mkdir -p $HOME/bin @@ -98,9 +127,9 @@ jobs: curl -sSfL https://github.com/ethereum/solidity/releases/download/v0.8.4/solc-static-linux -o /usr/local/bin/solc chmod +x /usr/local/bin/solc - name: Execute compile.sh to generate .r1cs and .wasm from .circom - run: ./folding-schemes/src/frontend/circom/test_folder/compile.sh + run: ./folding-schemes/src/frontend/circom/test_folder/compile.sh - name: Execute compile.sh to generate .json from noir - run: ./folding-schemes/src/frontend/noir/test_folder/compile.sh + run: ./folding-schemes/src/frontend/noir/test_folder/compile.sh - name: Run examples tests run: cargo test --examples - name: Run examples @@ -119,31 +148,42 @@ jobs: - uses: actions-rs/cargo@v1 with: command: fmt - args: --all -- --check + args: --all --check clippy: if: github.event.pull_request.draft == false name: Clippy lint checks runs-on: ubuntu-latest + strategy: + matrix: + feature_set: [basic, wasm] + include: + - feature_set: basic + features: --features default,light-test + # We only want to test `folding-schemes` package with `wasm` feature. + - feature_set: wasm + features: -p folding-schemes --features wasm,parallel --target wasm32-unknown-unknown steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: components: clippy - uses: Swatinem/rust-cache@v2 + - name: Add target + run: rustup target add wasm32-unknown-unknown - name: Run clippy uses: actions-rs/cargo@v1 with: command: clippy - args: --all-targets --all-features -- -D warnings + args: --no-default-features ${{ matrix.features }} -- -D warnings typos: if: github.event.pull_request.draft == false name: Spell Check with Typos runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Use typos with config file - uses: crate-ci/typos@master - with: - config: .github/workflows/typos.toml + - uses: actions/checkout@v4 + - name: Use typos with config file + uses: crate-ci/typos@master + with: + config: .github/workflows/typos.toml diff --git a/README.md b/README.md index 0e592dd..5b74296 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,41 @@ Available frontends to define the folded circuit: ## Usage +### Build & test +You can test the library for both, WASM-targets and regular ones. +#### Regular targets +Tier-S targets allow the user to simply run `cargo test` or `cargo build` without needing to worry about anything. +**We strongly recommend to test using the `light-test` feature.** Which will omit the computationally intensive parts of the tests such as +generating a SNARK of 4~5M constraints to then verify it. + +#### WASM targets +In order to build the lib for WASM-targets, use the following command: +`cargo build -p folding-schemes --no-default-features --target wasm32-unknown-unknown --features "wasm, parallel"`. +Where the target can be any WASM one and the `parallel` feature is optional. + +**Trying to build for a WASM-target without the `wasm` feature or viceversa will end up in a compilation error.** + ### Docs Detailed usage and design documentation can be found at [Sonobe docs](https://privacy-scaling-explorations.github.io/sonobe-docs/). +### WASM-compatibility & features + +The `sonobe/folding-schemes` crate is the only workspace member that supports WASM-target compilation. But, to have it working, `getrandom/js` needs +to be imported in the `Cargo.toml` of the crate that uses it as dependency. +```toml +[dependencies] +folding-schemes = { version = "0.1.0", default-features = false, features = ["parallel", "wasm"] } +getrandom = { version = "0.2", features = ["js"] } +``` +See more details about `getrandom` here: https://docs.rs/getrandom/latest/getrandom/#webassembly-support. + +Also, notice that: +- `wasm` feature **IS MANDATORY** if compilation to WASM targets is desired. +- `parallel` feature enables some parallelization optimizations available in the crate. +- `light-test` feature runs the computationally-intensive parts of the testing such as the full proof generation for the Eth-decider circuit +of Nova which is approximately 4-5M constraints. **This feature only matters when it comes to running Sonobe's tests.** + ### Folding Schemes introduction Folding schemes efficiently achieve incrementally verifiable computation (IVC), where the prover recursively proves the correct execution of the incremental computations. diff --git a/folding-schemes/Cargo.toml b/folding-schemes/Cargo.toml index 48ce64b..067b6e1 100644 --- a/folding-schemes/Cargo.toml +++ b/folding-schemes/Cargo.toml @@ -4,25 +4,26 @@ version = "0.1.0" edition = "2021" [dependencies] -ark-ec = "^0.4.0" -ark-ff = "^0.4.0" -ark-poly = "^0.4.0" -ark-std = "^0.4.0" -ark-crypto-primitives = { version = "^0.4.0", default-features = false, features = ["r1cs", "sponge", "crh"] } -ark-grumpkin = {version="0.4.0"} -ark-poly-commit = "^0.4.0" +ark-ec = { version = "^0.4.0", default-features = false, features = ["parallel"] } +ark-ff = { version = "^0.4.0", default-features = false, features = ["parallel", "asm"] } +ark-poly = { version = "^0.4.0", default-features = false, features = ["parallel"] } +ark-std = { version = "^0.4.0", default-features = false, features = ["parallel"] } +ark-crypto-primitives = { version = "^0.4.0", default-features = false, features = ["r1cs", "sponge", "crh", "parallel"] } +ark-grumpkin = { version = "0.4.0", default-features = false } +ark-poly-commit = { version = "^0.4.0", default-features = false, features = ["parallel"] } ark-relations = { version = "^0.4.0", default-features = false } -ark-r1cs-std = { version = "0.4.0", default-features = false } # this is patched at the workspace level -ark-snark = { version = "^0.4.0"} -ark-serialize = "^0.4.0" -ark-circom = { git = "https://github.com/arnaucube/circom-compat" } +# this is patched at the workspace level +ark-r1cs-std = { version = "0.4.0", default-features = false, features = ["parallel"] } +ark-snark = { version = "^0.4.0", default-features = false } +ark-serialize = { version = "^0.4.0", default-features = false } +ark-circom = { git = "https://github.com/arnaucube/circom-compat", default-features = false } +ark-groth16 = { version = "^0.4.0", default-features = false, features = ["parallel"]} +ark-bn254 = { version = "^0.4.0", default-features = false } thiserror = "1.0" -rayon = "1.7.0" +rayon = "1" num-bigint = "0.4" num-integer = "0.1" color-eyre = "=0.6.2" -ark-bn254 = {version="0.4.0"} -ark-groth16 = { version = "^0.4.0" } sha3 = "0.10" ark-noname = { git = "https://github.com/dmpierre/ark-noname", branch="feat/sonobe-integration" } noname = { git = "https://github.com/dmpierre/noname" } @@ -44,19 +45,17 @@ rand = "0.8.5" tracing = { version = "0.1", default-features = false, features = [ "attributes" ] } tracing-subscriber = { version = "0.2" } +# This allows the crate to be built when targeting WASM. +# See more at: https://docs.rs/getrandom/#webassembly-support +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +getrandom = { version = "0.2", features = ["js"] } + [features] -default = ["parallel"] +default = ["ark-circom/default", "parallel"] +parallel = [] +wasm = ["ark-circom/wasm"] light-test = [] -parallel = [ - "ark-std/parallel", - "ark-ff/parallel", - "ark-ec/parallel", - "ark-poly/parallel", - "ark-crypto-primitives/parallel", - "ark-r1cs-std/parallel", - ] - [[example]] name = "sha256" diff --git a/folding-schemes/src/folding/hypernova/mod.rs b/folding-schemes/src/folding/hypernova/mod.rs index 309e9a7..38d1ead 100644 --- a/folding-schemes/src/folding/hypernova/mod.rs +++ b/folding-schemes/src/folding/hypernova/mod.rs @@ -593,9 +593,21 @@ where return Err(Error::MaxStep); } - let mut i_bytes: [u8; 8] = [0; 8]; - i_bytes.copy_from_slice(&self.i.into_bigint().to_bytes_le()[..8]); - let i_usize: usize = usize::from_le_bytes(i_bytes); + let i_usize; + + #[cfg(target_pointer_width = "64")] + { + let mut i_bytes: [u8; 8] = [0; 8]; + i_bytes.copy_from_slice(&self.i.into_bigint().to_bytes_le()[..8]); + i_usize = usize::from_le_bytes(i_bytes); + } + + #[cfg(target_pointer_width = "32")] + { + let mut i_bytes: [u8; 4] = [0; 4]; + i_bytes.copy_from_slice(&self.i.into_bigint().to_bytes_le()[..4]); + i_usize = usize::from_le_bytes(i_bytes); + } let z_i1 = self .F diff --git a/folding-schemes/src/folding/nova/decider_eth.rs b/folding-schemes/src/folding/nova/decider_eth.rs index 7508172..d1a3ed1 100644 --- a/folding-schemes/src/folding/nova/decider_eth.rs +++ b/folding-schemes/src/folding/nova/decider_eth.rs @@ -201,7 +201,7 @@ where let (cmW_x, cmW_y) = NonNativeAffineVar::inputize(U.cmW)?; let (cmT_x, cmT_y) = NonNativeAffineVar::inputize(proof.cmT)?; - let public_input: Vec = vec![ + let public_input: Vec = [ vec![pp_hash, i], z_0, z_i, diff --git a/folding-schemes/src/folding/nova/mod.rs b/folding-schemes/src/folding/nova/mod.rs index f1ecaaf..0ae277c 100644 --- a/folding-schemes/src/folding/nova/mod.rs +++ b/folding-schemes/src/folding/nova/mod.rs @@ -502,9 +502,22 @@ where if self.i > C1::ScalarField::from_le_bytes_mod_order(&usize::MAX.to_le_bytes()) { return Err(Error::MaxStep); } - let mut i_bytes: [u8; 8] = [0; 8]; - i_bytes.copy_from_slice(&self.i.into_bigint().to_bytes_le()[..8]); - let i_usize: usize = usize::from_le_bytes(i_bytes); + + let i_usize; + + #[cfg(target_pointer_width = "64")] + { + let mut i_bytes: [u8; 8] = [0; 8]; + i_bytes.copy_from_slice(&self.i.into_bigint().to_bytes_le()[..8]); + i_usize = usize::from_le_bytes(i_bytes); + } + + #[cfg(target_pointer_width = "32")] + { + let mut i_bytes: [u8; 4] = [0; 4]; + i_bytes.copy_from_slice(&self.i.into_bigint().to_bytes_le()[..4]); + i_usize = usize::from_le_bytes(i_bytes); + } let z_i1 = self .F