0.5.18

cli.js

@@ -24,6 +24,7 @@
 const fs = require("fs");
 const path = require("path");
 const Scalar = require("ffjavascript").Scalar;
+const fastFile = require("fastfile");
 
 const compiler = require("./src/compiler");
 
@@ -60,87 +61,65 @@ const argv = require("yargs")
     .argv;
 
 
-let inputFile;
+async function run() {
-if (argv._.length == 0) {
+    let inputFile;
-    inputFile = "circuit.circom";
+    if (argv._.length == 0) {
-} else if (argv._.length == 1) {
+        inputFile = "circuit.circom";
-    inputFile = argv._[0];
+    } else if (argv._.length == 1) {
-} else  {
+        inputFile = argv._[0];
-    console.log("Only one circuit at a time is permited");
+    } else  {
-    process.exit(1);
+        console.log("Only one circuit at a time is permited");
-}
+        process.exit(1);
+    }
 
-const fullFileName = path.resolve(process.cwd(), inputFile);
+    const fullFileName = path.resolve(process.cwd(), inputFile);
-const fileName = path.basename(fullFileName, ".circom");
+    const fileName = path.basename(fullFileName, ".circom");
-const cSourceName = typeof(argv.csource) === "string" ?  argv.csource : fileName + ".cpp";
+    const cSourceName = typeof(argv.csource) === "string" ?  argv.csource : fileName + ".cpp";
-const wasmName = typeof(argv.wasm) === "string" ?  argv.wasm : fileName + ".wasm";
+    const wasmName = typeof(argv.wasm) === "string" ?  argv.wasm : fileName + ".wasm";
-const watName = typeof(argv.wat) === "string" ?  argv.wat : fileName + ".wat";
+    const watName = typeof(argv.wat) === "string" ?  argv.wat : fileName + ".wat";
-const r1csName = typeof(argv.r1cs) === "string" ?  argv.r1cs : fileName + ".r1cs";
+    const r1csName = typeof(argv.r1cs) === "string" ?  argv.r1cs : fileName + ".r1cs";
-const symName = typeof(argv.sym) === "string" ?  argv.sym : fileName + ".sym";
+    const symName = typeof(argv.sym) === "string" ?  argv.sym : fileName + ".sym";
 
-const options = {};
+    const options = {};
-options.reduceConstraints = !argv.fast;
+    options.reduceConstraints = !argv.fast;
-options.verbose = argv.verbose || false;
+    options.verbose = argv.verbose || false;
-options.sanityCheck = argv.sanitycheck;
+    options.sanityCheck = argv.sanitycheck;
 
-if (argv.csource) {
+    if (argv.csource) {
-    options.cSourceWriteStream = fs.createWriteStream(cSourceName);
+        options.cSourceFile = await fastFile.createOverride(cSourceName);
-}
+    }
-if (argv.wasm) {
+    if (argv.wasm) {
-    options.wasmWriteStream = fs.createWriteStream(wasmName);
+        options.wasmFile = await fastFile.createOverride(wasmName);
-}
+    }
-if (argv.wat) {
+    if (argv.wat) {
-    options.watWriteStream = fs.createWriteStream(watName);
+        options.watFile = await fastFile.createOverride(watName);
-}
+    }
-if (argv.r1cs) {
+    if (argv.r1cs) {
-    options.r1csFileName = r1csName;
+        options.r1csFileName = r1csName;
-}
+    }
-if (argv.sym) {
+    if (argv.sym) {
-    options.symWriteStream = fs.createWriteStream(symName);
+        options.symWriteStream = fs.createWriteStream(symName);
-}
+    }
-if (argv.newThreadTemplates) {
+    if (argv.newThreadTemplates) {
-    options.newThreadTemplates = new RegExp(argv.newThreadTemplates);
+        options.newThreadTemplates = new RegExp(argv.newThreadTemplates);
-}
+    }
-if (!argv.prime) {
+    if (!argv.prime) {
-    options.prime = Scalar.fromString("21888242871839275222246405745257275088548364400416034343698204186575808495617");
+        options.prime = Scalar.fromString("21888242871839275222246405745257275088548364400416034343698204186575808495617");
-} else if (["BLS12-381", "BLS12381"]. indexOf(argv.prime.toUpperCase()) >=0) {
+    } else if (["BLS12-381", "BLS12381"]. indexOf(argv.prime.toUpperCase()) >=0) {
-    options.prime = Scalar.fromString("73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001",16);
+        options.prime = Scalar.fromString("73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001",16);
-} else if (["BN-128", "BN128", "BN254", "BN-254"]. indexOf(argv.prime.toUpperCase()) >=0) {
+    } else if (["BN-128", "BN128", "BN254", "BN-254"]. indexOf(argv.prime.toUpperCase()) >=0) {
-    options.prime = Scalar.fromString("21888242871839275222246405745257275088548364400416034343698204186575808495617");
+        options.prime = Scalar.fromString("21888242871839275222246405745257275088548364400416034343698204186575808495617");
-} else {
+    } else {
-    options.prime = Scalar.fromString(argv.prime);
+        options.prime = Scalar.fromString(argv.prime);
-}
+    }
 
 
-compiler(fullFileName, options).then( () => {
+    await compiler(fullFileName, options);
-    let cSourceDone = false;
+
-    let wasmDone = false;
+    if (options.cSourceFile) await options.cSourceFile.close();
+    if (options.wasmFile) await options.wasmFile.close();
+    if (options.watFile) await options.watFile.close();
     let symDone = false;
-    let watDone = false;
-    if (options.cSourceWriteStream) {
-        options.cSourceWriteStream.on("finish", () => {
-            cSourceDone = true;
-            finishIfDone();
-        });
-    } else {
-        cSourceDone = true;
-    }
-    if (options.wasmWriteStream) {
-        options.wasmWriteStream.on("finish", () => {
-            wasmDone = true;
-            finishIfDone();
-        });
-    } else {
-        wasmDone = true;
-    }
-    if (options.watWriteStream) {
-        options.watWriteStream.on("finish", () => {
-            watDone = true;
-            finishIfDone();
-        });
-    } else {
-        watDone = true;
-    }
     if (options.symWriteStream) {
         options.symWriteStream.on("finish", () => {
             symDone = true;
@@ -150,12 +129,17 @@ compiler(fullFileName, options).then( () => {
         symDone = true;
     }
     function finishIfDone() {
-        if ((cSourceDone)&&(symDone)&&(wasmDone)&&(watDone)) {
+        if (symDone) {
             setTimeout(() => {
                 process.exit(0);
             }, 300);
         }
     }
+
+}
+
+run().then(()=> {
+    process.exit(0);
 }, (err) => {
 //    console.log(err);
     console.log(err.stack);
@@ -172,5 +156,3 @@ compiler(fullFileName, options).then( () => {
 });
 
 
-
-

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "circom",
-  "version": "0.5.14",
+  "version": "0.5.18",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -48,9 +48,9 @@
       "dev": true
     },
     "acorn": {
-      "version": "7.3.1",
+      "version": "7.4.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.3.1.tgz",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
-      "integrity": "sha512-tLc0wSnatxAQHVHUapaHdz72pi9KUyHjq5KyHjGg9Y8Ifdc79pTh2XvI6I1/chZbnM7QtNKzh66ooDogPZSleA==",
+      "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
       "dev": true
     },
     "acorn-jsx": {
@@ -60,9 +60,9 @@
       "dev": true
     },
     "ajv": {
-      "version": "6.12.2",
+      "version": "6.12.3",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz",
-      "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==",
+      "integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==",
       "dev": true,
       "requires": {
         "fast-deep-equal": "^3.1.1",
@@ -240,9 +240,9 @@
       }
     },
     "cli-width": {
-      "version": "2.2.1",
+      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz",
-      "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==",
+      "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==",
       "dev": true
     },
     "cliui": {
@@ -507,9 +507,9 @@
       },
       "dependencies": {
         "estraverse": {
-          "version": "5.1.0",
+          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
-          "integrity": "sha512-FyohXK+R0vE+y1nHLoBM7ZTyqRpqAlhdZHCWIWEviFLiGB8b04H6bQs8G+XTthacvT8VuwvteiP7RJSxMs8UEw==",
+          "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
           "dev": true
         }
       }
@@ -576,9 +576,9 @@
       "dev": true
     },
     "fastfile": {
-      "version": "0.0.7",
+      "version": "0.0.12",
-      "resolved": "https://registry.npmjs.org/fastfile/-/fastfile-0.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/fastfile/-/fastfile-0.0.12.tgz",
-      "integrity": "sha512-Zk7sdqsV6DsN/rhjULDfCCowPiMDsziTMFicdkrKN80yybr/6YFf9H91ELXN85dVEf6EYkVR5EHkZNc0dMqZKA=="
+      "integrity": "sha512-0EZo2y5eW8X0oiDDRvcnufjVxlM96CQL5hvmRQtbRABWlCkH73IHwkzl0qOSdxtchaMr+0TSB7GVqaVEixRr1Q=="
     },
     "ffiasm": {
       "version": "0.0.2",
@@ -780,21 +780,21 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "inquirer": {
-      "version": "7.2.0",
+      "version": "7.3.3",
-      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.3.3.tgz",
-      "integrity": "sha512-E0c4rPwr9ByePfNlTIB8z51kK1s2n6jrHuJeEHENl/sbq2G/S1auvibgEwNR4uSyiU+PiYHqSwsgGiXjG8p5ZQ==",
+      "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==",
       "dev": true,
       "requires": {
         "ansi-escapes": "^4.2.1",
-        "chalk": "^3.0.0",
+        "chalk": "^4.1.0",
         "cli-cursor": "^3.1.0",
-        "cli-width": "^2.0.0",
+        "cli-width": "^3.0.0",
         "external-editor": "^3.0.3",
         "figures": "^3.0.0",
-        "lodash": "^4.17.15",
+        "lodash": "^4.17.19",
         "mute-stream": "0.0.8",
         "run-async": "^2.4.0",
-        "rxjs": "^6.5.3",
+        "rxjs": "^6.6.0",
         "string-width": "^4.1.0",
         "strip-ansi": "^6.0.0",
         "through": "^2.3.6"
@@ -811,9 +811,9 @@
           }
         },
         "chalk": {
-          "version": "3.0.0",
+          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
-          "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
+          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
           "dev": true,
           "requires": {
             "ansi-styles": "^4.1.0",
@@ -986,9 +986,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.15",
+      "version": "4.17.19",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz",
-      "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+      "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==",
       "dev": true
     },
     "mimic-fn": {
@@ -1063,9 +1063,9 @@
       }
     },
     "onetime": {
-      "version": "5.1.0",
+      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
-      "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==",
+      "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==",
       "dev": true,
       "requires": {
         "mimic-fn": "^2.1.0"
@@ -1167,6 +1167,13 @@
       "requires": {
         "fastfile": "0.0.7",
         "ffjavascript": "0.2.4"
+      },
+      "dependencies": {
+        "fastfile": {
+          "version": "0.0.7",
+          "resolved": "https://registry.npmjs.org/fastfile/-/fastfile-0.0.7.tgz",
+          "integrity": "sha512-Zk7sdqsV6DsN/rhjULDfCCowPiMDsziTMFicdkrKN80yybr/6YFf9H91ELXN85dVEf6EYkVR5EHkZNc0dMqZKA=="
+        }
       }
     },
     "regexpp": {
@@ -1216,9 +1223,9 @@
       "dev": true
     },
     "rxjs": {
-      "version": "6.6.0",
+      "version": "6.6.2",
-      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.0.tgz",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.2.tgz",
-      "integrity": "sha512-3HMA8z/Oz61DUHe+SdOiQyzIf4tOx5oQHmMir7IZEu6TMqCLHT4LRcmNaUS0NwOz8VLvmmBduMsoaUvMaIiqzg==",
+      "integrity": "sha512-BHdBMVoWC2sL26w//BCu3YzKT4s2jip/WhwsGEDmeKYBhKDZeYezVUnHatYB7L85v5xs0BAQmg6BEYJEKxBabg==",
       "dev": true,
       "requires": {
         "tslib": "^1.9.0"
@@ -1316,9 +1323,9 @@
       }
     },
     "strip-json-comments": {
-      "version": "3.1.0",
+      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
-      "integrity": "sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w==",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
       "dev": true
     },
     "supports-color": {
@@ -1552,9 +1559,9 @@
       "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w=="
     },
     "yargs": {
-      "version": "15.3.1",
+      "version": "15.4.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.3.1.tgz",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz",
-      "integrity": "sha512-92O1HWEjw27sBfgmXiixJWT5hRBp2eobqXicLtPBIDBhYB+1HpwZlXmbW2luivBJHBzki+7VyCLRtAkScbTBQA==",
+      "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==",
       "requires": {
         "cliui": "^6.0.0",
         "decamelize": "^1.2.0",
@@ -1566,7 +1573,7 @@
         "string-width": "^4.2.0",
         "which-module": "^2.0.0",
         "y18n": "^4.0.0",
-        "yargs-parser": "^18.1.1"
+        "yargs-parser": "^18.1.2"
       }
     },
     "yargs-parser": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "circom",
-  "version": "0.5.14",
+  "version": "0.5.18",
   "description": "Language to generate logic circuits",
   "main": "index.js",
   "directories": {
@@ -31,7 +31,7 @@
   "dependencies": {
     "chai": "^4.2.0",
     "circom_runtime": "0.0.6",
-    "fastfile": "0.0.7",
+    "fastfile": "0.0.12",
     "ffiasm": "0.0.2",
     "ffjavascript": "0.2.4",
     "ffwasm": "0.0.7",

ports/c/builder.js

@@ -1,8 +1,8 @@
-const streamFromMultiArray = require("../../src/streamfromarray_txt.js");
 const utils = require("../../src/utils");
 const assert = require("assert");
 const Scalar = require("ffjavascript").Scalar;
 const F1Field = require("ffjavascript").F1Field;
+const BigArray = require("../../src/bigarray");
 
 function ref2src(c) {
     if ((c[0] == "R")||(c[0] == "RI")) {
@@ -341,16 +341,17 @@ class FunctionBuilderC {
 }
 
 class BuilderC {
-    constructor(p) {
+    constructor(p, verbose) {
         this.F = new F1Field(p);
 
         this.hashMaps={};
-        this.componentEntriesTables={};
+        this.componentEntriesTables=new BigArray();
         this.sizes ={};
         this.constants = [];
         this.functions = [];
-        this.components = [];
+        this.components = new BigArray();
         this.usedConstants = {};
+        this.verbose = verbose;
     }
 
     setHeader(header) {
@@ -363,7 +364,10 @@ class BuilderC {
     }
 
     addComponentEntriesTable(name, cet) {
-        this.componentEntriesTables[name] = cet;
+        this.componentEntriesTables.push({
+            name: name,
+            cet: cet
+        });
     }
 
     addSizes(name, accSizes) {
@@ -444,8 +448,10 @@ class BuilderC {
 
     _buildComponentEntriesTables(code) {
         code.push("// Component Entry tables");
-        for (let cetName in this.componentEntriesTables) {
+        for (let i=0; i< this.componentEntriesTables.length; i++) {
-            const cet = this.componentEntriesTables[cetName];
+            if ((this.verbose)&&(i%100000 ==0)) console.log(`_buildComponentEntriesTables ${i}/${this.componentEntriesTables.length}`);
+            const cetName = this.componentEntriesTables[i].name;
+            const cet = this.componentEntriesTables[i].cet;
 
             code.push(`Circom_ComponentEntry ${cetName}[${cet.length}] = {`);
             for (let j=0; j<cet.length; j++) {
@@ -477,6 +483,7 @@ class BuilderC {
         code.push("// Constants");
         code.push(`FrElement _constants[${self.constants.length}] = {`);
         for (let i=0; i<self.constants.length; i++) {
+            if ((this.verbose)&&(i%1000000 ==0)) console.log(`_buildConstants ${i}/${this.constants.length}`);
             code.push((i>0 ? "," : " ") + "{" + number2Code(self.constants[i]) + "}");
         }
         code.push("};");
@@ -554,6 +561,7 @@ class BuilderC {
         code.push("// Components");
         code.push(`Circom_Component _components[${this.components.length}] = {`);
         for (let i=0; i<this.components.length; i++) {
+            if ((this.verbose)&&(i%1000000 ==0)) console.log(`_buildComponents ${i}/${this.components.length}`);
             const c = this.components[i];
             const sep = i>0 ? "    ," : "     ";
             code.push(`${sep}{${c.hashMapName}, ${c.entryTableName}, ${c.functionName}, ${c.nInSignals}, ${c.newThread}}`);
@@ -566,6 +574,7 @@ class BuilderC {
         code.push(`u32 _mapIsInput[${this.mapIsInput.length}] = {`);
         let line = "";
         for (let i=0; i<this.mapIsInput.length; i++) {
+            if ((this.verbose)&&(i%1000000 ==0)) console.log(`_buildMapIsInput ${i}/${this.mapIsInput.length}`);
             line += i>0 ? ", " : "  ";
             line += toHex(this.mapIsInput[i]);
             if (((i+1) % 64)==0) {
@@ -589,6 +598,7 @@ class BuilderC {
         code.push(`int _wit2sig[${this.wit2sig.length}] = {`);
         let line = "";
         for (let i=0; i<this.wit2sig.length; i++) {
+            if ((this.verbose)&&(i%1000000 ==0)) console.log(`_buildWit2Sig ${i}/${this.wit2sig.length}`);
             line += i>0 ? "," : " ";
             line += this.wit2sig[i];
             if (((i+1) % 64) == 0) {
@@ -620,8 +630,9 @@ class BuilderC {
     }
 
 
-    build() {
+    async build(fd) {
-        const code=[];
+        const encoder = new TextEncoder("utf-8");
+        const code=new BigArray();
         this._buildHeader(code);
         this._buildSizes(code);
         this._buildConstants(code);
@@ -632,7 +643,17 @@ class BuilderC {
         this._buildMapIsInput(code);
         this._buildWit2Sig(code);
         this._buildCircuitVar(code);
-        return streamFromMultiArray(code);
+        await writeCode(code);
+
+        async function writeCode(c) {
+            if (c.push) {
+                for (let i=0; i<c.length; i++) {
+                    await writeCode(c[i]);
+                }
+            } else if (typeof c === "string") {
+                await fd.write(encoder.encode(c + "\n"));
+            }
+        }
     }
 }
 

ports/c/tester.js

@@ -13,6 +13,7 @@ const utils = require("../../src/utils");
 const loadR1cs = require("r1csfile").load;
 const ZqField = require("ffjavascript").ZqField;
 const buildZqField = require("ffiasm").buildZqField;
+const fastFile = require("fastfile");
 
 const {stringifyBigInts, unstringifyBigInts } = require("ffjavascript").utils;
 
@@ -29,13 +30,15 @@ async function  c_tester(circomFile, _options) {
     const baseName = path.basename(circomFile, ".circom");
     const options = Object.assign({}, _options);
 
-    options.cSourceWriteStream = fs.createWriteStream(path.join(dir.path, baseName + ".cpp"));
+    options.cSourceFile = await fastFile.createOverride(path.join(dir.path, baseName + ".cpp"));
     options.symWriteStream = fs.createWriteStream(path.join(dir.path, baseName + ".sym"));
     options.r1csFileName = path.join(dir.path, baseName + ".r1cs");
 
     options.p = options.p || Scalar.fromString("21888242871839275222246405745257275088548364400416034343698204186575808495617");
     await compiler(circomFile, options);
 
+    await options.cSourceFile.close();
+
     const source = await buildZqField(options.p, "Fr");
 
     // console.log(dir.path);

ports/wasm/builder.js

@@ -1,5 +1,3 @@
-const streamFromArrayTxt = require("../../src/streamfromarray_txt");
-const streamFromArrayBin = require("../../src/streamfromarray_bin");
 const assert = require("assert");
 const ModuleBuilder = require("wasmbuilder").ModuleBuilder;
 const ModuleBuilderWat = require("wasmbuilder").ModuleBuilderWat;
@@ -990,7 +988,8 @@ class BuilderWasm {
     }
 
 
-    build(outType) {
+    async build(fd, outType) {
+        const encoder = new TextEncoder("utf-8");
         let module;
         if (outType == "wasm") {
             module=new ModuleBuilder();
@@ -1017,12 +1016,25 @@ class BuilderWasm {
 
         module.setMemory(2000);
         if (outType == "wasm") {
-            return streamFromArrayBin(module.build());
+            const bytes = module.build();
+            const bytesArr = new Uint8Array(bytes);
+            await fd.write(bytesArr);
         } else if (outType == "wat") {
-            return streamFromArrayTxt(module.build());
+            const code = module.build();
+            await writeCode(code);
         } else {
             assert(false);
         }
+
+        async function writeCode(c) {
+            if (c.push) {
+                for (let i=0; i<c.length; i++) {
+                    await writeCode(c[i]);
+                }
+            } else if (typeof c === "string") {
+                await fd.write(encoder.encode(c + "\n"));
+            }
+        }
     }
 }
 

ports/wasm/tester.js

@@ -9,6 +9,7 @@ const compiler = require("../../src/compiler");
 const utils = require("../../src/utils");
 const loadR1cs = require("r1csfile").load;
 const ZqField = require("ffjavascript").ZqField;
+const fastFile = require("fastfile");
 
 const WitnessCalculatorBuilder = require("circom_runtime").WitnessCalculatorBuilder;
 
@@ -24,16 +25,14 @@ async function  wasm_tester(circomFile, _options) {
     const baseName = path.basename(circomFile, ".circom");
     const options = Object.assign({}, _options);
 
-    options.wasmWriteStream = fs.createWriteStream(path.join(dir.path, baseName + ".wasm"));
+    options.wasmFile = await fastFile.createOverride(path.join(dir.path, baseName + ".wasm"));
+
     options.symWriteStream = fs.createWriteStream(path.join(dir.path, baseName + ".sym"));
     options.r1csFileName = path.join(dir.path, baseName + ".r1cs");
 
-    const promisesArr = [];
-    promisesArr.push(new Promise(fulfill => options.wasmWriteStream.on("finish", fulfill)));
-
     await compiler(circomFile, options);
 
-    await Promise.all(promisesArr);
+    await options.wasmFile.close();
 
     const wasm = await fs.promises.readFile(path.join(dir.path, baseName + ".wasm"));
 

src/bigarray.js

@@ -26,8 +26,10 @@ class _BigArray {
         }
         return this;
     }
-    push (element) {
+    push () {
-        this.setElement (this.length, element);
+        for (let i=0; i<arguments.length; i++) {
+            this.setElement (this.length, arguments[i]);
+        }
     }
     getElement(idx) {
         idx = parseInt(idx);
@@ -46,6 +48,19 @@ class _BigArray {
         if (idx >= this.length) this.length = idx+1;
         return true;
     }
+    getKeys() {
+        const newA = new BigArray();
+        for (let i=0; i<this.arr.length; i++) {
+            if (this.arr[i]) {
+                for (let j=0; j<this.arr[i].length; j++) {
+                    if (typeof this.arr[i][j] !== "undefined") {
+                        newA.push(i*SUBARRAY_SIZE+j);
+                    }
+                }
+            }
+        }
+        return newA;
+    }
 }
 
 class BigArray {

src/build.js

@@ -21,6 +21,7 @@ const assert = require("assert");
 const utils = require("./utils");
 const gen = require("./gencode").gen;
 const createRefs = require("./gencode").createRefs;
+const BigArray = require("./bigarray");
 
 module.exports =  build;
 
@@ -37,28 +38,33 @@ function build(ctx) {
     ctx.addConstant(ctx.F.zero);
     ctx.addConstant(ctx.F.one);
 
+    if (ctx.verbose) console.log("buildHeader...");
     buildHeader(ctx);
+    if (ctx.verbose) console.log("buildEntryTables...");
     buildEntryTables(ctx);
     ctx.globalNames = ctx.uniqueNames;
 
+    if (ctx.verbose) console.log("buildCode...");
     buildCode(ctx);
 
+    if (ctx.verbose) console.log("buildComponentsArray...");
     buildComponentsArray(ctx);
 
+    if (ctx.verbose) console.log("buildMapIsInput...");
     buildMapIsInput(ctx);
+
+    if (ctx.verbose) console.log("buildWit2Sig...");
     buildWit2Sig(ctx);
 
 }
 
 function buildEntryTables(ctx) {
 
-    const codes_hashMaps = [];
-    const codes_componentEntries = [];
     const definedHashMaps = {};
     for (let i=0; i<ctx.components.length; i++) {
+        if (ctx.verbose && (i%100000 ==0)) console.log(`buildEntryTables component: ${i}/${ctx.components.length}`);
         const {htName, htMap} = addHashTable(i);
 
-        let code = "";
         const componentEntriesTableName = ctx.getUniqueName("_entryTable" + ctx.components[i].template);
 
         const componentEntriesTable = [];
@@ -74,32 +80,12 @@ function buildEntryTables(ctx) {
 
         ctx.builder.addComponentEntriesTable(componentEntriesTableName, componentEntriesTable);
 
-
-        code += `Circom_ComponentEntry ${componentEntriesTableName}[${htMap.length}] = {\n`;
-        for (let j=0; j<htMap.length; j++) {
-            const entry = ctx.components[i].names.o[htMap[j]];
-            code += j>0 ? "    ," : "     ";
-            const sizeName = ctx.addSizes(entry.sizes);
-
-            const ty = entry.type == "S" ? "_typeSignal" : "_typeComponent";
-            code += `{${entry.offset},${sizeName}, ${ty}}\n`;
-        }
-        code += "};\n";
-        codes_componentEntries.push(code);
-
         ctx.components[i].htName = htName;
         ctx.components[i].etName = componentEntriesTableName;
     }
 
 
-    return [
+    return;
-        "// HashMaps\n" ,
-        codes_hashMaps , "\n" ,
-        "\n" ,
-        "// Component Entries\n" ,
-        codes_componentEntries , "\n" ,
-        "\n"
-    ];
 
     function addHashTable(cIdx) {
         const keys = Object.keys(ctx.components[cIdx].names.o);
@@ -131,6 +117,7 @@ function buildCode(ctx) {
 
     const fnComponents = [];
     for (let i=0; i<ctx.components.length; i++) {
+        if (ctx.verbose && (i%100000 ==0)) console.log(`buildCode component: ${i}/${ctx.components.length}`);
         const {h, instanceDef} = hashComponentCall(ctx, i);
         const fName = ctx.components[i].template+"_"+h;
         if (!fDefined[fName]) {
@@ -180,6 +167,7 @@ function buildCode(ctx) {
 
 function buildComponentsArray(ctx) {
     for (let i=0; i< ctx.components.length; i++) {
+        if (ctx.verbose && (i%1000000 ==0)) console.log(`buildComponentsArray component: ${i}/${ctx.components.length}`);
         let newThread;
         if (ctx.newThreadTemplates) {
             if (ctx.newThreadTemplates.test(ctx.components[i].template)) {
@@ -217,6 +205,7 @@ function buildMapIsInput(ctx) {
     let map = [];
     let acc = 0;
     for (i=0; i<ctx.signals.length; i++) {
+        if (ctx.verbose && (i%1000000 ==0)) console.log(`buildMapIsInput signal: ${i}/${ctx.signals.length}`);
         if (ctx.signals[i].o & ctx.IN) {
             acc = acc | (1 << (i%32) );
         }
@@ -241,8 +230,9 @@ function buildWit2Sig(ctx) {
         ctx.totals[ctx.stPUBINPUT] +
         ctx.totals[ctx.stPRVINPUT] +
         ctx.totals[ctx.stINTERNAL];
-    const arr = Array(NVars);
+    const arr = new BigArray(NVars);
     for (let i=0; i<ctx.signals.length; i++) {
+        if (ctx.verbose && (i%1000000 ==0)) console.log(`buildWit2Sig signal: ${i}/${ctx.signals.length}`);
         const outIdx = ctx.signals[i].id;
         if (ctx.signals[i].e>=0) continue;     // If has an alias, continue..
         assert(typeof outIdx  != "undefined", `Signal ${i} does not have index`);

src/compiler.js

@@ -28,8 +28,30 @@ const utils = require("./utils");
 const buildR1cs = require("./r1csfile").buildR1cs;
 const BigArray = require("./bigarray");
 const buildSyms = require("./buildsyms");
+const {performance} = require("perf_hooks");
 
 module.exports = compile;
+const measures = {};
+
+function ms2String(v) {
+    v = Math.floor(v);
+    const ms = v % 1000;
+    v = Math.floor(v/1000);
+    const secs = v % 60;
+    v = Math.floor(v/60);
+    const mins = v % 60;
+    v = Math.floor(v/60);
+    const hours = v % 24;
+    const days = Math.floor(v/24);
+    let S = "";
+    if (days) S = S + days + "D ";
+    if ((S!="")||(hours)) S = S + hours.toString().padStart(2, "0") + ":";
+    if ((S!="")||(mins)) S = S + mins.toString().padStart(2, "0") + ":";
+    if ((S!="")||(secs)) S = S + secs.toString().padStart(2, "0");
+    S+=".";
+    S = S + ms.toString().padStart(3, "0");
+    return S;
+}
 
 async function compile(srcFile, options) {
     options.prime = options.prime || Scalar.fromString("21888242871839275222246405745257275088548364400416034343698204186575808495617");
@@ -44,9 +66,12 @@ async function compile(srcFile, options) {
     ctx.mainComponent = options.mainComponent || "main";
     ctx.newThreadTemplates = options.newThreadTemplates;
 
+    measures.constructionPhase = -performance.now();
     constructionPhase(ctx, srcFile);
+    measures.constructionPhase += performance.now();
 
     if (ctx.verbose) console.log("NConstraints Before: "+ctx.constraints.length);
+    if (ctx.verbose) console.log("NSignals Before: "+ctx.signals.length);
 
     if (ctx.error) {
         throw(ctx.error);
@@ -57,10 +82,15 @@ async function compile(srcFile, options) {
     }
 
     if (ctx.verbose) console.log("Classify Signals");
+    measures.classifySignals = -performance.now();
     classifySignals(ctx);
+    measures.classifySignals += performance.now();
 
     if (ctx.verbose) console.log("Reduce Constants");
+    measures.reduceConstants = -performance.now();
     reduceConstants(ctx);
+    measures.reduceConstants += performance.now();
+
     if (options.reduceConstraints) {
 
         if (ctx.verbose) console.log("Reduce Constraints");
@@ -73,60 +103,76 @@ async function compile(srcFile, options) {
             reduceConstrains(ctx);
         }
 */
+        measures.reduceConstraints = -performance.now();
+        await reduceConstrains(ctx);
+        measures.reduceConstraints += performance.now();
 
-        reduceConstrains(ctx);
     }
 
     if (ctx.verbose) console.log("NConstraints After: "+ctx.constraints.length);
 
+    measures.generateWitnessNames = -performance.now();
     generateWitnessNames(ctx);
+    measures.generateWitnessNames += performance.now();
 
     if (ctx.error) {
         throw(ctx.error);
     }
 
-    if (options.cSourceWriteStream) {
+    if (options.r1csFileName) {
-        if (ctx.verbose) console.log("Generating c...");
+        measures.generateR1cs = -performance.now();
-        ctx.builder = new BuilderC(options.prime);
+        await buildR1cs(ctx, options.r1csFileName);
-        build(ctx);
+        measures.generateR1cs += performance.now();
-        const rdStream = ctx.builder.build();
-        rdStream.pipe(options.cSourceWriteStream);
-
-        // await new Promise(fulfill => options.cSourceWriteStream.on("finish", fulfill));
     }
 
-    if ((options.wasmWriteStream)||(options.watWriteStream)) {
+    if (ctx.error) throw(ctx.error);
+
+    delete ctx.constraints;  // Liberate memory.
+
+    if (options.cSourceFile) {
+        if (ctx.verbose) console.log("Generating c...");
+        measures.generateC = -performance.now();
+        ctx.builder = new BuilderC(options.prime, ctx.verbose);
+        build(ctx);
+        await ctx.builder.build(options.cSourceFile);
+        measures.generateC += performance.now();
+    }
+
+    if (ctx.error) throw(ctx.error);
+
+    if ((options.wasmFile)||(options.watFile)) {
         if (ctx.verbose) console.log("Generating wasm...");
+        measures.generateWasm = -performance.now();
         ctx.builder = new BuilderWasm(options.prime);
         build(ctx);
-        if (options.wasmWriteStream) {
+        if (options.wasmFile) {
-            const rdStream = ctx.builder.build("wasm");
+            await ctx.builder.build(options.wasmFile, "wasm");
-            rdStream.pipe(options.wasmWriteStream);
         }
-        if (options.watWriteStream) {
+        if (options.watFile) {
-            const rdStream = ctx.builder.build("wat");
+            await ctx.builder.build(options.watFile, "wat");
-            rdStream.pipe(options.watWriteStream);
         }
-
+        measures.generateWasm += performance.now();
-        // await new Promise(fulfill => options.wasmWriteStream.on("finish", fulfill));
     }
 
     // const mainCode = gen(ctx,ast);
     if (ctx.error) throw(ctx.error);
 
-    if (options.r1csFileName) {
-        await buildR1cs(ctx, options.r1csFileName);
-    }
-
     if (options.symWriteStream) {
+        measures.generateSyms = -performance.now();
         const rdStream = buildSyms(ctx);
         rdStream.pipe(options.symWriteStream);
+        measures.generateSyms += performance.now();
 
-        // await new Promise(fulfill => options.symWriteStream.on("finish", fulfill));
+        await new Promise(fulfill => options.symWriteStream.on("finish", fulfill));
     }
 
 //    const def = buildCircuitDef(ctx, mainCode);
 
+    if (ctx.verbose) {
+        for (let [mStr, mValue] of Object.entries(measures)) {
+            console.log(mStr + ": " + ms2String(mValue));
+        }
+    }
 }
 
 
@@ -152,6 +198,7 @@ function classifySignals(ctx) {
 
     // First classify the signals
     for (let s=0; s<ctx.signals.length; s++) {
+        if ((ctx.verbose)&&(s%100000 == 0)) console.log(`classify signals: ${s}/${ctx.signals.length}`);
         const signal = ctx.signals[s];
         let tAll = ctx.stINTERNAL;
         let lSignal = signal;
@@ -255,17 +302,19 @@ function reduceConstants(ctx) {
     ctx.constraints = newConstraints;
 }
 
-function reduceConstrains(ctx) {
+async function reduceConstrains(ctx) {
-    const sig2constraint = {};
+    const sig2constraint = new BigArray();
-    let removedSignals = {};
+    let removedSignals = new BigArray();
     let nRemoved;
     let lIdx;
 
 
-    let possibleConstraints = new Array(ctx.constraints.length);
+    let possibleConstraints = new BigArray(ctx.constraints.length);
     let nextPossibleConstraints;
     for (let i=0; i<ctx.constraints.length; i++) {
-        const insertedSig = {};
+        if ((ctx.verbose)&&(i%100000 == 0)) console.log(`indexing constraints: ${i}/${ctx.constraints.length}`);
+
+        const insertedSig = { 0: true};  // Do not insert one.
         const c = ctx.constraints[i];
         for (let s in c.a.coefs) {
             if (!insertedSig[s]) {
@@ -292,12 +341,13 @@ function reduceConstrains(ctx) {
     }
 
     while (possibleConstraints.length >0) {
-        nextPossibleConstraints = {};
+        nextPossibleConstraints = new BigArray();
-        removedSignals = {};
+        removedSignals = new BigArray();
         nRemoved = 0;
-        lIdx = {};
+        lIdx = new BigArray();
         for (let i=0;i<possibleConstraints.length;i++) {
             if ((ctx.verbose)&&(i%10000 == 0)) {
+                await Promise.resolve();
                 console.log(`reducing constraints: ${i}/${possibleConstraints.length}   reduced: ${nRemoved}`);
             }
             const c = ctx.constraints[possibleConstraints[i]];
@@ -348,9 +398,13 @@ function reduceConstrains(ctx) {
             }
         }
 
-        nextPossibleConstraints = Object.keys(nextPossibleConstraints);
+        nextPossibleConstraints = nextPossibleConstraints.getKeys();
 
         for (let i=0; i<nextPossibleConstraints.length;i++) {
+            if ((ctx.verbose)&&(i%10000 == 0)) {
+                await Promise.resolve();
+                console.log(`substituting constraints: ${i}/${nextPossibleConstraints.length}`);
+            }
             const c = ctx.constraints[nextPossibleConstraints[i]];
             if (c) {
                 const nc = {
@@ -366,7 +420,11 @@ function reduceConstrains(ctx) {
             }
         }
 
-        for (let s in removedSignals) {
+        const removedSignalsList = removedSignals.getKeys;
+
+        for (let i=0; i<removedSignalsList.length; i++) {
+            if ((ctx.verbose    )&&(i%100000 == 0)) console.log(`removing signals: ${i}/${removedSignalsList.length}`);
+            const s = removedSignalsList[i];
 
             let lSignal = ctx.signals[s];
             while (lSignal.e>=0) {
@@ -377,11 +435,11 @@ function reduceConstrains(ctx) {
         }
 
         possibleConstraints = nextPossibleConstraints;
-
     }
 
     let o=0;
     for (let i=0; i<ctx.constraints.length;i++) {
+        if ((ctx.verbose)&&(i%100000 == 0)) console.log(`reordering constraints: ${i}/${ctx.constraints.length}`);
         if (ctx.constraints[i]) {
             if (!ctx.lc.isZero(ctx.constraints[i])) {
                 ctx.constraints[o] = ctx.constraints[i];

src/construction_phase.js

@@ -574,7 +574,7 @@ function execFunctionCall(ctx, ast) {
 
     const fnc = ctx.functions[ast.name];
 
-    if (!fnc) return ctx.throwError("Function not defined");
+    if (!fnc) return ctx.throwError(ast, "Function not defined");
 
     const paramValues = [];
     for (let i=0; i< ast.params.length; i++) {

src/lcalgebra.js

@@ -123,6 +123,7 @@ class LCAlgebra {
     }
 
     _signal2lc(a) {
+        const self = this;
         if (a.t == "S") {
             const lc = {
                 t: "LC",

src/r1csfile.js

@@ -1,14 +1,18 @@
 
 const fastFile = require("fastfile");
 const assert = require("assert");
+const BigArray = require("./bigarray");
 
 module.exports.buildR1cs = buildR1cs;
 
+
 async function buildR1cs(ctx, fileName) {
 
     const fd = await fastFile.createOverride(fileName);
 
 
+    const buffBigInt = new Uint8Array(ctx.F.n8);
+
     const type = "r1cs";
     const buff = new Uint8Array(4);
     for (let i=0; i<4; i++) buff[i] = type.charCodeAt(i);
@@ -67,7 +71,7 @@ async function buildR1cs(ctx, fileName) {
     await fd.writeULE64(0); // Temporally set to 0 length
 
 
-    const arr = new Array(NWires);
+    const arr = new BigArray(NWires);
     for (let i=0; i<ctx.signals.length; i++) {
         const outIdx = ctx.signals[i].id;
         if (ctx.signals[i].e>=0) continue;     // If has an alias, continue..
@@ -91,33 +95,51 @@ async function buildR1cs(ctx, fileName) {
 
     await fd.close();
 
-    async function writeConstraint(c) {
+    function writeConstraint(c) {
-        await writeLC(c.a);
+        const n8 = ctx.F.n8;
-        await writeLC(c.b);
+        const idxA = Object.keys(c.a.coefs);
-        await writeLC(ctx.lc.neg(c.c));
+        const idxB = Object.keys(c.b.coefs);
-    }
+        const idxC = Object.keys(c.c.coefs);
-
+        const buff = new Uint8Array((idxA.length+idxB.length+idxC.length)*(n8+4) + 12);
-    async function writeLC(lc) {
+        const buffV = new DataView(buff.buffer);
-        const idxs = Object.keys(lc.coefs);
+        let o=0;
-        await fd.writeULE32(idxs.length);
-        for (let s in lc.coefs) {
-            let lSignal = ctx.signals[s];
 
+        buffV.setUint32(o, idxA.length, true); o+=4;
+        for (let i=0; i<idxA.length; i++) {
+            const coef = idxA[i];
+            let lSignal = ctx.signals[coef];
             while (lSignal.e >=0 ) lSignal = ctx.signals[lSignal.e];
-
+            buffV.setUint32(o, lSignal.id, true); o+=4;
-            await fd.writeULE32(lSignal.id);
+            ctx.F.toRprLE(buff, o, c.a.coefs[coef]); o+=n8;
-            await writeBigInt(lc.coefs[s]);
         }
+
+        buffV.setUint32(o, idxB.length, true); o+=4;
+        for (let i=0; i<idxB.length; i++) {
+            const coef = idxB[i];
+            let lSignal = ctx.signals[coef];
+            while (lSignal.e >=0 ) lSignal = ctx.signals[lSignal.e];
+            buffV.setUint32(o, lSignal.id, true); o+=4;
+
+            ctx.F.toRprLE(buff, o, c.b.coefs[coef]); o+=n8;
+        }
+
+        buffV.setUint32(o, idxC.length, true); o+=4;
+        for (let i=0; i<idxC.length; i++) {
+            const coef = idxC[i];
+            let lSignal = ctx.signals[coef];
+            while (lSignal.e >=0 ) lSignal = ctx.signals[lSignal.e];
+            buffV.setUint32(o, lSignal.id, true); o+=4;
+            ctx.F.toRprLE(buff, o, ctx.F.neg(c.c.coefs[coef])); o+=n8;
+        }
+
+        return fd.write(buff);
     }
 
     async function writeBigInt(n, pos) {
 
-        const s = n.toString(16);
+        ctx.F.toRprLE(buffBigInt, 0, n);
-        const b = Buffer.from(s.padStart(n8*2, "0"), "hex");
-        const buff = new Uint8Array(b.length);
-        for (let i=0; i<b.length; i++) buff[i] = b[b.length-1-i];
 
-        await fd.write(buff, pos);
+        await fd.write(buffBigInt, pos);
 
     }
 }

src/streamfromarray_txt.js

@@ -1,4 +1,3 @@
-
 const Readable = require("stream").Readable;
 
 module.exports = function streamFromArrayTxt(ma) {
@@ -22,7 +21,7 @@ module.exports = function streamFromArrayTxt(ma) {
 
 
     function getFirstIdx(ma) {
-        if (!Array.isArray(ma)) return [];
+        if (typeof ma.push !== "function" ) return [];
         return [0, ...getFirstIdx(ma[0])];
     }