diff --git a/noir-projects/noir-protocol-circuits/Nargo.template.toml b/noir-projects/noir-protocol-circuits/Nargo.template.toml index 10d69052ff87..bfd7bfe753e9 100644 --- a/noir-projects/noir-protocol-circuits/Nargo.template.toml +++ b/noir-projects/noir-protocol-circuits/Nargo.template.toml @@ -1,3 +1,7 @@ +# This is a template file. +# The actual Nargo.toml for this workspace is generated by `./scripts/generate_variants.js` to include +# the many autogenerated variants of the reset circuit. + [workspace] members = [ "crates/ts-types", diff --git a/noir-projects/noir-protocol-circuits/scripts/generate_variants.js b/noir-projects/noir-protocol-circuits/scripts/generate_variants.js index 2cd3b14fd6dc..07e87658e22f 100755 --- a/noir-projects/noir-protocol-circuits/scripts/generate_variants.js +++ b/noir-projects/noir-protocol-circuits/scripts/generate_variants.js @@ -1,13 +1,20 @@ +#!/usr/bin/env node + const TOML = require("@iarna/toml"); const fs = require("fs"); const path = require("path"); const config = require("../private_kernel_reset_config.json"); const root = path.join(__dirname, "../"); + +// The source circuit folders that serve as templates for variant generation const sourceFolder = "private-kernel-reset"; const sourceSimulatedFolder = "private-kernel-reset-simulated"; + +// Where generated variant circuits are written const autogeneratedCircuitsFolder = "crates/autogenerated"; +// Extract dimension names from config - order matters! Must match TypeScript's privateKernelResetDimensionNames const dimensionNames = Object.keys(config.dimensions); const aliases = { @@ -15,10 +22,12 @@ const aliases = { full: [64, 64, 64, 64, 64, 64, 64, 64, 64], }; +// Converts a dimensions array to a tag string, e.g., [32, 4, 32, ...] -> "32_4_32_..." function getResetTag(dimensions) { return dimensions.join("_"); } +// Validates that dimensions is an array of 9 numbers function areValidDimensions(dimensions) { return ( dimensions.length === dimensionNames.length && @@ -26,19 +35,31 @@ function areValidDimensions(dimensions) { ); } +// Checks if dimensions match the full/max configuration (no variant needed - use the original circuit) function isFullDimensions(dimensions) { return dimensions.every((v, i) => v === aliases.full[i]); } +/** + * Builds the list of all dimension combinations to generate. + * + * If allowedVariants is provided (via CLI args), parses those specific variants. + * Otherwise, generates the full list by combining: + * 1. Cartesian product of all "variants" values from config + * 2. "standalone" variants (single dimension set, others zero) + * 3. "specialCases" (predefined full configurations like all-4s, all-16s, etc.) + */ function getDimensionsList(allowedVariants) { if (allowedVariants.length) { - // allowedVariant can be a key in aliases, or a tag. + // CLI args provided - parse them as aliases or tag strings (e.g., "tiny" or "32_4_32_4_4_4_4_4_4") return allowedVariants - .map((name) => aliases[name] || allowedVariants.split("_").map((v) => +v)) + .map((name) => aliases[name] || name.split("_").map((v) => +v)) .filter(areValidDimensions); } - // Variants. + // No CLI args - generate all combinations from config + + // 1. Cartesian product of variant values across all dimensions const dimensionsList = dimensionNames.reduce( (acc, name) => acc.flatMap((combination) => @@ -47,7 +68,8 @@ function getDimensionsList(allowedVariants) { [[]] ); - // Standalone. + // 2. Standalone variants: only one dimension is set, rest are zero + // (useful for cases where only one type of operation needs a large capacity) dimensionNames.forEach((name, i) => { config.dimensions[name].standalone.forEach((val) => { const dimensions = Array(dimensionNames.length).fill(0); @@ -56,12 +78,14 @@ function getDimensionsList(allowedVariants) { }); }); - // Special cases. + // 3. Special cases: predefined combinations (e.g., all-4s, all-16s, all-32s, all-64s) config.specialCases.forEach((dimensions) => dimensionsList.push(dimensions)); return dimensionsList; } +// Writes private_kernel_reset_dimensions.json - a JSON array of all dimension combinations. +// This file is read by TypeScript at runtime to know which variants are available. function generateDimensions(dimensionsList) { const dimensionValuesStrings = dimensionsList.map( (dimensions) => ` [${dimensions.join(", ")}]` @@ -75,7 +99,17 @@ function generateDimensions(dimensionsList) { ); } +/** + * Updates a single dimension's global constants in the Noir source code. + * + * For each dimension, we update: + * - {NAME}_AMOUNT: the actual iteration count (can be 0) + * - {NAME}_HINTS_LEN: the hints array length (minimum 1 [WHY?]) + * + * Also removes the `constants::` import block since variants use literal values, not constants. + */ function updateConstants(code, name, value) { + // Replace {NAME}_AMOUNT global with the new value const amountName = `${name}_AMOUNT`; const regex = new RegExp( `^global\\s+${amountName}:\\su32\\s=\\s(.*);.*$`, @@ -84,10 +118,10 @@ function updateConstants(code, name, value) { if (!code.match(regex)) { throw new Error(`Could not find dimension ${name} in main.nr`); } - // Update value. code = code.replace(regex, `global ${amountName}: u32 = ${value};`); - // Update hints length if applies. + // Replace {NAME}_HINTS_LEN if it exists (not all dimensions have hints arrays) + // When AMOUNT is 0, HINTS_LEN must still be >= 1 [WHY?] const hintName = `${name}_HINTS_LEN`; const hintRegex = new RegExp( `^global\\s+${hintName}:\\su32\\s=\\s(.*);.*$`, @@ -98,7 +132,8 @@ function updateConstants(code, name, value) { code = code.replace(hintRegex, `global ${hintName}: u32 = ${hintLen};`); } - // Remove constants. + // Remove the constants:: import block (e.g., MAX_NOTE_HASH_READ_REQUESTS_PER_TX) + // since we've replaced the constant references with literal values return code.replace( /use dep::types::\{\s*constants::\{[^}]*?\},\s*([^}]*?)\};/, (_, rest) => { @@ -116,26 +151,38 @@ function updateConstants(code, name, value) { ); } +/** + * Generates all variant circuit folders for the given dimensions list. + * + * For each dimension combination (except full-size, which uses the original circuit), creates: + * - crates/autogenerated/{folder}-{tag}/src/main.nr (modified source with new constants) + * - crates/autogenerated/{folder}-{tag}/Nargo.toml (package manifest with adjusted paths) + * + * Also adds each variant to the workspace members list. + */ function generateCircuits(dimensionsList, nargoToml, isSimulated) { const originalFolder = !isSimulated ? sourceFolder : sourceSimulatedFolder; const originalCratePath = path.join(root, "crates", originalFolder); + + // Read the source crate's Nargo.toml to use as a template for variants const originalNargoToml = TOML.parse( fs.readFileSync(path.join(originalCratePath, "Nargo.toml"), "utf8") ); const originalName = originalNargoToml.package.name; for (const dimensions of dimensionsList) { - // The default private-kernel-reset(-simulated) has full dimensions. + // Skip full dimensions - those use the original circuit, not a variant if (isFullDimensions(dimensions)) { continue; } + // Clone the template Nargo.toml and modify it for this variant const variantNargoToml = structuredClone(originalNargoToml); - const tag = getResetTag(dimensions); variantNargoToml.package.name = `${originalName}_${tag}`; - for ([depName, depDescriptor] of Object.entries( + // Adjust dependency paths (variants are one level deeper: crates/autogenerated/variant/) + for (let [_, depDescriptor] of Object.entries( variantNargoToml.dependencies )) { if (depDescriptor.path) { @@ -143,11 +190,11 @@ function generateCircuits(dimensionsList, nargoToml, isSimulated) { } } + // Read the source main.nr and update all dimension constants let mainDotNoirCode = fs.readFileSync( path.join(originalCratePath, "src/main.nr"), "utf8" ); - for (let i = 0; i < dimensions.length; i++) { mainDotNoirCode = updateConstants( mainDotNoirCode, @@ -156,35 +203,47 @@ function generateCircuits(dimensionsList, nargoToml, isSimulated) { ); } + // Write the variant files const variantFolder = path.join( autogeneratedCircuitsFolder, `${originalFolder}-${tag}` ); - - fs.mkdirSync(path.join(variantFolder, "src"), { - recursive: true, - }); - + fs.mkdirSync(path.join(variantFolder, "src"), { recursive: true }); fs.writeFileSync(path.join(variantFolder, "src/main.nr"), mainDotNoirCode); - fs.writeFileSync( path.join(variantFolder, "Nargo.toml"), TOML.stringify(variantNargoToml) ); + // Add to workspace members so Nargo knows about this crate nargoToml.workspace.members.push(variantFolder); } } +/** + * Main entry point. Generates all reset circuit variants. + * + * Usage: + * node generate_variants.js # Generate all variants from config + * node generate_variants.js tiny # Generate only the "tiny" variant + * node generate_variants.js 32_4_32_... # Generate a specific variant by tag + */ function main() { const allowedVariants = process.argv.slice(2) || []; - const nargoToml = TOML.parse( - fs.readFileSync("./Nargo.template.toml", "utf8") + // Read the workspace template (Nargo.template.toml) and strip leading comments + const nargoTemplate = fs.readFileSync("./Nargo.template.toml", "utf8"); + const lines = nargoTemplate.split("\n"); + const firstNonCommentIndex = lines.findIndex( + (line) => !line.startsWith("#") && line.trim() !== "" ); + const nargoTemplateWithoutComments = lines.slice(firstNonCommentIndex).join("\n"); + const nargoToml = TOML.parse(nargoTemplateWithoutComments); + // Build the list of dimension combinations to generate const dimensionsList = getDimensionsList(allowedVariants); + // Clear and recreate the autogenerated circuits folder const autogeneratedCircuitsPath = path.join( root, autogeneratedCircuitsFolder @@ -194,16 +253,199 @@ function main() { } fs.mkdirSync(autogeneratedCircuitsPath); + // Generate all outputs: + // 1. private_kernel_reset_dimensions.json (list of all variants for TypeScript) generateDimensions(dimensionsList); - generateCircuits(dimensionsList, nargoToml, false); - generateCircuits(dimensionsList, nargoToml, true); + // 2. Variant circuits (both real and simulated versions) + generateCircuits(dimensionsList, nargoToml, false); // private-kernel-reset variants + generateCircuits(dimensionsList, nargoToml, true); // private-kernel-reset-simulated variants + // 3. Write the final workspace Nargo.toml with all variant crates included const content = [ - "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. It is generated by `noir-projects/noir-protocol-circuits/scripts/generate_variants.js`.\n", TOML.stringify(nargoToml), ].join("\n"); fs.writeFileSync("./Nargo.toml", content); } -main(); +// TODO: use an actual testing framework instead of this. Some of these tests do feel important. + +// Command: +// `node scripts/generate_variants.js --test` +// +// Run tests if --test flag is passed, otherwise run main +if (process.argv.includes("--test")) { + runTests(); +} else { + main(); +} + +function runTests() { + let passed = 0; + let failed = 0; + + function test(name, fn) { + try { + fn(); + console.log(`✓ ${name}`); + passed++; + } catch (e) { + console.error(`✗ ${name}`); + console.error(` ${e.message}`); + failed++; + } + } + + function assertEqual(actual, expected, msg) { + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error( + `${msg}\n Expected: ${JSON.stringify(expected)}\n Got: ${JSON.stringify(actual)}` + ); + } + } + + // Test: Dimension names match TypeScript definition + test("dimension names match TypeScript privateKernelResetDimensionNames", () => { + // This order must match privateKernelResetDimensionNames in + // yarn-project/stdlib/src/kernel/private_kernel_reset_dimensions.ts + const expectedDimensionNames = [ + "NOTE_HASH_PENDING_READ", + "NOTE_HASH_SETTLED_READ", + "NULLIFIER_PENDING_READ", + "NULLIFIER_SETTLED_READ", + "KEY_VALIDATION", + "TRANSIENT_DATA_SQUASHING", + "NOTE_HASH_SILOING", + "NULLIFIER_SILOING", + "PRIVATE_LOG_SILOING", + ]; + assertEqual( + dimensionNames, + expectedDimensionNames, + "Dimension names in config do not match expected order" + ); + }); + + // Test: areValidDimensions accepts valid dimensions + test("areValidDimensions accepts valid 9-element number array", () => { + assertEqual(areValidDimensions([1, 2, 3, 4, 5, 6, 7, 8, 9]), true, "Should accept valid dimensions"); + }); + + // Test: areValidDimensions rejects wrong length + test("areValidDimensions rejects wrong length", () => { + assertEqual(areValidDimensions([1, 2, 3]), false, "Should reject short array"); + assertEqual(areValidDimensions([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), false, "Should reject long array"); + }); + + // Test: areValidDimensions rejects non-numbers + test("areValidDimensions rejects non-numbers", () => { + assertEqual( + areValidDimensions(["1", "2", "3", "4", "5", "6", "7", "8", "9"]), + false, + "Should reject strings" + ); + }); + + // Test: getResetTag produces correct format + test("getResetTag produces underscore-separated tag", () => { + assertEqual(getResetTag([32, 4, 32, 4, 4, 4, 4, 4, 4]), "32_4_32_4_4_4_4_4_4", "Should join with underscores"); + }); + + // Test: getDimensionsList parses tag strings correctly + test("getDimensionsList parses tag strings", () => { + const result = getDimensionsList(["32_4_32_4_4_4_4_4_4"]); + assertEqual(result, [[32, 4, 32, 4, 4, 4, 4, 4, 4]], "Should parse tag into number array"); + }); + + // Test: getDimensionsList resolves aliases + test("getDimensionsList resolves aliases", () => { + const result = getDimensionsList(["tiny"]); + assertEqual(result, [[4, 4, 4, 4, 4, 4, 4, 4, 4]], "Should resolve 'tiny' alias"); + }); + + // Test: getDimensionsList filters invalid tags + test("getDimensionsList filters invalid tags", () => { + const result = getDimensionsList(["invalid_tag", "1_2_3"]); + assertEqual(result, [], "Should filter out invalid dimensions"); + }); + + // Test: isFullDimensions identifies full dimensions + test("isFullDimensions identifies full dimensions", () => { + assertEqual(isFullDimensions([64, 64, 64, 64, 64, 64, 64, 64, 64]), true, "Should identify full dimensions"); + assertEqual(isFullDimensions([32, 4, 32, 4, 4, 4, 4, 4, 4]), false, "Should reject non-full dimensions"); + }); + + // Test: updateConstants regex matches real template file + test("updateConstants regex matches all dimensions in template", () => { + const templatePath = path.join(root, "crates", sourceFolder, "src/main.nr"); + const templateCode = fs.readFileSync(templatePath, "utf8"); + + for (const name of dimensionNames) { + // This will throw if the regex doesn't match + try { + updateConstants(templateCode, name, 42); + } catch (e) { + throw new Error(`updateConstants failed for dimension ${name}: ${e.message}`); + } + } + }); + + // Test: updateConstants correctly replaces AMOUNT values + test("updateConstants replaces AMOUNT values correctly", () => { + const code = `global NOTE_HASH_PENDING_READ_AMOUNT: u32 = MAX_NOTE_HASH_READ_REQUESTS_PER_TX;`; + const result = updateConstants(code, "NOTE_HASH_PENDING_READ", 32); + if (!result.includes("global NOTE_HASH_PENDING_READ_AMOUNT: u32 = 32;")) { + throw new Error(`Expected AMOUNT to be replaced with 32, got: ${result}`); + } + }); + + // Test: updateConstants correctly replaces HINTS_LEN values + test("updateConstants replaces HINTS_LEN values correctly", () => { + const code = [ + `global NOTE_HASH_PENDING_READ_HINTS_LEN: u32 = MAX_NOTE_HASH_READ_REQUESTS_PER_TX;`, + `global NOTE_HASH_PENDING_READ_AMOUNT: u32 = MAX_NOTE_HASH_READ_REQUESTS_PER_TX;`, + ].join("\n"); + const result = updateConstants(code, "NOTE_HASH_PENDING_READ", 16); + if (!result.includes("global NOTE_HASH_PENDING_READ_HINTS_LEN: u32 = 16;")) { + throw new Error(`Expected HINTS_LEN to be replaced with 16, got: ${result}`); + } + }); + + // Test: updateConstants sets HINTS_LEN to 1 when AMOUNT is 0 (array cannot be empty) + test("updateConstants sets HINTS_LEN to 1 when AMOUNT is 0", () => { + const code = [ + `global NOTE_HASH_PENDING_READ_HINTS_LEN: u32 = MAX_NOTE_HASH_READ_REQUESTS_PER_TX;`, + `global NOTE_HASH_PENDING_READ_AMOUNT: u32 = MAX_NOTE_HASH_READ_REQUESTS_PER_TX;`, + ].join("\n"); + const result = updateConstants(code, "NOTE_HASH_PENDING_READ", 0); + if (!result.includes("global NOTE_HASH_PENDING_READ_HINTS_LEN: u32 = 1;")) { + throw new Error(`Expected HINTS_LEN to be 1 when AMOUNT is 0, got: ${result}`); + } + if (!result.includes("global NOTE_HASH_PENDING_READ_AMOUNT: u32 = 0;")) { + throw new Error(`Expected AMOUNT to be 0, got: ${result}`); + } + }); + + // Test: generated variant has correct constants + test("generated variant has correct constants for all dimensions", () => { + const templatePath = path.join(root, "crates", sourceFolder, "src/main.nr"); + let code = fs.readFileSync(templatePath, "utf8"); + + const testDimensions = [32, 4, 32, 4, 4, 4, 4, 4, 4]; + for (let i = 0; i < dimensionNames.length; i++) { + code = updateConstants(code, dimensionNames[i], testDimensions[i]); + } + + // Verify each dimension's AMOUNT is set correctly + for (let i = 0; i < dimensionNames.length; i++) { + const expectedPattern = `global ${dimensionNames[i]}_AMOUNT: u32 = ${testDimensions[i]};`; + if (!code.includes(expectedPattern)) { + throw new Error(`Expected to find "${expectedPattern}" in generated code`); + } + } + }); + + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed > 0 ? 1 : 0); +}