Add regula-wasi OPA test infrastructure #67
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,175 @@ | ||
| #!/usr/bin/env bun | ||
| /** | ||
| * Script to run Regula OPA tests on all terraform policies. | ||
| * | ||
| * Usage: | ||
| * bun .scripts/run-opa-tests.ts [--filter <pattern>] [--verbose] | ||
| * | ||
| * Options: | ||
| * --filter <pat> Only run tests matching the pattern (e.g., "aws/AWS Backup") | ||
| * --verbose Show verbose output including test details | ||
| * | ||
| * Examples: | ||
| * bun .scripts/run-opa-tests.ts # Run all tests | ||
| * bun .scripts/run-opa-tests.ts --filter "AWS Lambda" # Run only Lambda tests | ||
| * bun .scripts/run-opa-tests.ts --filter gcp # Run only GCP tests | ||
| */ | ||
|
|
||
| import { spawnSync } from "child_process"; | ||
| import { readdirSync, statSync } from "fs"; | ||
| import { join, dirname } from "path"; | ||
|
|
||
| const TERRAFORM_DIR = join(import.meta.dirname, "..", "terraform"); | ||
|
|
||
| function findRegoFiles(dir: string): string[] { | ||
| const results: string[] = []; | ||
|
|
||
| function walk(currentDir: string) { | ||
| const entries = readdirSync(currentDir); | ||
| for (const entry of entries) { | ||
| const fullPath = join(currentDir, entry); | ||
| const stat = statSync(fullPath); | ||
| if (stat.isDirectory()) { | ||
| walk(fullPath); | ||
| } else if (entry.endsWith(".rego")) { | ||
| results.push(fullPath); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| walk(dir); | ||
| return results; | ||
| } | ||
|
|
||
| function findTestDirectories(baseDir: string): string[] { | ||
| const dirs = new Set<string>(); | ||
| const regoFiles = findRegoFiles(baseDir); | ||
|
|
||
| for (const file of regoFiles) { | ||
| dirs.add(dirname(file)); | ||
| } | ||
|
|
||
| return Array.from(dirs).sort(); | ||
| } | ||
|
|
||
| interface TestResult { | ||
| path: string; | ||
| success: boolean; | ||
| output: string; | ||
| configsLoaded: number; | ||
| } | ||
|
|
||
| function runRegulaTest(dir: string): TestResult { | ||
| const result: TestResult = { | ||
| path: dir, | ||
| success: false, | ||
| output: "", | ||
| configsLoaded: 0 | ||
| }; | ||
|
|
||
| const proc = spawnSync("npx", ["regula-wasi", "test", dir], { | ||
| encoding: "utf-8", | ||
| timeout: 120000, | ||
| cwd: process.cwd() | ||
| }); | ||
|
|
||
| result.output = (proc.stdout || "") + (proc.stderr || ""); | ||
| result.success = proc.status === 0; | ||
|
|
||
| // Extract number of loaded configs | ||
| const match = result.output.match(/Loaded (\d+) IaC configurations/); | ||
| if (match) { | ||
| result.configsLoaded = parseInt(match[1], 10); | ||
| } | ||
|
|
||
| return result; | ||
| } | ||
|
|
||
| async function main() { | ||
| const args = process.argv.slice(2); | ||
| const verbose = args.includes("--verbose"); | ||
| const filterIndex = args.indexOf("--filter"); | ||
| const filter = filterIndex !== -1 ? args[filterIndex + 1] : null; | ||
|
|
||
| console.log("=== Regula OPA Test Runner ===\n"); | ||
|
|
||
| // Check regula-wasi version | ||
| const versionProc = spawnSync("npx", ["regula-wasi", "version"], { | ||
| encoding: "utf-8", | ||
| timeout: 30000 | ||
| }); | ||
|
|
||
| if (versionProc.status !== 0) { | ||
| console.error("Error: regula-wasi is not installed. Run: npm install --save-dev regula-wasi"); | ||
| process.exit(1); | ||
| } | ||
|
|
||
| const versionOutput = (versionProc.stdout || "") + (versionProc.stderr || ""); | ||
| const opaVersion = versionOutput.match(/OPA v([\d.]+)/)?.[1] || "unknown"; | ||
| console.log(`Using regula-wasi with OPA v${opaVersion}\n`); | ||
|
|
||
| // Find all test directories | ||
| let testDirs = findTestDirectories(TERRAFORM_DIR); | ||
|
|
||
| if (filter) { | ||
| testDirs = testDirs.filter(d => d.toLowerCase().includes(filter.toLowerCase())); | ||
| console.log(`Filtering to ${testDirs.length} directories matching: ${filter}\n`); | ||
|
Comment on lines
+91
to
+116
There was a problem hiding this comment. Guard against Proposed fix const verbose = args.includes("--verbose");
const filterIndex = args.indexOf("--filter");
- const filter = filterIndex !== -1 ? args[filterIndex + 1] : null;
+ const filter = filterIndex !== -1 ? args[filterIndex + 1] : null;
+ if (filterIndex !== -1 && !filter) {
+ console.error("Error: --filter requires a value.");
+ process.exit(1);
+ }🤖 Prompt for AI Agents |
||
| } | ||
|
|
||
| console.log(`Found ${testDirs.length} test directories\n`); | ||
|
|
||
| if (testDirs.length === 0) { | ||
| console.log("No test directories found."); | ||
| return; | ||
| } | ||
|
|
||
| const results: TestResult[] = []; | ||
| let passed = 0; | ||
| let failed = 0; | ||
| let totalConfigs = 0; | ||
|
|
||
| for (const dir of testDirs) { | ||
| const relativePath = dir.replace(TERRAFORM_DIR + "/", ""); | ||
| process.stdout.write(`Testing: ${relativePath}... `); | ||
|
|
||
| const result = runRegulaTest(dir); | ||
| totalConfigs += result.configsLoaded; | ||
|
|
||
| if (result.success) { | ||
| console.log(`✓ PASS (${result.configsLoaded} configs)`); | ||
| passed++; | ||
| } else { | ||
| console.log("✗ FAIL"); | ||
| if (verbose) { | ||
| console.log(result.output); | ||
| } | ||
| failed++; | ||
| } | ||
|
|
||
| results.push(result); | ||
| } | ||
|
|
||
| console.log("\n=== Summary ==="); | ||
| console.log(`Directories: ${testDirs.length}`); | ||
| console.log(`IaC Configs: ${totalConfigs}`); | ||
| console.log(`Passed: ${passed}`); | ||
| console.log(`Failed: ${failed}`); | ||
|
|
||
| if (failed > 0) { | ||
| console.log("\nFailed tests:"); | ||
| for (const result of results) { | ||
| if (!result.success) { | ||
| const relativePath = result.path.replace(TERRAFORM_DIR + "/", ""); | ||
| console.log(` - ${relativePath}`); | ||
| } | ||
| } | ||
| process.exit(1); | ||
| } | ||
|
|
||
| console.log("\n✓ All tests passed!"); | ||
| } | ||
|
|
||
| main().catch(e => { | ||
| console.error(e); | ||
| process.exit(1); | ||
| }); | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| # Starchitect CloudGuard | ||
|
|
||
| Cloud security compliance policies and runtime checks for AWS and GCP. | ||
|
|
||
| ## Project Structure | ||
|
|
||
| ``` | ||
| starchitect-cloudguard/ | ||
| ├── terraform/ # OPA/Rego security policies (IaC scanning) | ||
| │ ├── aws/ # AWS policies (66 service categories) | ||
| │ └── gcp/ # GCP policies (11 service categories) | ||
| ├── runtime/ # TypeScript runtime checks (live cloud API) | ||
| │ ├── aws/ # AWS runtime checks with tests | ||
| │ └── gcp/ # GCP runtime checks with tests | ||
| ├── cli/ # CLI application (oclif-based) | ||
| └── .scripts/ # Build and test scripts | ||
| ``` | ||
|
Comment on lines
+7
to
+17
There was a problem hiding this comment. Add a language to the fenced block. Proposed fix-```
+```text
starchitect-cloudguard/
├── terraform/ # OPA/Rego security policies (IaC scanning)
│ ├── aws/ # AWS policies (66 service categories)
│ └── gcp/ # GCP policies (11 service categories)
├── runtime/ # TypeScript runtime checks (live cloud API)
│ ├── aws/ # AWS runtime checks with tests
│ └── gcp/ # GCP runtime checks with tests
├── cli/ # CLI application (oclif-based)
└── .scripts/ # Build and test scriptsIn |
||
|
|
||
| ## OPA/Rego Policies | ||
|
|
||
| Located in `terraform/aws/` and `terraform/gcp/`. Each policy directory contains: | ||
|
|
||
| - `*.rego` - Policy file using Fugue/Regula framework | ||
| - `*_pass.tf` - Terraform fixture that should pass the policy | ||
| - `*_fail.tf` - Terraform fixture that should fail the policy | ||
|
|
||
| ### Running OPA Tests | ||
|
|
||
| ```bash | ||
| # Run all OPA tests | ||
| npm run test:opa | ||
|
|
||
| # Run with verbose output | ||
| npm run test:opa:verbose | ||
|
|
||
| # Filter to specific policies | ||
| bun ./.scripts/run-opa-tests.ts --filter "AWS Lambda" | ||
| bun ./.scripts/run-opa-tests.ts --filter gcp | ||
| ``` | ||
|
|
||
| Uses `regula-wasi` (nonfx fork with OPA v1.12.2) to evaluate policies against test fixtures. | ||
|
|
||
| ## Runtime Tests | ||
|
|
||
| TypeScript tests in `runtime/` use BUN test runner with Jest-style syntax: | ||
|
|
||
| ```bash | ||
| # Run all runtime tests | ||
| npm test | ||
| # or | ||
| bun test | ||
| ``` | ||
|
|
||
| ## Commands | ||
|
|
||
| | Command | Description | | ||
| | ------------------ | ---------------------------- | | ||
| | `npm test` | Run runtime tests (bun test) | | ||
| | `npm run test:opa` | Run OPA/Rego policy tests | | ||
| | `npm run build` | TypeScript compilation | | ||
| | `npm run lint` | Run ESLint | | ||
| | `npm run prettify` | Format code with Prettier | | ||
|
|
||
| ## Writing Policies | ||
|
|
||
| Rego policies use the Fugue framework: | ||
|
|
||
| ```rego | ||
| package rules.example_policy | ||
|
|
||
| import data.fugue | ||
|
|
||
| __rego__metadoc__ := { | ||
| "id": "EXAMPLE.1", | ||
| "title": "Example policy title", | ||
| "description": "Description of what this policy checks", | ||
| "custom": {"severity": "High", "author": "Starchitect Agent"}, | ||
| } | ||
|
|
||
| resource_type := "MULTIPLE" | ||
|
|
||
| resources = fugue.resources("aws_example_resource") | ||
|
|
||
| policy[p] { | ||
| resource := resources[_] | ||
| # check passes | ||
| p = fugue.allow_resource(resource) | ||
| } | ||
|
|
||
| policy[p] { | ||
| resource := resources[_] | ||
| # check fails | ||
| p = fugue.deny_resource_with_message(resource, "Error message") | ||
| } | ||
| ``` | ||
|
|
||
| ## Dependencies | ||
|
|
||
| - **regula-wasi**: OPA/Rego policy evaluation (WASI build with OPA v1.12.2) | ||
| - **bun**: Test runner and TypeScript execution | ||
| - **AWS SDK v3**: For runtime checks and test mocking | ||
| - **Google Cloud clients**: For GCP runtime checks | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Handle spawnSync errors/timeouts explicitly.
If
npxis missing or a timeout occurs,proc.error/proc.status === nullshould be surfaced.Proposed fix
const proc = spawnSync("npx", ["regula-wasi", "test", dir], { encoding: "utf-8", timeout: 120000, cwd: process.cwd() }); - result.output = (proc.stdout || "") + (proc.stderr || ""); - result.success = proc.status === 0; + if (proc.error) { + result.output = String(proc.error); + return result; + } + if (proc.status === null) { + result.output = (proc.stdout || "") + (proc.stderr || "") + "\nProcess terminated (timeout or signal)."; + return result; + } + result.output = (proc.stdout || "") + (proc.stderr || ""); + result.success = proc.status === 0;📝 Committable suggestion
🤖 Prompt for AI Agents