Nativejq - many simple jq commands failed

package.json

@@ -49,7 +49,7 @@
   "scripts": {
     "build": "rm -rf dist && tsc && pnpm build:lib && pnpm build:browser && pnpm build:cli && pnpm build:shell && pnpm build:worker && pnpm build:clean && sed '1,/^-->/d' AGENTS.npm.md > dist/AGENTS.md",
     "build:clean": "find dist -name '*.test.js' -delete && find dist -name '*.test.d.ts' -delete",
-    "build:worker": "esbuild src/commands/python3/worker.ts --bundle --platform=node --format=esm --outfile=src/commands/python3/worker.js --external:pyodide && cp src/commands/python3/worker.js dist/commands/python3/worker.js && mkdir -p dist/bin/chunks && cp src/commands/python3/worker.js dist/bin/chunks/worker.js && mkdir -p dist/bundle/chunks && cp src/commands/python3/worker.js dist/bundle/chunks/worker.js",
+    "build:worker": "esbuild src/commands/python3/worker.ts --bundle --platform=node --format=esm --outfile=src/commands/python3/worker.js --external:pyodide && cp src/commands/python3/worker.js dist/commands/python3/worker.js && mkdir -p dist/bin/chunks && cp src/commands/python3/worker.js dist/bin/chunks/worker.js && mkdir -p dist/bundle/chunks && cp src/commands/python3/worker.js dist/bundle/chunks/worker.js && esbuild src/commands/jq/jq-worker.ts --bundle --platform=node --format=esm --outfile=dist/commands/jq/jq-worker.js && cp dist/commands/jq/jq-worker.js dist/bundle/chunks/jq-worker.js && cp dist/commands/jq/jq-worker.js dist/bin/chunks/jq-worker.js",
     "build:lib": "esbuild dist/index.js --bundle --splitting --platform=node --format=esm --minify --outdir=dist/bundle --chunk-names=chunks/[name]-[hash] --external:diff --external:minimatch --external:sprintf-js --external:turndown --external:sql.js --external:pyodide --external:@mongodb-js/zstd --external:node-liblzma --external:compressjs",
     "build:browser": "esbuild dist/browser.js --bundle --platform=browser --format=esm --minify --outfile=dist/bundle/browser.js --external:diff --external:minimatch --external:sprintf-js --external:turndown --external:node:* --external:@mongodb-js/zstd --external:node-liblzma --external:compressjs --define:__BROWSER__=true",
     "build:cli": "esbuild dist/cli/just-bash.js --bundle --splitting --platform=node --format=esm --minify --outdir=dist/bin --entry-names=[name] --chunk-names=chunks/[name]-[hash] --banner:js='#!/usr/bin/env node' --external:sql.js --external:pyodide --external:@mongodb-js/zstd --external:node-liblzma --external:compressjs",
@@ -95,6 +95,7 @@
     "fast-xml-parser": "^5.3.3",
     "file-type": "^21.2.0",
     "ini": "^6.0.0",
+    "jq-web": "^0.6.2",
     "minimatch": "^10.1.1",
     "modern-tar": "^0.7.3",
     "papaparse": "^5.5.3",

src/commands/base64/base64.binary.test.ts

@@ -130,5 +130,72 @@ describe("base64 with binary data", () => {
 
       expect(result.stdout).toBe("test content");
     });
+
+    it("should handle large binary files (1MB+)", async () => {
+      // Create a 1MB binary file with all byte values repeated
+      const size = 1024 * 1024; // 1MB
+      const data = new Uint8Array(size);
+      for (let i = 0; i < size; i++) {
+        data[i] = i % 256;
+      }
+
+      const env = new Bash({
+        files: {
+          "/large.bin": data,
+        },
+      });
+
+      // Encode the large file
+      await env.exec("base64 /large.bin > /encoded.txt");
+
+      // Decode it back
+      await env.exec("base64 -d /encoded.txt > /decoded.bin");
+
+      // Verify the decoded file matches the original
+      const decoded = await env.fs.readFileBuffer(
+        env.fs.resolvePath("/", "/decoded.bin"),
+      );
+
+      expect(decoded.length).toBe(size);
+      // Check first, middle, and last bytes
+      expect(decoded[0]).toBe(0);
+      expect(decoded[255]).toBe(255);
+      expect(decoded[size / 2]).toBe((size / 2) % 256);
+      expect(decoded[size - 1]).toBe((size - 1) % 256);
+
+      // Verify a sample of bytes throughout the file
+      for (let i = 0; i < size; i += 10000) {
+        expect(decoded[i]).toBe(i % 256);
+      }
+    });
+
+    it("should handle large files via pipe", async () => {
+      // Create a 512KB binary file
+      const size = 512 * 1024;
+      const data = new Uint8Array(size);
+      for (let i = 0; i < size; i++) {
+        data[i] = (i * 7) % 256; // Different pattern
+      }
+
+      const env = new Bash({
+        files: {
+          "/medium.bin": data,
+        },
+      });
+
+      // Round-trip through pipe
+      await env.exec("cat /medium.bin | base64 | base64 -d > /output.bin");
+
+      // Verify the output matches the original
+      const output = await env.fs.readFileBuffer(
+        env.fs.resolvePath("/", "/output.bin"),
+      );
+
+      expect(output.length).toBe(size);
+      // Check a sample of bytes
+      for (let i = 0; i < size; i += 5000) {
+        expect(output[i]).toBe((i * 7) % 256);
+      }
+    });
   });
 });

src/commands/base64/base64.ts

@@ -93,7 +93,19 @@ export const base64Command: Command = {
         // For decoding, read as text and strip whitespace
         const readResult = await readBinary(ctx, files, "base64");
         if (!readResult.ok) return readResult.error;
-        // Use binary string (latin1) to preserve bytes for input
+
+        // Use Buffer if available (Node.js) for better large file handling
+        if (typeof Buffer !== "undefined") {
+          const buffer = Buffer.from(readResult.data);
+          const cleaned = buffer.toString("utf8").replace(/\s/g, "");
+          const decoded = Buffer.from(cleaned, "base64");
+          // Convert to binary string (each char code = byte value)
+          // Use Buffer's latin1 encoding which treats each byte as a character
+          const result = decoded.toString("latin1");
+          return { stdout: result, stderr: "", exitCode: 0 };
+        }
+
+        // Browser fallback - use binary string (latin1) to preserve bytes for input
         const input = String.fromCharCode(...readResult.data);
         const cleaned = input.replace(/\s/g, "");
         // Decode base64 to binary string (each char code = byte value)
@@ -105,8 +117,15 @@ export const base64Command: Command = {
       const readResult = await readBinary(ctx, files, "base64");
       if (!readResult.ok) return readResult.error;
 
-      // Convert binary to base64
-      let encoded = btoa(String.fromCharCode(...readResult.data));
+      // Use Buffer if available (Node.js) for better large file handling
+      let encoded: string;
+      if (typeof Buffer !== "undefined") {
+        const buffer = Buffer.from(readResult.data);
+        encoded = buffer.toString("base64");
+      } else {
+        // Browser fallback - convert binary to base64
+        encoded = btoa(String.fromCharCode(...readResult.data));
+      }
 
       if (wrapCols > 0) {
         const lines: string[] = [];

src/commands/jq/jq-worker.ts

@@ -0,0 +1,66 @@
+/**
+ * Worker thread for executing jq-web with timeout protection.
+ * This allows us to terminate long-running jq operations.
+ */
+
+import { parentPort } from "node:worker_threads";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+
+interface WorkerMessage {
+  input: string;
+  filter: string;
+  flags: string[];
+}
+
+interface WorkerResult {
+  success: true;
+  output: string;
+  exitCode: number;
+}
+
+interface WorkerError {
+  success: false;
+  error: string;
+  exitCode: number;
+  stderr?: string;
+}
+
+if (!parentPort) {
+  throw new Error("This file must be run as a worker thread");
+}
+
+parentPort.on("message", async (message: WorkerMessage) => {
+  try {
+    const jqPromise: Promise<any> = require("jq-web");
+    const jq = await jqPromise;
+
+    try {
+      const output = jq.raw(message.input, message.filter, message.flags);
+      const result: WorkerResult = {
+        success: true,
+        output,
+        exitCode: 0,
+      };
+      parentPort!.postMessage(result);
+    } catch (e: any) {
+      const error: WorkerError = {
+        success: false,
+        error: e.message,
+        exitCode: e.exitCode || 3,
+        stderr: e.stderr,
+      };
+      parentPort!.postMessage(error);
+    }
+  } catch (e: any) {
+    const error: WorkerError = {
+      success: false,
+      error: e.message,
+      exitCode: 1,
+    };
+    parentPort!.postMessage(error);
+  }
+});
+
+// Made with Bob

src/commands/jq/jq.functions.test.ts

@@ -313,16 +313,18 @@ describe("jq builtin functions", () => {
       expect(result.exitCode).toBe(0);
     });
 
-    it("should return null for pow with non-numeric args", async () => {
+    it("should error for pow with non-numeric args", async () => {
       const env = new Bash();
       const result = await env.exec("jq -n 'pow(\"a\"; 2)'");
-      expect(result.stdout).toBe("null\n");
+      expect(result.exitCode).toBe(5);
+      expect(result.stderr).toContain("number required");
     });
 
-    it("should return null for atan2 with non-numeric args", async () => {
+    it("should error for atan2 with non-numeric args", async () => {
       const env = new Bash();
       const result = await env.exec("jq -n 'atan2(\"a\"; 2)'");
-      expect(result.stdout).toBe("null\n");
+      expect(result.exitCode).toBe(5);
+      expect(result.stderr).toContain("number required");
     });
   });
 });

src/commands/jq/jq.limits.test.ts

@@ -1,14 +1,17 @@
 import { describe, expect, it } from "vitest";
 import { Bash } from "../../Bash.js";
-import { ExecutionLimitError } from "../../interpreter/errors.js";
 
 /**
  * JQ Execution Limits Tests
  *
- * These tests verify that jq commands cannot cause runaway compute.
- * JQ programs should complete in bounded time regardless of input.
+ * NOTE: We now use jq-web (real jq compiled to WebAssembly) with worker-based
+ * timeout protection. Real jq does not have artificial iteration limits.
  *
- * IMPORTANT: All tests should complete quickly (<1s each).
+ * These tests verify that:
+ * 1. Infinite loops are terminated by timeout (1 second)
+ * 2. Normal operations that complete quickly work correctly
+ *
+ * IMPORTANT: Timeout tests may take up to 1 second each.
  */
 
 describe("JQ Execution Limits", () => {
@@ -18,8 +21,8 @@ describe("JQ Execution Limits", () => {
       // until condition that never becomes true
       const result = await env.exec(`echo 'null' | jq 'until(false; .)'`);
 
-      expect(result.stderr).toContain("too many iterations");
-      expect(result.exitCode).toBe(ExecutionLimitError.EXIT_CODE);
+      expect(result.stderr).toContain("timeout");
+      expect(result.exitCode).toBe(124); // Standard timeout exit code
     });
 
     it("should allow until that terminates", async () => {
@@ -37,8 +40,8 @@ describe("JQ Execution Limits", () => {
       // while condition that's always true
       const result = await env.exec(`echo '0' | jq '[while(true; . + 1)]'`);
 
-      expect(result.stderr).toContain("too many iterations");
-      expect(result.exitCode).toBe(ExecutionLimitError.EXIT_CODE);
+      expect(result.stderr).toContain("timeout");
+      expect(result.exitCode).toBe(124);
     });
 
     it("should allow while that terminates", async () => {
@@ -51,26 +54,26 @@ describe("JQ Execution Limits", () => {
   });
 
   describe("repeat protection", () => {
-    it("should protect against infinite repeat", async () => {
+    it("should protect against infinite repeat without limit", async () => {
       const env = new Bash();
-      // repeat with identity produces infinite stream
+      // repeat without limit produces infinite stream
       const result = await env.exec(
-        `echo '1' | jq '[limit(100000; repeat(.))]'`,
+        `echo '1' | jq 'repeat(.)'`,
       );
 
-      expect(result.stderr).toContain("too many iterations");
-      expect(result.exitCode).toBe(ExecutionLimitError.EXIT_CODE);
+      expect(result.stderr).toContain("timeout");
+      expect(result.exitCode).toBe(124);
     });
 
-    it("should allow repeat that terminates naturally", async () => {
+    it("should allow repeat with limit", async () => {
       const env = new Bash();
-      // repeat with update that eventually returns empty stops
+      // repeat with limit terminates after specified iterations
       const result = await env.exec(
-        `echo '5' | jq -c '[limit(10; repeat(if . > 0 then . - 1 else empty end))]'`,
+        `echo '1' | jq -c '[limit(5; repeat(.))]'`,
       );
 
       expect(result.exitCode).toBe(0);
-      expect(result.stdout.trim()).toBe("[5,4,3,2,1,0]");
+      expect(result.stdout.trim()).toBe("[1,1,1,1,1]");
     });
   });
 

src/commands/jq/jq.test.ts

@@ -253,8 +253,8 @@ describe("jq", () => {
       const output = JSON.parse(result.stdout);
       // group_by sorts by key: false < true (alphabetically)
       expect(output).toEqual([
-        { merged: true, count: 2 },
         { merged: false, count: 1 },
+        { merged: true, count: 2 },
       ]);
     });