feat: expose maxMemory to prevent OOM from untrusted input (#1075)

Adds `BashBuilder::max_memory(bytes)` in the Rust core that caps
`max_total_variable_bytes` and clamps `max_function_body_bytes`.
Exposed through JS bindings (`maxMemory`) and Python bindings
(`max_memory`) on both `Bash` and `BashTool` classes.

Includes Rust unit test + doctest and 5 JS security tests.

Closes #1072

Author: chaliy

crates/bashkit-js/__test__/security.spec.ts

@@ -88,6 +88,75 @@ test("WB: fork bomb pattern blocked (TM-DOS-021)", (t) => {
   );
 });
 
+// ============================================================================
+// 1b. WHITE-BOX — Memory Limit Enforcement (TM-DOS-059)
+// ============================================================================
+
+test("WB: maxMemory caps exponential string doubling (TM-DOS-059)", (t) => {
+  // 1 KB limit — string doubling silently stops when budget is exceeded
+  const bash = new Bash({
+    maxMemory: 1024,
+    maxLoopIterations: 10000,
+    maxCommands: 10000,
+  });
+  const r = bash.executeSync(
+    'x=AAAAAAAAAA; i=0; while [ $i -lt 25 ]; do x="$x$x"; i=$((i+1)); done; echo ${#x}',
+  );
+  // String must be capped well below what 25 doublings would produce (335 544 320)
+  const len = parseInt(r.stdout.trim(), 10);
+  t.true(len <= 1024, `string length ${len} must be ≤ 1024`);
+});
+
+test("WB: maxMemory — small scripts still work within budget", (t) => {
+  const bash = new Bash({ maxMemory: 1024 * 1024 }); // 1 MB
+  const r = bash.executeSync('x="hello world"; echo $x');
+  t.is(r.exitCode, 0);
+  t.is(r.stdout.trim(), "hello world");
+});
+
+test("WB: maxMemory — recovery after exceeding limit", (t) => {
+  const bash = new Bash({
+    maxMemory: 1024,
+    maxLoopIterations: 10000,
+    maxCommands: 10000,
+  });
+  // Exceed limit (variable silently stops growing)
+  bash.executeSync(
+    'x=AAAAAAAAAA; i=0; while [ $i -lt 25 ]; do x="$x$x"; i=$((i+1)); done',
+  );
+  // Next exec should still work
+  const r = bash.executeSync("echo recovered");
+  t.is(r.exitCode, 0);
+  t.is(r.stdout.trim(), "recovered");
+});
+
+test("WB: maxMemory via BashTool (TM-DOS-059)", (t) => {
+  const tool = new BashTool({
+    maxMemory: 1024,
+    maxLoopIterations: 10000,
+    maxCommands: 10000,
+  });
+  const r = tool.executeSync(
+    'x=AAAAAAAAAA; i=0; while [ $i -lt 25 ]; do x="$x$x"; i=$((i+1)); done; echo ${#x}',
+  );
+  const len = parseInt(r.stdout.trim(), 10);
+  t.true(len <= 1024, `BashTool: string length ${len} must be ≤ 1024`);
+});
+
+test("WB: default memory limit prevents OOM without maxMemory", (t) => {
+  // Without maxMemory, default 10 MB limit still applies
+  const bash = new Bash({ maxLoopIterations: 10000, maxCommands: 10000 });
+  const r = bash.executeSync(
+    'x=AAAAAAAAAA; i=0; while [ $i -lt 30 ]; do x="$x$x"; i=$((i+1)); done; echo ${#x}',
+  );
+  // 30 doublings of 10 bytes = 10 GB without limits; default 10 MB cap stops it
+  const len = parseInt(r.stdout.trim(), 10);
+  t.true(
+    len <= 10_000_000,
+    `default limit: string length ${len} must be ≤ 10MB`,
+  );
+});
+
 // ============================================================================
 // 2. WHITE-BOX — Output Truncation (TM-DOS-002)
 // ============================================================================

crates/bashkit-js/src/lib.rs

@@ -433,6 +433,12 @@ pub struct BashOptions {
     pub max_parser_operations: Option<u32>,
     pub max_stdout_bytes: Option<u32>,
     pub max_stderr_bytes: Option<u32>,
+    /// Maximum interpreter memory in bytes (variables, arrays, functions).
+    ///
+    /// Caps `max_total_variable_bytes` and clamps `max_function_body_bytes`.
+    /// Prevents OOM from untrusted input such as exponential string doubling.
+    /// Default (when omitted): 10 MB.
+    pub max_memory: Option<f64>,
     /// Whether to capture the final environment state in ExecResult.
     pub capture_final_env: Option<bool>,
     /// Files to mount in the virtual filesystem.
@@ -461,6 +467,7 @@ fn default_opts() -> BashOptions {
         max_parser_operations: None,
         max_stdout_bytes: None,
         max_stderr_bytes: None,
+        max_memory: None,
         capture_final_env: None,
         files: None,
         mounts: None,
@@ -490,6 +497,7 @@ struct SharedState {
     max_parser_operations: Option<u32>,
     max_stdout_bytes: Option<u32>,
     max_stderr_bytes: Option<u32>,
+    max_memory: Option<f64>,
     capture_final_env: Option<bool>,
     mounts: Option<Vec<MountConfig>>,
     python: bool,
@@ -1459,6 +1467,10 @@ fn build_bash_from_state(state: &SharedState, files: Option<&HashMap<String, Str
 
     builder = builder.limits(build_limits(state));
 
+    if let Some(max_mem) = state.max_memory {
+        builder = builder.max_memory(max_mem as usize);
+    }
+
     // Mount files into the virtual filesystem
     if let Some(files) = files {
         for (path, content) in files {
@@ -1533,6 +1545,7 @@ fn shared_state_from_opts(
         max_parser_operations: opts.max_parser_operations,
         max_stdout_bytes: opts.max_stdout_bytes,
         max_stderr_bytes: opts.max_stderr_bytes,
+        max_memory: opts.max_memory,
         capture_final_env: opts.capture_final_env,
         mounts: mounts.clone(),
         python: py,
@@ -1565,6 +1578,7 @@ fn shared_state_from_opts(
         max_parser_operations: opts.max_parser_operations,
         max_stdout_bytes: opts.max_stdout_bytes,
         max_stderr_bytes: opts.max_stderr_bytes,
+        max_memory: opts.max_memory,
         capture_final_env: opts.capture_final_env,
         mounts,
         python: py,

crates/bashkit-js/wrapper.ts

@@ -32,6 +32,18 @@ export interface BashOptions {
   hostname?: string;
   maxCommands?: number;
   maxLoopIterations?: number;
+  /**
+   * Maximum interpreter memory in bytes (variables, arrays, functions).
+   *
+   * Caps the total byte budget for variable storage and function bodies.
+   * Prevents OOM from untrusted input such as exponential string doubling.
+   *
+   * @example
+   * ```typescript
+   * const bash = new Bash({ maxMemory: 10 * 1024 * 1024 }); // 10 MB
+   * ```
+   */
+  maxMemory?: number;
   /**
    * Files to mount in the virtual filesystem.
    * Keys are absolute paths, values are content strings or lazy providers.
@@ -122,6 +134,7 @@ function toNativeOptions(
     hostname: options?.hostname,
     maxCommands: options?.maxCommands,
     maxLoopIterations: options?.maxLoopIterations,
+    maxMemory: options?.maxMemory,
     files: resolvedFiles,
     python: options?.python,
     externalFunctions: options?.externalFunctions,

crates/bashkit-python/src/lib.rs

@@ -662,6 +662,7 @@ pub struct PyBash {
     real_mounts: Vec<RealMountConfig>,
     max_commands: Option<u64>,
     max_loop_iterations: Option<u64>,
+    max_memory: Option<u64>,
 }
 
 #[pymethods]
@@ -672,6 +673,7 @@ impl PyBash {
         hostname=None,
         max_commands=None,
         max_loop_iterations=None,
+        max_memory=None,
         python=false,
         external_functions=None,
         external_handler=None,
@@ -689,6 +691,7 @@ impl PyBash {
         hostname: Option<String>,
         max_commands: Option<u64>,
         max_loop_iterations: Option<u64>,
+        max_memory: Option<u64>,
         python: bool,
         external_functions: Option<Vec<String>>,
         external_handler: Option<Py<PyAny>>,
@@ -717,6 +720,10 @@ impl PyBash {
         }
         builder = builder.limits(limits);
 
+        if let Some(mm) = max_memory {
+            builder = builder.max_memory(usize::try_from(mm).unwrap_or(usize::MAX));
+        }
+
         let (mounted_text_files, real_mounts) = parse_mount_configs(
             mount_text,
             mount_readonly_text,
@@ -786,6 +793,7 @@ impl PyBash {
             real_mounts,
             max_commands,
             max_loop_iterations,
+            max_memory,
         })
     }
 
@@ -935,6 +943,7 @@ impl PyBash {
         let hostname = self.hostname.clone();
         let max_commands = self.max_commands;
         let max_loop_iterations = self.max_loop_iterations;
+        let max_memory = self.max_memory;
         let python = self.python;
         let external_functions = self.external_functions.clone();
         let mounted_text_files = self.mounted_text_files.clone();
@@ -961,6 +970,9 @@ impl PyBash {
                     limits = limits.max_loop_iterations(usize::try_from(mli).unwrap_or(usize::MAX));
                 }
                 builder = builder.limits(limits);
+                if let Some(mm) = max_memory {
+                    builder = builder.max_memory(usize::try_from(mm).unwrap_or(usize::MAX));
+                }
                 builder = apply_python_config(builder, python, external_functions, handler_clone);
                 builder = apply_fs_config(builder, &mounted_text_files, &real_mounts);
                 *bash = builder.build();
@@ -1067,6 +1079,7 @@ pub struct BashTool {
     real_mounts: Vec<RealMountConfig>,
     max_commands: Option<u64>,
     max_loop_iterations: Option<u64>,
+    max_memory: Option<u64>,
 }
 
 impl BashTool {
@@ -1101,6 +1114,7 @@ impl BashTool {
         hostname=None,
         max_commands=None,
         max_loop_iterations=None,
+        max_memory=None,
         mount_text=None,
         mount_readonly_text=None,
         mount_real_readonly=None,
@@ -1113,6 +1127,7 @@ impl BashTool {
         hostname: Option<String>,
         max_commands: Option<u64>,
         max_loop_iterations: Option<u64>,
+        max_memory: Option<u64>,
         mount_text: Option<Vec<(String, String)>>,
         mount_readonly_text: Option<Vec<(String, String)>>,
         mount_real_readonly: Option<Vec<String>>,
@@ -1138,6 +1153,10 @@ impl BashTool {
         }
         builder = builder.limits(limits);
 
+        if let Some(mm) = max_memory {
+            builder = builder.max_memory(usize::try_from(mm).unwrap_or(usize::MAX));
+        }
+
         let (mounted_text_files, real_mounts) = parse_mount_configs(
             mount_text,
             mount_readonly_text,
@@ -1163,6 +1182,7 @@ impl BashTool {
             real_mounts,
             max_commands,
             max_loop_iterations,
+            max_memory,
         })
     }
 
@@ -1295,6 +1315,7 @@ impl BashTool {
         let real_mounts = self.real_mounts.clone();
         let max_commands = self.max_commands;
         let max_loop_iterations = self.max_loop_iterations;
+        let max_memory = self.max_memory;
         let cancelled = self.cancelled.clone();
 
         py.detach(|| {
@@ -1315,6 +1336,9 @@ impl BashTool {
                     limits = limits.max_loop_iterations(usize::try_from(mli).unwrap_or(usize::MAX));
                 }
                 builder = builder.limits(limits);
+                if let Some(mm) = max_memory {
+                    builder = builder.max_memory(usize::try_from(mm).unwrap_or(usize::MAX));
+                }
                 builder = apply_fs_config(builder, &mounted_text_files, &real_mounts);
                 *bash = builder.build();
                 // Swap the cancellation token to the new interpreter's token so

crates/bashkit/src/lib.rs

@@ -1100,6 +1100,30 @@ impl BashBuilder {
         self
     }
 
+    /// Cap total interpreter memory to `bytes`.
+    ///
+    /// Convenience wrapper over [`memory_limits`](Self::memory_limits) that
+    /// sets `max_total_variable_bytes` to `bytes` and clamps
+    /// `max_function_body_bytes` to `min(bytes, default)`. Count-based
+    /// sub-limits (variable count, array entries, function count) stay at
+    /// their defaults.
+    ///
+    /// # Example
+    /// ```
+    /// # use bashkit::Bash;
+    /// let bash = Bash::builder()
+    ///     .max_memory(10 * 1024 * 1024)   // 10 MB
+    ///     .build();
+    /// ```
+    pub fn max_memory(self, bytes: usize) -> Self {
+        let defaults = MemoryLimits::default();
+        self.memory_limits(
+            MemoryLimits::new()
+                .max_total_variable_bytes(bytes)
+                .max_function_body_bytes(bytes.min(defaults.max_function_body_bytes)),
+        )
+    }
+
     /// Set the trace mode for structured execution tracing.
     ///
     /// - `TraceMode::Off` (default): No events, zero overhead
@@ -5367,4 +5391,23 @@ echo missing fi"#,
     async fn test_streaming_equivalence_subshell() {
         assert_streaming_equivalence("x=$(echo hello); echo $x").await;
     }
+
+    #[tokio::test]
+    async fn test_max_memory_caps_string_growth() {
+        let mut bash = Bash::builder()
+            .max_memory(1024)
+            .limits(
+                ExecutionLimits::new()
+                    .max_commands(10_000)
+                    .max_loop_iterations(10_000),
+            )
+            .build();
+        let result = bash
+            .exec(r#"x=AAAAAAAAAA; i=0; while [ $i -lt 25 ]; do x="$x$x"; i=$((i+1)); done; echo ${#x}"#)
+            .await
+            .unwrap();
+        let len: usize = result.stdout.trim().parse().unwrap();
+        // 25 doublings of 10 bytes = 335 544 320 without limits; must be capped ≤ 1024
+        assert!(len <= 1024, "string length {len} must be ≤ 1024");
+    }
 }

specs/009-implementation-status.md

@@ -450,6 +450,7 @@ Default limits (configurable):
 | Commands | 10,000 |
 | Loop iterations | 100,000 |
 | Function depth | 100 |
+| Interpreter memory | 10MB (`max_memory` / `maxMemory`) |
 | Output size | 10MB |
 | Parser timeout | 5 seconds |
 | Parser operations (fuel) | 100,000 |
@@ -473,7 +474,7 @@ NAPI-RS bindings in `crates/bashkit-js/`. TypeScript wrapper in `wrapper.ts`.
 
 **Platform matrix:** macOS (x86_64, aarch64), Linux (x86_64, aarch64), Windows (x86_64), WASM
 
-**Tests:** `crates/bashkit-js/__test__/` — VFS roundtrip, interop, error handling, security (90+ white/black-box tests covering TM-DOS, TM-ESC, TM-INF, TM-INT, TM-ISO, TM-UNI, TM-INJ, TM-NET)
+**Tests:** `crates/bashkit-js/__test__/` — VFS roundtrip, interop, error handling, security (100+ white/black-box tests covering TM-DOS, TM-ESC, TM-INF, TM-INT, TM-ISO, TM-UNI, TM-INJ, TM-NET)
 
 ### Python (`bashkit`)