fix(e2e): improve test reliability and fix broken assertions (#398)

* fix(e2e): improve test reliability and fix broken assertions

- Update streaming event names (startStreamingReasoning, completeStreamingReasoning)
- Fix lazy-init.toml to use local e2e-mcp-server path instead of npx
- Add --run-id option to runtime CLI
- Refactor PDF/image tests to use flow-based assertions instead of flaky keyword matching
- Add --config to publish.test.ts unpublish commands
- Add infrastructure failure detection helper for Docker tests
- Support additionalVolumes in Docker runtime for local package mounting

All 142 E2E tests now pass reliably.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix(core): add runId to RunParamsInput type

- Add runId?: string to RunParamsInput for CLI --run-id option
- Add changeset for E2E test reliability fixes

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* style: format streaming.test.ts

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix(core): add volume option to CommandOptions schema

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor(runtime): remove unused ReasoningDetail type export

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: FL4TLiN3

.changeset/fix-e2e-test-reliability.md

@@ -0,0 +1,16 @@
+---
+"@perstack/core": patch
+"@perstack/runtime": patch
+"@perstack/docker": patch
+"@perstack/e2e-mcp-server": patch
+"perstack": patch
+---
+
+fix(e2e): improve test reliability and fix broken assertions
+
+- Update streaming event names to match state-machine-redesign changes
+- Fix lazy-init.toml to use local e2e-mcp-server path
+- Add --run-id option to runtime CLI
+- Refactor PDF/image tests to use flow-based assertions
+- Add infrastructure failure detection for Docker tests
+- Support additionalVolumes in Docker runtime

apps/e2e-mcp-server/bin/server.ts

@@ -1,4 +1,3 @@
-#!/usr/bin/env node
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
 import { createServer } from "../src/server.js"
 

apps/e2e-mcp-server/tsup.config.ts

@@ -1,10 +1,25 @@
 import { defineConfig, type Options } from "tsup"
 import { baseConfig } from "../../tsup.config.js"
-export const e2eMcpServerConfig: Options = {
+
+// Library entry - normal external dependencies
+export const libConfig: Options = {
   ...baseConfig,
   entry: {
-    "bin/server": "bin/server.ts",
     "src/index": "src/index.ts",
   },
 }
-export default defineConfig(e2eMcpServerConfig)
+
+// Standalone server binary - bundle all dependencies for Docker execution
+export const serverConfig: Options = {
+  ...baseConfig,
+  entry: {
+    "bin/server": "bin/server.ts",
+  },
+  dts: false, // No types needed for binary
+  noExternal: [/.*/], // Bundle all dependencies
+  banner: {
+    js: "#!/usr/bin/env node",
+  },
+}
+
+export default defineConfig([libConfig, serverConfig])

apps/perstack/src/run.ts

@@ -59,6 +59,12 @@ export const runCommand = new Command()
   .option("-i, --interactive-tool-call-result", "Query is interactive tool call result")
   .option("--runtime <runtime>", "Execution runtime (docker, local, cursor, claude-code, gemini)")
   .option("--workspace <workspace>", "Workspace directory for Docker runtime")
+  .option(
+    "--volume <volume>",
+    "Additional volume mount for Docker runtime (format: hostPath:containerPath:mode, can be specified multiple times)",
+    (value: string, previous: string[]) => previous.concat(value),
+    [] as string[],
+  )
   .option(
     "--filter <types>",
     "Filter events by type (comma-separated, e.g., completeRun,stopRunByError)",
@@ -126,6 +132,7 @@ export const runCommand = new Command()
         eventListener,
         workspace: input.options.workspace,
         additionalEnvKeys: input.options.env,
+        additionalVolumes: input.options.volume,
       })
     } catch (error) {
       if (error instanceof Error) {

apps/runtime/bin/cli.ts

@@ -54,6 +54,7 @@ program
     "Timeout for each generation in milliseconds, default is 60000 (1 minute)",
   )
   .option("--job-id <jobId>", "Job ID for identifying the job")
+  .option("--run-id <runId>", "Run ID for identifying the run")
   .option(
     "--env-path <path>",
     "Path to the environment file (can be specified multiple times), default is .env and .env.local",
@@ -98,6 +99,7 @@ program
         {
           setting: {
             jobId: input.options.jobId,
+            runId: input.options.runId,
             expertKey: input.expertKey,
             input: { text: input.query },
             experts,

apps/runtime/src/helpers/thinking.ts

@@ -48,6 +48,3 @@ export function extractThinkingText(reasoning: ReasoningPart[] | undefined): str
     .map((r) => r.text)
     .join("\n")
 }
-
-// Re-export for backwards compatibility
-export type { ReasoningPart as ReasoningDetail }

e2e/experts/docker-attack-scenarios.toml

@@ -21,8 +21,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-metadata".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -42,8 +42,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-ssrf".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -63,8 +63,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-filesystem".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -85,8 +85,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-symlink".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -106,8 +106,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-proxy".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -127,8 +127,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-env".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -148,8 +148,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-exfiltrate".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -169,8 +169,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-dns-exfil".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -190,8 +190,8 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-harvest-env".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com"]
 lazyInit = false
 
@@ -211,7 +211,7 @@ pick = ["attemptCompletion", "think"]
 
 [experts."attack-allowed-domains".skills."attacker"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.anthropic.com", "httpbin.org"]
 lazyInit = false

e2e/experts/docker-security.toml

@@ -87,14 +87,14 @@ pick = ["attemptCompletion", "think"]
 
 [experts."docker-security-multi-skill".skills."network-github"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["api.github.com"]
 lazyInit = false
 
 [experts."docker-security-multi-skill".skills."network-httpbin"]
 type = "mcpStdioSkill"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["/repo/apps/e2e-mcp-server/dist/bin/server.js"]
 allowedDomains = ["httpbin.org"]
 lazyInit = false

e2e/experts/lazy-init.toml

@@ -24,8 +24,8 @@ lazyInit = false
 [experts."e2e-lazy-init-all-false".skills."attacker"]
 type = "mcpStdioSkill"
 description = "E2E MCP server (no lazy init)"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["apps/e2e-mcp-server/dist/bin/server.js"]
 lazyInit = false
 
 # Expert with multiple skills: one lazyInit=false (required), one lazyInit=true
@@ -49,6 +49,6 @@ lazyInit = false
 [experts."e2e-lazy-init-mixed".skills."attacker"]
 type = "mcpStdioSkill"
 description = "E2E MCP server (lazy init)"
-command = "npx"
-packageName = "@perstack/e2e-mcp-server"
+command = "node"
+args = ["apps/e2e-mcp-server/dist/bin/server.js"]
 lazyInit = true

e2e/lib/event-parser.ts

@@ -14,15 +14,14 @@ export type CheckpointState = {
   partialToolResults: ToolCallInfo[]
 }
 
+// Note: callDelegate, callInteractiveTool, finishAllToolCalls were removed in state-machine-redesign
 const RELEVANT_EVENT_TYPES = [
   "startRun",
+  "resumeFromStop",
   "callTools",
-  "callDelegate",
-  "callInteractiveTool",
   "stopRunByDelegate",
   "stopRunByInteractiveTool",
   "resumeToolCalls",
-  "finishAllToolCalls",
   "completeRun",
   "resolveToolResults",
 ] as const