From b1a9d3aaf4e14be604c6989854d8ce4ff368a5d7 Mon Sep 17 00:00:00 2001 From: HiranoMasaaki Date: Wed, 25 Feb 2026 06:43:47 +0000 Subject: [PATCH] chore: remove create-expert e2e test suite The create-expert e2e test takes too long to run. Remove the test file, CI matrix entry, and README references. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/e2e.yml | 2 - e2e/README.md | 11 -- e2e/create-expert/create-expert.test.ts | 184 ------------------------ 3 files changed, 197 deletions(-) delete mode 100644 e2e/create-expert/create-expert.test.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f51ce2aa..c909e753 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -47,8 +47,6 @@ jobs: - suite: cli-streaming # Streaming event tests (~50s) files: e2e/perstack-cli/streaming.test.ts - - suite: create-expert - files: e2e/create-expert steps: - name: Checkout uses: actions/checkout@v6 diff --git a/e2e/README.md b/e2e/README.md index 154773e8..5d76ef96 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -25,8 +25,6 @@ bun test e2e/ -- --testNamePattern "delegate" ``` e2e/ -├── create-expert/ # Create expert tests -│ └── create-expert.test.ts # Expert creation and modification ├── perstack-cli/ # perstack CLI tests │ ├── bundled-base.test.ts # Bundled base skill │ ├── continue.test.ts # Continue job, resume from checkpoint @@ -61,15 +59,6 @@ e2e/ ## Functional Test Categories -### create-expert/ - -#### Create Expert (`create-expert.test.ts`) - -| Test | Purpose | -| ----------------------------------------- | ----------------------------------------- | -| `should create a new perstack.toml` | Verify new expert creation via delegation | -| `should modify an existing perstack.toml` | Verify existing experts are preserved | - ### perstack-cli/ #### Continue Job (`continue.test.ts`) diff --git a/e2e/create-expert/create-expert.test.ts b/e2e/create-expert/create-expert.test.ts deleted file mode 100644 index a7d60670..00000000 --- a/e2e/create-expert/create-expert.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { describe, expect, it } from "bun:test" -import { spawn } from "node:child_process" -import fs from "node:fs" -import os from "node:os" -import path from "node:path" -import { assertEventSequenceContains } from "../lib/assertions.js" -import { extractToolCalls, filterEventsByType } from "../lib/event-parser.js" -import { injectProviderArgs } from "../lib/round-robin.js" -import { type CommandResult, type RunResult, withEventParsing } from "../lib/runner.js" - -const PROJECT_ROOT = path.resolve(process.cwd()) -const CLI_PATH = path.join(PROJECT_ROOT, "apps/create-expert/dist/bin/cli.js") -const LLM_TIMEOUT = 900_000 - -function runCreateExpert(query: string, cwd: string, timeout = LLM_TIMEOUT): Promise { - const args = injectProviderArgs(["--headless", query]) - return new Promise((resolve, reject) => { - let stdout = "" - let stderr = "" - const proc = spawn("bun", [CLI_PATH, ...args], { - cwd, - env: { ...process.env }, - stdio: ["pipe", "pipe", "pipe"], - }) - const timer = setTimeout(() => { - proc.kill("SIGTERM") - reject(new Error(`Timeout after ${timeout}ms`)) - }, timeout) - proc.stdout.on("data", (data: Buffer) => { - stdout += data.toString() - }) - proc.stderr.on("data", (data: Buffer) => { - stderr += data.toString() - }) - proc.on("close", (code) => { - clearTimeout(timer) - resolve({ stdout, stderr, exitCode: code ?? 0 }) - }) - proc.on("error", (err) => { - clearTimeout(timer) - reject(err) - }) - }).then(withEventParsing) -} - -function createTempDir(): string { - return fs.mkdtempSync(path.join(os.tmpdir(), "create-expert-")) -} - -function getAllCalledToolNames(result: RunResult): string[] { - return filterEventsByType(result.events, "callTools").flatMap((e) => - extractToolCalls(e).map((tc) => tc.toolName), - ) -} - -function diagnostics(result: RunResult): string { - const errorEvents = result.events - .filter((e) => e.type === "stopRunByError") - .map((e) => { - const err = (e as { error?: { name: string; message: string } }).error - return err ? `${err.name}: ${err.message}` : "unknown error" - }) - const parts = [`exitCode=${result.exitCode}`] - if (result.stderr) parts.push(`stderr=${result.stderr.slice(0, 500)}`) - if (errorEvents.length > 0) parts.push(`errors=[${errorEvents.join(", ")}]`) - return parts.join("\n") -} - -describe("create-expert", () => { - it( - "should create a new perstack.toml with MCP skill integration", - async () => { - let result: RunResult | undefined - let tomlContent = "" - let nonBaseSkillMatches: RegExpMatchArray | null = null - - for (let attempt = 0; attempt < 3; attempt++) { - const tempDir = createTempDir() - - result = await runCreateExpert( - "Create a GitHub repository analyzer expert that reads GitHub issues and pull requests via the GitHub API to generate project status reports", - tempDir, - ) - - expect(result.exitCode, diagnostics(result)).toBe(0) - - const controlFlow = result.events - .filter((e) => ["startRun", "stopRunByDelegate", "completeRun"].includes(e.type)) - .map((e) => e.type) - expect(controlFlow[0]).toBe("startRun") - expect(controlFlow).toContain("stopRunByDelegate") - expect(controlFlow.at(-1)).toBe("completeRun") - - const startEvents = filterEventsByType(result.events, "startRun") - const completeEvents = filterEventsByType(result.events, "completeRun") - expect( - startEvents.some((e) => (e as { expertKey: string }).expertKey === "create-expert"), - ).toBe(true) - expect( - completeEvents.some((e) => (e as { expertKey: string }).expertKey === "create-expert"), - ).toBe(true) - - expect(completeEvents.length).toBeGreaterThanOrEqual(3) - - const toolNames = getAllCalledToolNames(result) - expect(toolNames).toContain("writeTextFile") - expect(toolNames).toContain("addDelegateFromConfig") - - expect(toolNames, "searchMcpRegistry should be called by skill-finder").toContain( - "searchMcpRegistry", - ) - - const skillReportPath = path.join(tempDir, "skill-report.md") - expect(fs.existsSync(skillReportPath)).toBe(true) - - const tomlPath = path.join(tempDir, "perstack.toml") - expect(fs.existsSync(tomlPath)).toBe(true) - tomlContent = fs.readFileSync(tomlPath, "utf-8") - const expertMatches = tomlContent.match(/\[experts\."[^"]+"\]/g) - expect(expertMatches).not.toBeNull() - expect(expertMatches!.length).toBeGreaterThanOrEqual(1) - - nonBaseSkillMatches = tomlContent.match( - /\[experts\."[^"]+".skills\."(?!@perstack\/base")[^"]+"\]/g, - ) - if (nonBaseSkillMatches && nonBaseSkillMatches.length > 0) { - break - } - } - - expect( - nonBaseSkillMatches && nonBaseSkillMatches.length > 0, - "at least one expert should have a non-base MCP skill", - ).toBe(true) - }, - LLM_TIMEOUT, - ) - - it( - "should modify an existing perstack.toml", - async () => { - const tempDir = createTempDir() - - const existingToml = `model = "claude-sonnet-4-5" - -[provider] -providerName = "anthropic" - -[experts."existing-expert"] -version = "1.0.0" -description = "An existing expert" -instruction = "You are an existing expert." - -[experts."existing-expert".skills."@perstack/base"] -type = "mcpStdioSkill" -command = "npx" -packageName = "@perstack/base" -pick = ["attemptCompletion"] -` - fs.writeFileSync(path.join(tempDir, "perstack.toml"), existingToml) - - const result = await runCreateExpert("Add a testing expert that runs unit tests", tempDir) - - expect(result.exitCode, diagnostics(result)).toBe(0) - - expect( - assertEventSequenceContains(result.events, ["startRun", "stopRunByDelegate", "completeRun"]) - .passed, - ).toBe(true) - - const toolNames = getAllCalledToolNames(result) - expect(toolNames).toContain("writeTextFile") - expect(toolNames).toContain("addDelegateFromConfig") - - const tomlPath = path.join(tempDir, "perstack.toml") - const tomlContent = fs.readFileSync(tomlPath, "utf-8") - expect(tomlContent).toContain('[experts."existing-expert"]') - const expertMatches = tomlContent.match(/\[experts\."[^"]+"\]/g) - expect(expertMatches).not.toBeNull() - expect(expertMatches!.length).toBeGreaterThanOrEqual(2) - }, - LLM_TIMEOUT, - ) -})