From a4f6fecc940129eaa207f288c473716b3137f037 Mon Sep 17 00:00:00 2001 From: HiranoMasaaki Date: Mon, 23 Feb 2026 17:00:11 +0000 Subject: [PATCH] fix: use startRun in single-type filter E2E test to avoid LLM flakiness The --filter completeRun test consistently fails because completeRun is only emitted after a successful LLM call. When the LLM errors (common under concurrent test load), the runtime emits stopRunByError instead, which the filter blocks, resulting in zero events with exit code 0. Switch to --filter startRun which is emitted before any LLM calls, making the test reliable while still validating the single-type filter mechanism. Co-Authored-By: Claude Opus 4.6 --- e2e/perstack-cli/options.test.ts | 41 ++++++++++++++------------------ 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/e2e/perstack-cli/options.test.ts b/e2e/perstack-cli/options.test.ts index c1792d82..3ab1cd56 100644 --- a/e2e/perstack-cli/options.test.ts +++ b/e2e/perstack-cli/options.test.ts @@ -162,35 +162,30 @@ describe.concurrent("CLI Options", () => { describe.concurrent("CLI Options - Filter", () => { /** Verifies --filter option with single type. - * Retries up to 3 times because --filter completeRun only outputs the final - * event; when the LLM call errors the runtime emits stopRunByError (blocked - * by the filter) and exits 0 with zero events. + * Uses startRun (emitted before LLM calls) to avoid flakiness from LLM + * errors that produce stopRunByError instead of completeRun. */ it( - "should filter events to only completeRun", + "should filter events to only startRun", async () => { - const args = [ - "run", - "--config", - GLOBAL_RUNTIME_CONFIG, - "--filter", - "completeRun", - "e2e-global-runtime", - "Say hello", - ] - let result = withEventParsing(await runCli(args, { timeout: LLM_TIMEOUT })) - for ( - let attempt = 1; - attempt < 3 && result.exitCode === 0 && result.events.length === 0; - attempt++ - ) { - result = withEventParsing(await runCli(args, { timeout: LLM_TIMEOUT })) - } + const cmdResult = await runCli( + [ + "run", + "--config", + GLOBAL_RUNTIME_CONFIG, + "--filter", + "startRun", + "e2e-global-runtime", + "Say hello", + ], + { timeout: LLM_TIMEOUT }, + ) + const result = withEventParsing(cmdResult) expect(result.exitCode).toBe(0) - // All events should be completeRun + // All events should be startRun const eventTypes = result.events.map((e) => e.type) - expect(eventTypes.every((t) => t === "completeRun")).toBe(true) + expect(eventTypes.every((t) => t === "startRun")).toBe(true) expect(eventTypes.length).toBeGreaterThan(0) }, LLM_TIMEOUT,