From 179365fc79c99a692c5ba02aa4be11da0a80ae4e Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:31:16 -0400 Subject: [PATCH 1/7] Group hook registry and beforeEach execution per group --- .../src/worker/localTestRegistry.ts | 15 ++++++++++++- .../src/worker/testDeclaration.ts | 22 ++++++++++++++++--- packages/magnitude-test/src/worker/util.ts | 8 +++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/packages/magnitude-test/src/worker/localTestRegistry.ts b/packages/magnitude-test/src/worker/localTestRegistry.ts index 97e0b3f8..01ea7d53 100644 --- a/packages/magnitude-test/src/worker/localTestRegistry.ts +++ b/packages/magnitude-test/src/worker/localTestRegistry.ts @@ -1,6 +1,6 @@ import { TestFunction, TestGroup, TestOptions } from "@/discovery/types"; import cuid2 from "@paralleldrive/cuid2"; -import { getTestWorkerData, postToParent, testFunctions, messageEmitter, TestWorkerIncomingMessage, hooks, testRegistry, testPromptStack } from "./util"; +import { getTestWorkerData, postToParent, testFunctions, messageEmitter, TestWorkerIncomingMessage, hooks, testRegistry, testPromptStack, groupHooks } from "./util"; import { TestCaseAgent } from "@/agent"; import { TestResult, TestState, TestStateTracker } from "@/runner/state"; import { buildDefaultBrowserAgentOptions } from "magnitude-core"; @@ -46,6 +46,9 @@ let currentGroup: TestGroup | undefined; export function setCurrentGroup(group?: TestGroup) { currentGroup = group; } +export function getCurrentGroup(): TestGroup | undefined { + return currentGroup; +} export function currentGroupOptions(): TestOptions { return structuredClone(currentGroup?.options) ?? {}; } @@ -179,6 +182,16 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { throw error; } } + if (testMetadata.group && groupHooks[testMetadata.group]) { + for (const beforeEachHook of groupHooks[testMetadata.group].beforeEach) { + try { + await beforeEachHook(); + } catch (error) { + console.error(`Group beforeEach hook failed for test '${testMetadata.title}' in group '${testMetadata.group}':`, error); + throw error; + } + } + } pendingAfterEach.add(testId); await testFn(agent); diff --git a/packages/magnitude-test/src/worker/testDeclaration.ts b/packages/magnitude-test/src/worker/testDeclaration.ts index 5c974390..984f8150 100644 --- a/packages/magnitude-test/src/worker/testDeclaration.ts +++ b/packages/magnitude-test/src/worker/testDeclaration.ts @@ -1,7 +1,7 @@ import { TestDeclaration, TestOptions, TestFunction, TestGroupFunction } from '../discovery/types'; import { addProtocolIfMissing, processUrl } from '@/util'; -import { getTestWorkerData, hooks, TestHooks, testPromptStack } from '@/worker/util'; -import { currentGroupOptions, registerTest, setCurrentGroup } from '@/worker/localTestRegistry'; +import { getTestWorkerData, hooks, TestHooks, testPromptStack, groupHooks } from '@/worker/util'; +import { currentGroupOptions, registerTest, setCurrentGroup, getCurrentGroup } from '@/worker/localTestRegistry'; const workerData = getTestWorkerData(); @@ -81,7 +81,23 @@ function createHookRegistrar(kind: keyof TestHooks) { if (typeof fn !== "function") { throw new Error(`${kind} expects a function`); } - hooks[kind].push(fn); + + const group = getCurrentGroup(); + if (group) { + // Register as group-level hook + if (!groupHooks[group.name]) { + groupHooks[group.name] = { + beforeAll: [], + afterAll: [], + beforeEach: [], + afterEach: [], + }; + } + groupHooks[group.name][kind].push(fn); + } else { + // Register as file-level hook + hooks[kind].push(fn); + } }; } diff --git a/packages/magnitude-test/src/worker/util.ts b/packages/magnitude-test/src/worker/util.ts index 52255d8f..9abef918 100644 --- a/packages/magnitude-test/src/worker/util.ts +++ b/packages/magnitude-test/src/worker/util.ts @@ -12,6 +12,7 @@ declare global { var __magnitudeTestHooks: TestHooks | undefined; var __magnitudeTestPromptStack: Record | undefined; var __magnitudeTestRegistry: Map | undefined; + var __magnitudeGroupTestHooks: GroupTestHooks | undefined; } if (!globalThis.__magnitudeTestFunctions) { @@ -37,6 +38,9 @@ export type TestMetadata = { group?: string; }; +/** Group-level test hooks keyed by group name */ +export type GroupTestHooks = Record; + if (!globalThis.__magnitudeTestHooks) { globalThis.__magnitudeTestHooks = { beforeAll: [], @@ -57,6 +61,10 @@ if (!globalThis.__magnitudeTestRegistry) { } export const testRegistry = globalThis.__magnitudeTestRegistry; +if (!globalThis.__magnitudeGroupTestHooks) { + globalThis.__magnitudeGroupTestHooks = {}; +} +export const groupHooks = globalThis.__magnitudeGroupTestHooks; export type TestWorkerIncomingMessage = { type: "execute" testId: string; From 9140345df34cc870a5d6637b767a620ffa568349 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Tue, 29 Jul 2025 19:12:16 -0400 Subject: [PATCH 2/7] afterEach group hook execution --- .../src/worker/localTestRegistry.ts | 93 ++++++++++++------- packages/magnitude-test/src/worker/util.ts | 11 +-- 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/packages/magnitude-test/src/worker/localTestRegistry.ts b/packages/magnitude-test/src/worker/localTestRegistry.ts index 01ea7d53..43bb0017 100644 --- a/packages/magnitude-test/src/worker/localTestRegistry.ts +++ b/packages/magnitude-test/src/worker/localTestRegistry.ts @@ -1,4 +1,4 @@ -import { TestFunction, TestGroup, TestOptions } from "@/discovery/types"; +import { TestFunction, TestGroup, TestOptions, RegisteredTest } from "@/discovery/types"; import cuid2 from "@paralleldrive/cuid2"; import { getTestWorkerData, postToParent, testFunctions, messageEmitter, TestWorkerIncomingMessage, hooks, testRegistry, testPromptStack, groupHooks } from "./util"; import { TestCaseAgent } from "@/agent"; @@ -17,6 +17,7 @@ export function registerTest(testFn: TestFunction, title: string, url: string) { testFunctions.set(testId, testFn); testRegistry.set(testId, { + id: testId, title, url, filepath: workerData.relativeFilePath, @@ -39,7 +40,7 @@ let beforeAllExecuted = false; let beforeAllError: Error | null = null; let afterAllExecuted = false; let isShuttingDown = false; -let pendingAfterEach: Set = new Set(); +let pendingAfterEach: Map = new Map(); // No state reset is needed because each test file is run in a separate worker let currentGroup: TestGroup | undefined; @@ -53,37 +54,67 @@ export function currentGroupOptions(): TestOptions { return structuredClone(currentGroup?.options) ?? {}; } +async function executeAfterEachHooks(test: RegisteredTest) { + if (test.group && groupHooks[test.group]) { + for (const afterEachHook of groupHooks[test.group].afterEach) { + try { + await afterEachHook(); + } catch (error) { + console.error(`Group afterEach hook failed for test '${test.title}' in group '${test.group}':`, error); + throw error; + } + } + } + + for (const afterEachHook of hooks.afterEach) { + try { + await afterEachHook(); + } catch (error) { + console.error(`afterEach hook failed for test '${test.title}':`, error); + throw error; + } + } +} + messageEmitter.removeAllListeners('message'); messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { if (message.type === 'graceful_shutdown') { isShuttingDown = true; - if (pendingAfterEach.size > 0) { - try { - await Promise.all( - [...pendingAfterEach].map(async (_testId) => { - for (const afterEachHook of hooks.afterEach) { - await afterEachHook(); - } - }) - ); - } catch (error) { - console.error("afterEach hooks failed during graceful shutdown:", error); + try { + if (pendingAfterEach.size > 0) { + try { + await Promise.all( + [...pendingAfterEach.values()].map(async (test) => { + try { + await executeAfterEachHooks(test); + } catch (error) { + console.error(`afterEach hooks failed during graceful shutdown for test '${test.title}':`, error); + // Don't throw here - we want to continue with other tests + } + }) + ); + } catch (error) { + console.error("afterEach hooks failed during graceful shutdown:", error); + } } - } - if (!afterAllExecuted) { - try { - for (const afterAllHook of hooks.afterAll) { - await afterAllHook(); + if (!afterAllExecuted) { + try { + for (const afterAllHook of hooks.afterAll) { + await afterAllHook(); + } + afterAllExecuted = true; + } catch (error) { + console.error("afterAll hook failed during graceful shutdown:\n", error); } - afterAllExecuted = true; - } catch (error) { - console.error("afterAll hook failed during graceful shutdown:\n", error); } - } - postToParent({ type: 'graceful_shutdown_complete' }); + postToParent({ type: 'graceful_shutdown_complete' }); + } catch (error) { + console.error("Critical error during graceful shutdown:", error); + postToParent({ type: 'graceful_shutdown_complete' }); + } return; } @@ -192,20 +223,12 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { } } } - pendingAfterEach.add(testId); - + pendingAfterEach.set(testId, testMetadata); await testFn(agent); if (!isShuttingDown) { pendingAfterEach.delete(testId); - for (const afterEachHook of hooks.afterEach) { - try { - await afterEachHook(); - } catch (error) { - console.error(`afterEach hook failed for test '${testMetadata.title}':`, error); - throw error; - } - } + await executeAfterEachHooks(testMetadata); } finalState = { @@ -219,9 +242,7 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { if (!isShuttingDown) { pendingAfterEach.delete(testId); try { - for (const afterEachHook of hooks.afterEach) { - await afterEachHook(); - } + await executeAfterEachHooks(testMetadata); } catch (afterEachError) { console.error(`afterEach hook failed for failing test '${testMetadata.title}':`, afterEachError); const originalMessage = error instanceof Error ? error.message : String(error); diff --git a/packages/magnitude-test/src/worker/util.ts b/packages/magnitude-test/src/worker/util.ts index 9abef918..9ef7c403 100644 --- a/packages/magnitude-test/src/worker/util.ts +++ b/packages/magnitude-test/src/worker/util.ts @@ -11,7 +11,7 @@ declare global { var __magnitudeMessageEmitter: EventEmitter | undefined; var __magnitudeTestHooks: TestHooks | undefined; var __magnitudeTestPromptStack: Record | undefined; - var __magnitudeTestRegistry: Map | undefined; + var __magnitudeTestRegistry: Map | undefined; var __magnitudeGroupTestHooks: GroupTestHooks | undefined; } @@ -31,13 +31,6 @@ export type TestHooks = Record< (() => void | Promise)[] >; -export type TestMetadata = { - title: string; - url: string; - filepath: string; - group?: string; -}; - /** Group-level test hooks keyed by group name */ export type GroupTestHooks = Record; @@ -57,7 +50,7 @@ if (!globalThis.__magnitudeTestPromptStack) { export const testPromptStack = globalThis.__magnitudeTestPromptStack; if (!globalThis.__magnitudeTestRegistry) { - globalThis.__magnitudeTestRegistry = new Map(); + globalThis.__magnitudeTestRegistry = new Map(); } export const testRegistry = globalThis.__magnitudeTestRegistry; From dcac87142ee37d9c194a2df18735bf31bd2f7ca3 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Tue, 29 Jul 2025 21:27:16 -0400 Subject: [PATCH 3/7] Execute beforeAll and afterAll hooks registered in groups --- .../src/worker/localTestRegistry.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/magnitude-test/src/worker/localTestRegistry.ts b/packages/magnitude-test/src/worker/localTestRegistry.ts index 43bb0017..ed80d260 100644 --- a/packages/magnitude-test/src/worker/localTestRegistry.ts +++ b/packages/magnitude-test/src/worker/localTestRegistry.ts @@ -41,6 +41,8 @@ let beforeAllError: Error | null = null; let afterAllExecuted = false; let isShuttingDown = false; let pendingAfterEach: Map = new Map(); +let groupBeforeAllExecuted: Set = new Set(); +let groupBeforeAllErrors: Map = new Map(); // No state reset is needed because each test file is run in a separate worker let currentGroup: TestGroup | undefined; @@ -76,6 +78,33 @@ async function executeAfterEachHooks(test: RegisteredTest) { } } +async function executeGroupBeforeAllHooks(test: RegisteredTest) { + if (!test.group || !groupHooks[test.group]) { + return; + } + + if (groupBeforeAllExecuted.has(test.group)) { + const error = groupBeforeAllErrors.get(test.group); + if (error) { + throw new Error(`Group beforeAll hook failed for group '${test.group}': ${error.message}`); + } + return; + } + + groupBeforeAllExecuted.add(test.group); + + try { + for (const beforeAllHook of groupHooks[test.group].beforeAll) { + await beforeAllHook(); + } + } catch (error) { + const hookError = error instanceof Error ? error : new Error(String(error)); + console.error(`Group beforeAll hook failed for group '${test.group}':`, hookError); + groupBeforeAllErrors.set(test.group, hookError); + throw hookError; + } +} + messageEmitter.removeAllListeners('message'); messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { if (message.type === 'graceful_shutdown') { @@ -101,6 +130,18 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { if (!afterAllExecuted) { try { + for (const groupName of groupBeforeAllExecuted) { + if (groupHooks[groupName] && groupHooks[groupName].afterAll.length > 0) { + try { + for (const afterAllHook of groupHooks[groupName].afterAll) { + await afterAllHook(); + } + } catch (error) { + console.error(`Group afterAll hook failed during graceful shutdown for group '${groupName}':`, error); + } + } + } + for (const afterAllHook of hooks.afterAll) { await afterAllHook(); } @@ -205,6 +246,8 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { throw new Error(`beforeAll hook failed: ${beforeAllError.message}`); } + await executeGroupBeforeAllHooks(test); + for (const beforeEachHook of hooks.beforeEach) { try { await beforeEachHook(); From db348661075b98e83c2ca6e177d833210971ca10 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 29 Aug 2025 19:59:39 -0400 Subject: [PATCH 4/7] Group IDs --- .../magnitude-test/src/discovery/types.ts | 6 ++-- .../src/worker/localTestRegistry.ts | 35 +++++++++++++------ .../src/worker/testDeclaration.ts | 17 +++++---- packages/magnitude-test/src/worker/util.ts | 1 + 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/packages/magnitude-test/src/discovery/types.ts b/packages/magnitude-test/src/discovery/types.ts index 2263f2ac..3f7b1dfe 100644 --- a/packages/magnitude-test/src/discovery/types.ts +++ b/packages/magnitude-test/src/discovery/types.ts @@ -58,11 +58,12 @@ export type TestGroupFunction = () => void; export interface TestGroup { name: string; options?: TestOptions; + id?: string; } export interface TestGroupDeclaration { - (id: string, options: TestOptions, groupFn: TestGroupFunction): void; - (id: string, groupFn: TestGroupFunction): void; + (name: string, options: TestOptions, groupFn: TestGroupFunction): void; + (name: string, groupFn: TestGroupFunction): void; } export interface TestDeclaration { @@ -86,4 +87,5 @@ export interface RegisteredTest { // meta filepath: string, group?: string, + groupHierarchy?: Array<{ name: string; id?: string }>, } \ No newline at end of file diff --git a/packages/magnitude-test/src/worker/localTestRegistry.ts b/packages/magnitude-test/src/worker/localTestRegistry.ts index ed80d260..9918543a 100644 --- a/packages/magnitude-test/src/worker/localTestRegistry.ts +++ b/packages/magnitude-test/src/worker/localTestRegistry.ts @@ -1,10 +1,11 @@ import { TestFunction, TestGroup, TestOptions, RegisteredTest } from "@/discovery/types"; import cuid2 from "@paralleldrive/cuid2"; -import { getTestWorkerData, postToParent, testFunctions, messageEmitter, TestWorkerIncomingMessage, hooks, testRegistry, testPromptStack, groupHooks } from "./util"; +import { getTestWorkerData, postToParent, testFunctions, messageEmitter, TestWorkerIncomingMessage, hooks, groupHooks, testRegistry } from "./util"; import { TestCaseAgent } from "@/agent"; import { TestResult, TestState, TestStateTracker } from "@/runner/state"; import { buildDefaultBrowserAgentOptions } from "magnitude-core"; import { sendTelemetry } from "@/runner/telemetry"; +import { testPromptStack } from "./util"; // This module has to be separate so it only gets imported once after possible compilation by jiti. @@ -15,15 +16,16 @@ const generateId = cuid2.init({ length: 12 }); export function registerTest(testFn: TestFunction, title: string, url: string) { const testId = generateId(); testFunctions.set(testId, testFn); + const groupHierarchy = getCurrentGroupHierarchy().map(({ name, id }) => ({ name, id })); testRegistry.set(testId, { id: testId, title, url, filepath: workerData.relativeFilePath, - group: currentGroup?.name + group: getCurrentGroup()?.name, + groupHierarchy }); - postToParent({ type: 'registered', test: { @@ -31,7 +33,8 @@ export function registerTest(testFn: TestFunction, title: string, url: string) { title, url, filepath: workerData.relativeFilePath, - group: currentGroup?.name + group: getCurrentGroup()?.name, + groupHierarchy } }); } @@ -45,15 +48,27 @@ let groupBeforeAllExecuted: Set = new Set(); let groupBeforeAllErrors: Map = new Map(); // No state reset is needed because each test file is run in a separate worker -let currentGroup: TestGroup | undefined; -export function setCurrentGroup(group?: TestGroup) { - currentGroup = group; +let currentGroupStack: TestGroup[] = []; +export function pushCurrentGroup(group: TestGroup) { + currentGroupStack.push(group); +} +export function popCurrentGroup() { + currentGroupStack.pop(); } export function getCurrentGroup(): TestGroup | undefined { - return currentGroup; + return currentGroupStack.length > 0 ? currentGroupStack[currentGroupStack.length - 1] : undefined; +} +export function getCurrentGroupHierarchy(): TestGroup[] { + return [...currentGroupStack]; } export function currentGroupOptions(): TestOptions { - return structuredClone(currentGroup?.options) ?? {}; + let mergedOptions: TestOptions = {}; + for (const group of currentGroupStack) { + if (group.options) { + mergedOptions = { ...mergedOptions, ...group.options }; + } + } + return structuredClone(mergedOptions); } async function executeAfterEachHooks(test: RegisteredTest) { @@ -246,7 +261,7 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { throw new Error(`beforeAll hook failed: ${beforeAllError.message}`); } - await executeGroupBeforeAllHooks(test); + await executeGroupBeforeAllHooks(testMetadata); for (const beforeEachHook of hooks.beforeEach) { try { diff --git a/packages/magnitude-test/src/worker/testDeclaration.ts b/packages/magnitude-test/src/worker/testDeclaration.ts index 984f8150..1c28b322 100644 --- a/packages/magnitude-test/src/worker/testDeclaration.ts +++ b/packages/magnitude-test/src/worker/testDeclaration.ts @@ -1,10 +1,12 @@ import { TestDeclaration, TestOptions, TestFunction, TestGroupFunction } from '../discovery/types'; import { addProtocolIfMissing, processUrl } from '@/util'; import { getTestWorkerData, hooks, TestHooks, testPromptStack, groupHooks } from '@/worker/util'; -import { currentGroupOptions, registerTest, setCurrentGroup, getCurrentGroup } from '@/worker/localTestRegistry'; +import { currentGroupOptions, registerTest, pushCurrentGroup, popCurrentGroup, getCurrentGroup } from '@/worker/localTestRegistry'; +import cuid2 from "@paralleldrive/cuid2"; const workerData = getTestWorkerData(); +const genGroupId = cuid2.init({ length: 6 }); function testDecl( title: string, optionsOrTestFn: TestOptions | TestFunction, @@ -50,7 +52,7 @@ function testDecl( } testDecl.group = function ( - id: string, + name: string, optionsOrTestFn: TestOptions | TestGroupFunction, testFnOrNothing?: TestGroupFunction ): void { @@ -69,9 +71,12 @@ testDecl.group = function ( testFn = testFnOrNothing; } - setCurrentGroup({ name: id, options }); - testFn(); - setCurrentGroup(undefined); + pushCurrentGroup({ name, id: `grp${genGroupId()}`, options }); + try { + testFn(); + } finally { + popCurrentGroup(); + } } export const test = testDecl as TestDeclaration; @@ -104,4 +109,4 @@ function createHookRegistrar(kind: keyof TestHooks) { export const beforeAll = createHookRegistrar("beforeAll"); export const afterAll = createHookRegistrar("afterAll"); export const beforeEach = createHookRegistrar("beforeEach"); -export const afterEach = createHookRegistrar("afterEach"); +export const afterEach = createHookRegistrar("afterEach"); \ No newline at end of file diff --git a/packages/magnitude-test/src/worker/util.ts b/packages/magnitude-test/src/worker/util.ts index 9ef7c403..7f0fac26 100644 --- a/packages/magnitude-test/src/worker/util.ts +++ b/packages/magnitude-test/src/worker/util.ts @@ -31,6 +31,7 @@ export type TestHooks = Record< (() => void | Promise)[] >; + /** Group-level test hooks keyed by group name */ export type GroupTestHooks = Record; From b272ea03c12e36fda41264f02f35fd74b579a650 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 29 Aug 2025 20:38:04 -0400 Subject: [PATCH 5/7] Hierarchical group test hooks --- .../src/worker/localTestRegistry.ts | 105 ++++++++++++------ .../src/worker/testDeclaration.ts | 23 ++-- packages/magnitude-test/src/worker/util.ts | 22 +++- 3 files changed, 94 insertions(+), 56 deletions(-) diff --git a/packages/magnitude-test/src/worker/localTestRegistry.ts b/packages/magnitude-test/src/worker/localTestRegistry.ts index 9918543a..6bcbb840 100644 --- a/packages/magnitude-test/src/worker/localTestRegistry.ts +++ b/packages/magnitude-test/src/worker/localTestRegistry.ts @@ -1,11 +1,10 @@ import { TestFunction, TestGroup, TestOptions, RegisteredTest } from "@/discovery/types"; import cuid2 from "@paralleldrive/cuid2"; -import { getTestWorkerData, postToParent, testFunctions, messageEmitter, TestWorkerIncomingMessage, hooks, groupHooks, testRegistry } from "./util"; +import { getTestWorkerData, postToParent, testFunctions, messageEmitter, TestWorkerIncomingMessage, hooks, testRegistry, testPromptStack, getOrInitGroupHookSet } from "./util"; import { TestCaseAgent } from "@/agent"; import { TestResult, TestState, TestStateTracker } from "@/runner/state"; import { buildDefaultBrowserAgentOptions } from "magnitude-core"; import { sendTelemetry } from "@/runner/telemetry"; -import { testPromptStack } from "./util"; // This module has to be separate so it only gets imported once after possible compilation by jiti. @@ -13,6 +12,15 @@ const workerData = getTestWorkerData(); const generateId = cuid2.init({ length: 12 }); +/** Get all prefix keys for a hierarchy in outer → inner order */ +function keysForPrefixes(hierarchy: TestGroup[]): string[] { + const keys: string[] = []; + for (let i = 1; i <= hierarchy.length; i++) { + keys.push(hierarchy.slice(0, i).map(g => g.id).join('>')); + } + return keys; +} + export function registerTest(testFn: TestFunction, title: string, url: string) { const testId = generateId(); testFunctions.set(testId, testFn); @@ -46,6 +54,7 @@ let isShuttingDown = false; let pendingAfterEach: Map = new Map(); let groupBeforeAllExecuted: Set = new Set(); let groupBeforeAllErrors: Map = new Map(); +let executedBeforeAllOrder: string[] = []; // No state reset is needed because each test file is run in a separate worker let currentGroupStack: TestGroup[] = []; @@ -72,13 +81,19 @@ export function currentGroupOptions(): TestOptions { } async function executeAfterEachHooks(test: RegisteredTest) { - if (test.group && groupHooks[test.group]) { - for (const afterEachHook of groupHooks[test.group].afterEach) { - try { - await afterEachHook(); - } catch (error) { - console.error(`Group afterEach hook failed for test '${test.title}' in group '${test.group}':`, error); - throw error; + // Run group-level afterEach hooks in inner → outer order + if (test.groupHierarchy && test.groupHierarchy.length > 0) { + const prefixKeys = keysForPrefixes(test.groupHierarchy); + for (const key of prefixKeys.reverse()) { + const hookSet = getOrInitGroupHookSet(key); + for (const afterEachHook of hookSet.afterEach) { + try { + await afterEachHook(); + } catch (error) { + const groupNames = test.groupHierarchy.map(g => g.name).join(' > '); + console.error(`Group afterEach hook failed for test '${test.title}' in hierarchy '${groupNames}':`, error); + throw error; + } } } } @@ -94,29 +109,37 @@ async function executeAfterEachHooks(test: RegisteredTest) { } async function executeGroupBeforeAllHooks(test: RegisteredTest) { - if (!test.group || !groupHooks[test.group]) { + if (!test.groupHierarchy || test.groupHierarchy.length === 0) { return; } - if (groupBeforeAllExecuted.has(test.group)) { - const error = groupBeforeAllErrors.get(test.group); - if (error) { - throw new Error(`Group beforeAll hook failed for group '${test.group}': ${error.message}`); + const prefixKeys = keysForPrefixes(test.groupHierarchy); + + for (const key of prefixKeys) { + if (groupBeforeAllExecuted.has(key)) { + const error = groupBeforeAllErrors.get(key); + if (error) { + const groupNames = test.groupHierarchy.map(g => g.name).join(' > '); + throw new Error(`Group beforeAll hook failed for hierarchy '${groupNames}': ${error.message}`); + } + continue; } - return; - } - groupBeforeAllExecuted.add(test.group); + groupBeforeAllExecuted.add(key); + executedBeforeAllOrder.push(key); - try { - for (const beforeAllHook of groupHooks[test.group].beforeAll) { - await beforeAllHook(); + try { + const hookSet = getOrInitGroupHookSet(key); + for (const beforeAllHook of hookSet.beforeAll) { + await beforeAllHook(); + } + } catch (error) { + const hookError = error instanceof Error ? error : new Error(String(error)); + const groupNames = test.groupHierarchy.map(g => g.name).join(' > '); + console.error(`Group beforeAll hook failed for hierarchy '${groupNames}':`, hookError); + groupBeforeAllErrors.set(key, hookError); + throw hookError; } - } catch (error) { - const hookError = error instanceof Error ? error : new Error(String(error)); - console.error(`Group beforeAll hook failed for group '${test.group}':`, hookError); - groupBeforeAllErrors.set(test.group, hookError); - throw hookError; } } @@ -145,14 +168,16 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { if (!afterAllExecuted) { try { - for (const groupName of groupBeforeAllExecuted) { - if (groupHooks[groupName] && groupHooks[groupName].afterAll.length > 0) { + // Run group afterAll hooks in reverse execution order (inner → outer) + for (const key of executedBeforeAllOrder.reverse()) { + const hookSet = getOrInitGroupHookSet(key); + if (hookSet.afterAll.length > 0) { try { - for (const afterAllHook of groupHooks[groupName].afterAll) { + for (const afterAllHook of hookSet.afterAll) { await afterAllHook(); } } catch (error) { - console.error(`Group afterAll hook failed during graceful shutdown for group '${groupName}':`, error); + console.error(`Group afterAll hook failed during graceful shutdown for hierarchy key '${key}':`, error); } } } @@ -243,6 +268,7 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { let finalState: TestState; let finalResult: TestResult; + try { if (!beforeAllExecuted && hooks.beforeAll.length > 0) { try { @@ -271,13 +297,20 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { throw error; } } - if (testMetadata.group && groupHooks[testMetadata.group]) { - for (const beforeEachHook of groupHooks[testMetadata.group].beforeEach) { - try { - await beforeEachHook(); - } catch (error) { - console.error(`Group beforeEach hook failed for test '${testMetadata.title}' in group '${testMetadata.group}':`, error); - throw error; + + // Run group-level beforeEach hooks in outer → inner order + if (testMetadata.groupHierarchy && testMetadata.groupHierarchy.length > 0) { + const prefixKeys = keysForPrefixes(testMetadata.groupHierarchy); + for (const key of prefixKeys) { + const hookSet = getOrInitGroupHookSet(key); + for (const beforeEachHook of hookSet.beforeEach) { + try { + await beforeEachHook(); + } catch (error) { + const groupNames = testMetadata.groupHierarchy.map(g => g.name).join(' > '); + console.error(`Group beforeEach hook failed for test '${testMetadata.title}' in hierarchy '${groupNames}':`, error); + throw error; + } } } } diff --git a/packages/magnitude-test/src/worker/testDeclaration.ts b/packages/magnitude-test/src/worker/testDeclaration.ts index 1c28b322..c83bc49a 100644 --- a/packages/magnitude-test/src/worker/testDeclaration.ts +++ b/packages/magnitude-test/src/worker/testDeclaration.ts @@ -1,7 +1,7 @@ import { TestDeclaration, TestOptions, TestFunction, TestGroupFunction } from '../discovery/types'; import { addProtocolIfMissing, processUrl } from '@/util'; -import { getTestWorkerData, hooks, TestHooks, testPromptStack, groupHooks } from '@/worker/util'; -import { currentGroupOptions, registerTest, pushCurrentGroup, popCurrentGroup, getCurrentGroup } from '@/worker/localTestRegistry'; +import { getTestWorkerData, hooks, TestHooks, testPromptStack, getOrInitGroupHookSet } from '@/worker/util'; +import { currentGroupOptions, registerTest, pushCurrentGroup, popCurrentGroup, getCurrentGroupHierarchy } from '@/worker/localTestRegistry'; import cuid2 from "@paralleldrive/cuid2"; const workerData = getTestWorkerData(); @@ -87,18 +87,11 @@ function createHookRegistrar(kind: keyof TestHooks) { throw new Error(`${kind} expects a function`); } - const group = getCurrentGroup(); - if (group) { - // Register as group-level hook - if (!groupHooks[group.name]) { - groupHooks[group.name] = { - beforeAll: [], - afterAll: [], - beforeEach: [], - afterEach: [], - }; - } - groupHooks[group.name][kind].push(fn); + const hierarchy = getCurrentGroupHierarchy(); + if (hierarchy.length > 0) { + const key = hierarchy.map(g => g.id).join('>'); + const hookSet = getOrInitGroupHookSet(key); + hookSet[kind].push(fn); } else { // Register as file-level hook hooks[kind].push(fn); @@ -109,4 +102,4 @@ function createHookRegistrar(kind: keyof TestHooks) { export const beforeAll = createHookRegistrar("beforeAll"); export const afterAll = createHookRegistrar("afterAll"); export const beforeEach = createHookRegistrar("beforeEach"); -export const afterEach = createHookRegistrar("afterEach"); \ No newline at end of file +export const afterEach = createHookRegistrar("afterEach"); diff --git a/packages/magnitude-test/src/worker/util.ts b/packages/magnitude-test/src/worker/util.ts index 7f0fac26..f17fdc60 100644 --- a/packages/magnitude-test/src/worker/util.ts +++ b/packages/magnitude-test/src/worker/util.ts @@ -31,10 +31,6 @@ export type TestHooks = Record< (() => void | Promise)[] >; - -/** Group-level test hooks keyed by group name */ -export type GroupTestHooks = Record; - if (!globalThis.__magnitudeTestHooks) { globalThis.__magnitudeTestHooks = { beforeAll: [], @@ -45,6 +41,11 @@ if (!globalThis.__magnitudeTestHooks) { } export const hooks = globalThis.__magnitudeTestHooks; + + +/** Group-level test hooks keyed by hierarchy key */ +export type GroupTestHooks = Record; + if (!globalThis.__magnitudeTestPromptStack) { globalThis.__magnitudeTestPromptStack = {}; } @@ -54,11 +55,22 @@ if (!globalThis.__magnitudeTestRegistry) { globalThis.__magnitudeTestRegistry = new Map(); } export const testRegistry = globalThis.__magnitudeTestRegistry; - if (!globalThis.__magnitudeGroupTestHooks) { globalThis.__magnitudeGroupTestHooks = {}; } export const groupHooks = globalThis.__magnitudeGroupTestHooks; +/** Helper to get or initialize hook set for a hierarchy key */ +export function getOrInitGroupHookSet(key: string): TestHooks { + if (!groupHooks[key]) { + groupHooks[key] = { + beforeAll: [], + afterAll: [], + beforeEach: [], + afterEach: [], + }; + } + return groupHooks[key]; +} export type TestWorkerIncomingMessage = { type: "execute" testId: string; From 4813ccbc90a6c967e47b578a384879ad6152e74b Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Sat, 30 Aug 2025 00:20:49 -0400 Subject: [PATCH 6/7] Render nested tests nicely --- .../magnitude-test/src/term-app/uiRenderer.ts | 210 ++++++++++-------- 1 file changed, 116 insertions(+), 94 deletions(-) diff --git a/packages/magnitude-test/src/term-app/uiRenderer.ts b/packages/magnitude-test/src/term-app/uiRenderer.ts index 6ade2f52..0b2b0bb7 100644 --- a/packages/magnitude-test/src/term-app/uiRenderer.ts +++ b/packages/magnitude-test/src/term-app/uiRenderer.ts @@ -130,67 +130,96 @@ export function generateTestString(test: RegisteredTest, state: RunnerTestState, return output; } -// Helper function to group tests for display -function groupRegisteredTestsForDisplay(tests: RegisteredTest[]): - Record }> { - const files: Record }> = {}; +// Tree node structure for hierarchical display +interface TreeNode { + tests: RegisteredTest[]; + children: Record; +} + +// Helper function to build a tree structure from hierarchical tests +function buildTestTree(tests: RegisteredTest[]): TreeNode { + const root: TreeNode = { tests: [], children: {} }; + for (const test of tests) { - if (!files[test.filepath]) { - files[test.filepath] = { ungrouped: [], groups: {} }; - } - if (test.group) { - if (!files[test.filepath].groups[test.group]) { - files[test.filepath].groups[test.group] = []; - } - files[test.filepath].groups[test.group].push(test); + if (!test.groupHierarchy || test.groupHierarchy.length === 0) { + // Tests without hierarchy go in the root + root.tests.push(test); } else { - files[test.filepath].ungrouped.push(test); + // Build the tree path for hierarchical tests + let currentNode = root; + for (const group of test.groupHierarchy) { + if (!currentNode.children[group.name]) { + currentNode.children[group.name] = { tests: [], children: {} }; + } + currentNode = currentNode.children[group.name]; + } + currentNode.tests.push(test); } } - return files; + + return root; +} + +// Helper function to group tests by file and build trees +function groupTestsByFile(tests: RegisteredTest[]): Record { + const files: Record = {}; + + // Group tests by file first + for (const test of tests) { + if (!files[test.filepath]) { + files[test.filepath] = []; + } + files[test.filepath].push(test); + } + + // Build tree for each file + const fileTrees: Record = {}; + for (const [filepath, fileTests] of Object.entries(files)) { + fileTrees[filepath] = buildTestTree(fileTests); + } + + return fileTrees; } +// Helper function to recursively generate tree display +function generateTreeDisplay(node: TreeNode, indent: number, output: string[]): void { + // Display tests at this level + for (const test of node.tests) { + const state = currentTestStates[test.id]; + if (state) { + const testLines = generateTestString(test, state, indent); + output.push(...testLines); + } + } + + // Display child groups + for (const [groupName, childNode] of Object.entries(node.children)) { + const groupHeader = `${ANSI_BRIGHT_BLUE}${ANSI_BOLD}↳ ${groupName}${ANSI_RESET}`; + output.push(UI_LEFT_PADDING + ' '.repeat(indent) + groupHeader); + + // Recursively display child node with increased indent + generateTreeDisplay(childNode, indent + 2, output); + } +} + /** * Generate the test list portion of the UI */ export function generateTestListString(): string[] { const output: string[] = []; const fileIndent = 0; - const groupIndent = fileIndent + 2; - const testBaseIndent = groupIndent; - const groupedDisplayTests = groupRegisteredTestsForDisplay(allRegisteredTests); + const fileTrees = groupTestsByFile(allRegisteredTests); - for (const [filepath, { ungrouped, groups }] of Object.entries(groupedDisplayTests)) { + for (const [filepath, treeRoot] of Object.entries(fileTrees)) { const fileHeader = `${ANSI_BRIGHT_BLUE}${ANSI_BOLD}☰ ${filepath}${ANSI_RESET}`; output.push(UI_LEFT_PADDING + ' '.repeat(fileIndent) + fileHeader); - if (ungrouped.length > 0) { - for (const test of ungrouped) { - const state = currentTestStates[test.id]; - if (state) { - const testLines = generateTestString(test, state, testBaseIndent); - output.push(...testLines); - } - } - } - - if (Object.entries(groups).length > 0) { - for (const [groupName, groupTests] of Object.entries(groups)) { - const groupHeader = `${ANSI_BRIGHT_BLUE}${ANSI_BOLD}↳ ${groupName}${ANSI_RESET}`; - output.push(UI_LEFT_PADDING + ' '.repeat(groupIndent) + groupHeader); + // Generate tree display starting from root + generateTreeDisplay(treeRoot, fileIndent + 2, output); - for (const test of groupTests) { - const state = currentTestStates[test.id]; - if (state) { - const testLines = generateTestString(test, state, testBaseIndent + 2); - output.push(...testLines); - } - } - } - } - output.push(UI_LEFT_PADDING); // Blank line between files/main groups + output.push(UI_LEFT_PADDING); // Blank line between files } return output; } @@ -204,7 +233,7 @@ export function generateSummaryString(): string[] { let totalOutputTokens = 0; const statusCounts = { pending: 0, running: 0, passed: 0, failed: 0, cancelled: 0, total: 0 }; const failuresWithContext: { filepath: string; groupName?: string; testTitle: string; failure: TestFailure }[] = []; - + const testContextMap = new Map(); allRegisteredTests.forEach(test => { testContextMap.set(test.id, { filepath: test.filepath, groupName: test.group, testTitle: test.title }); @@ -241,7 +270,7 @@ export function generateSummaryString(): string[] { costDescription = ` (\$${cost.toFixed(2)})`; } let tokenText = `${ANSI_GRAY}tokens: ${totalInputTokens} in, ${totalOutputTokens} out${costDescription}${ANSI_RESET}`; - + output.push(UI_LEFT_PADDING + statusLine.trimEnd() + (statusLine && tokenText ? ' ' : '') + tokenText.trimStart()); if (hasFailures) { @@ -257,60 +286,53 @@ export function generateSummaryString(): string[] { return output; } +// Helper function to calculate tree height recursively +function calculateTreeHeight(node: TreeNode, testStates: AllTestStates, indent: number): number { + let height = 0; + + // Count tests at this level + for (const test of node.tests) { + const state = testStates[test.id]; + if (state) { + height++; // Test title line + if (state.stepsAndChecks) { + state.stepsAndChecks.forEach((item: RunnerStepDescriptor | RunnerCheckDescriptor) => { + height++; // Item description line + if (item.variant === 'step') { + if (renderSettings.showActions) { + height += item.actions?.length ?? 0; + } + if (renderSettings.showThoughts) { + height += item.thoughts?.length ?? 0; + } + } + }); + } + if (state.failure) { + height++; // generateFailureString returns 1 line + } + } + } + + // Count child groups recursively + for (const [groupName, childNode] of Object.entries(node.children)) { + height++; // Group header line + height += calculateTreeHeight(childNode, testStates, indent + 2); + } + + return height; +} + /** * Calculate the height needed for the test list (now just line count) */ export function calculateTestListHeight(tests: RegisteredTest[], testStates: AllTestStates): number { let height = 0; - const groupedDisplayTests = groupRegisteredTestsForDisplay(tests); - - const addStepsAndChecksHeight = (state: RunnerTestState) => { - if (state.stepsAndChecks) { - state.stepsAndChecks.forEach((item: RunnerStepDescriptor | RunnerCheckDescriptor) => { - height++; // Item description line - if (item.variant === 'step') { - if (renderSettings.showActions) { - height += item.actions?.length ?? 0; - } - if (renderSettings.showThoughts) { - height += item.thoughts?.length ?? 0; - } - } - }); - } - }; + const fileTrees = groupTestsByFile(tests); - for (const [filepath, { ungrouped, groups }] of Object.entries(groupedDisplayTests)) { + for (const [filepath, treeRoot] of Object.entries(fileTrees)) { height++; // File header line - - if (ungrouped.length > 0) { - for (const test of ungrouped) { - const state = testStates[test.id]; - if (state) { - height++; // Test title line - addStepsAndChecksHeight(state); - if (state.failure) { - height++; // generateFailureString returns 1 line - } - } - } - } - - if (Object.entries(groups).length > 0) { - for (const [groupName, groupTests] of Object.entries(groups)) { - height++; // Group header line - for (const test of groupTests) { - const state = testStates[test.id]; - if (state) { - height++; // Test title line - addStepsAndChecksHeight(state); - if (state.failure) { - height++; // generateFailureString returns 1 line - } - } - } - } - } + height += calculateTreeHeight(treeRoot, testStates, 2); // Start with indent of 2 height++; // Blank line between files } return height; @@ -347,9 +369,9 @@ export function redraw() { let summaryLineCount = calculateSummaryHeight(currentTestStates); if (Object.values(currentTestStates).length === 0) { // No tests, no summary summaryLineCount = 0; - testListLineCount = 0; + testListLineCount = 0; } - + const outputLines: string[] = []; // outputLines.push(''); // Initial blank line for spacing from prompt - REMOVED @@ -367,7 +389,7 @@ export function redraw() { } const frameContent = outputLines.join('\n'); - + logUpdate.clear(); // Clear previous output before drawing new frame logUpdate(frameContent); From 12e5a86d7bde3510e0c82f54e5a2fdcd4f6986fd Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Tue, 29 Jul 2025 21:54:43 -0400 Subject: [PATCH 7/7] Changeset --- .changeset/neat-birds-beam.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/neat-birds-beam.md diff --git a/.changeset/neat-birds-beam.md b/.changeset/neat-birds-beam.md new file mode 100644 index 00000000..58384b8a --- /dev/null +++ b/.changeset/neat-birds-beam.md @@ -0,0 +1,5 @@ +--- +"magnitude-test": patch +--- + +beforeAll, beforeEach, afterEach, afterAll hooks may now be registered within groups to scope them only to tests within the group. For the time being, afterAll hooks regardless of group will only run after all tests in a module are complete. This may change in the future.