From 1bf4f13300ae811e87038e3fdd96cf96446ef2a0 Mon Sep 17 00:00:00 2001 From: ashutosh-rath02 Date: Wed, 16 Jul 2025 17:29:03 +0530 Subject: [PATCH] feat: add HTML report export to CLI with agent thoughts and modular template --- packages/magnitude-test/src/cli.ts | 14 +++- .../magnitude-test/src/runner/htmlReport.ts | 79 +++++++++++++++++++ packages/magnitude-test/src/runner/state.ts | 24 ++++-- .../src/runner/testSuiteRunner.ts | 26 ++++-- .../src/worker/localTestRegistry.ts | 3 +- 5 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 packages/magnitude-test/src/runner/htmlReport.ts diff --git a/packages/magnitude-test/src/cli.ts b/packages/magnitude-test/src/cli.ts index 676f2418..a77a3378 100644 --- a/packages/magnitude-test/src/cli.ts +++ b/packages/magnitude-test/src/cli.ts @@ -23,6 +23,7 @@ import { TermAppRenderer } from '@/term-app'; // Import TermAppRenderer // Removed import { initializeUI, updateUI, cleanupUI } from '@/term-app'; import { startWebServers, stopWebServers } from './webServer'; import chalk from 'chalk'; +import { renderHtmlReport } from './runner/htmlReport'; interface CliOptions { workers?: number; @@ -143,8 +144,9 @@ program .option('-w, --workers ', 'number of parallel workers for test execution', '1') .option('-p, --plain', 'disable pretty output and print lines instead') .option('-d, --debug', 'enable debug logs') + .option('--output-html ', 'write test results and agent history to an HTML file') // Changed action signature from (filters, options) to (filter, options) - .action(async (filter, options: CliOptions) => { + .action(async (filter, options: CliOptions & { outputHtml?: string }) => { dotenv.config(); let logLevel: string; @@ -269,7 +271,15 @@ program } try { - const overallSuccess = await testSuiteRunner.runTests(); + const { success: overallSuccess, results: testResults } = await testSuiteRunner.runTests(); + + // HTML report export + if (options.outputHtml) { + const html = renderHtmlReport(testResults); + require('fs').writeFileSync(options.outputHtml, html, 'utf-8'); + console.log(`\nWrote HTML report to ${options.outputHtml}`); + } + process.exit(overallSuccess ? 0 : 1); } catch (error) { logger.error("Test suite execution failed:", error); diff --git a/packages/magnitude-test/src/runner/htmlReport.ts b/packages/magnitude-test/src/runner/htmlReport.ts new file mode 100644 index 00000000..1d81eb84 --- /dev/null +++ b/packages/magnitude-test/src/runner/htmlReport.ts @@ -0,0 +1,79 @@ +// HTML report generation utility for Magnitude test results + +interface HtmlReportTest { + test: any; + result: any; +} + +export function renderHtmlReport(testResults: HtmlReportTest[]): string { + const summary = { + passed: testResults.filter(({ result }) => result.passed).length, + failed: testResults.filter(({ result }) => !result.passed).length, + total: testResults.length, + }; + let html = ` + + + + Magnitude Test Report + + + +

Magnitude Test Report

+
+
Total: ${summary.total}
+
Passed: ${summary.passed}
+
Failed: ${summary.failed}
+
+
+`; + for (const { test, result } of testResults) { + html += `
+
${test.title}
+
URL: ${test.url}
+
${result.passed ? 'PASSED' : 'FAILED'}
+ ${!result.passed && result.failure ? `
Error: ${result.failure.message}
` : ''} +`; + if (result.state && result.state.stepsAndChecks) { + html += `
Steps & Checks:
`; + for (const item of result.state.stepsAndChecks) { + if (item.variant === 'step') { + html += `
Step: ${item.description} [${item.status}]
`; + if (item.thoughts && item.thoughts.length > 0) { + html += `
Thoughts:
    `; + for (const thought of item.thoughts) { + html += `
  • ${thought}
    • `; + } + html += `
`; + } + if (item.actions && item.actions.length > 0) { + for (const action of item.actions) { + html += `
- ${action.pretty}
`; + } + } + } else if (item.variant === 'check') { + html += `
Check: ${item.description} [${item.status}]
`; + } + } + } else { + html += `
No step/check data available.
`; + } + html += `
`; + } + html += `
\n\n`; + return html; +} \ No newline at end of file diff --git a/packages/magnitude-test/src/runner/state.ts b/packages/magnitude-test/src/runner/state.ts index a81cd871..ed77e441 100644 --- a/packages/magnitude-test/src/runner/state.ts +++ b/packages/magnitude-test/src/runner/state.ts @@ -12,7 +12,8 @@ export interface StepDescriptor { description: string, actions: ActionDescriptor[], //actions: Action[] - status: 'pending' | 'running' | 'passed' | 'failed' | 'cancelled' + status: 'pending' | 'running' | 'passed' | 'failed' | 'cancelled', + thoughts?: string[] } export interface CheckDescriptor { @@ -52,12 +53,15 @@ export interface TestState { failure?: TestFailure } -export type TestResult = { - passed: true -} | { - passed: false - failure: TestFailure +export type TestResult = { + passed: true; + state?: TestState } + | { + passed: false; + failure: TestFailure; + state?: TestState +}; export interface TestFailure { message: string @@ -106,6 +110,7 @@ export class TestStateTracker { this.agent.checkEvents.on('checkStarted', this.onCheckStarted, this); this.agent.checkEvents.on('checkDone', this.onCheckDone, this); + this.agent.events.on('thought', this.onThought, this); // this.agent.events.on('action', this.onAction, this); @@ -217,6 +222,13 @@ export class TestStateTracker { this.events.emit('stateChanged', this.state); } + onThought(thought: string) { + if (this.lastStepOrCheck && this.lastStepOrCheck.variant === 'step') { + if (!this.lastStepOrCheck.thoughts) this.lastStepOrCheck.thoughts = []; + this.lastStepOrCheck.thoughts.push(thought); + } + } + // onStepStart(description: string) { // const stepDescriptor: StepDescriptor = { // variant: 'step', diff --git a/packages/magnitude-test/src/runner/testSuiteRunner.ts b/packages/magnitude-test/src/runner/testSuiteRunner.ts index 23a0460b..908a5cc2 100644 --- a/packages/magnitude-test/src/runner/testSuiteRunner.ts +++ b/packages/magnitude-test/src/runner/testSuiteRunner.ts @@ -23,6 +23,7 @@ export class TestSuiteRunner { private tests: RegisteredTest[]; private executors: Map = new Map(); + private lastTestResults: Array<{ test: RegisteredTest, result: TestResult }> = []; constructor( config: TestSuiteRunnerConfig @@ -87,7 +88,13 @@ export class TestSuiteRunner { } } - async runTests(): Promise { + async runTests(): Promise<{ + success: boolean, + results: Array<{ + test: RegisteredTest, + result: TestResult + }> + }> { if (!this.tests) throw new Error('No tests were registered'); this.renderer = this.runnerConfig.createRenderer(this.tests); this.renderer.start?.(); @@ -111,17 +118,20 @@ export class TestSuiteRunner { }); let overallSuccess = true; + let results: Array<{ test: RegisteredTest, result: TestResult }> = []; try { const poolResult: WorkerPoolResult = await workerPool.runTasks( taskFunctions, (taskOutcome: TestResult) => !taskOutcome.passed ); - for (const result of poolResult.results) { + for (let i = 0; i < poolResult.results.length; i++) { + const result = poolResult.results[i]; + const test = this.tests[i]; if (result === undefined || !result.passed) { overallSuccess = false; - break; } + results.push({ test, result: result ?? { passed: false, failure: { message: 'No result' } } }); } if (!poolResult.completed) { // If pool aborted for any reason (incl. a task failure) overallSuccess = false; @@ -130,9 +140,13 @@ export class TestSuiteRunner { } catch (error) { overallSuccess = false; } + this.lastTestResults = results; this.renderer.stop?.(); - return overallSuccess; + return { success: overallSuccess, results }; + } + public getLastTestResults() { + return this.lastTestResults; } } @@ -176,7 +190,7 @@ const createNodeTestWorker: CreateTestWorker = async (workerData) => res(msg.result); } else if (msg.type === "test_error") { worker.off("message", messageHandler); - rej(new Error(msg.error)); + res({ passed: false, failure: { message: msg.error } }); } else if (msg.type === "test_state_change") { onStateChange(msg.state); } @@ -269,7 +283,7 @@ const createBunTestWorker: CreateTestWorker = async (workerData) => res(msg.result); } else if (msg.type === "test_error") { emit.off('message', messageHandler); - rej(new Error(msg.error)); + res({ passed: false, failure: { message: msg.error } }); } else if (msg.type === "test_state_change") { onStateChange(msg.state); } diff --git a/packages/magnitude-test/src/worker/localTestRegistry.ts b/packages/magnitude-test/src/worker/localTestRegistry.ts index 9de42917..1fbaad1b 100644 --- a/packages/magnitude-test/src/worker/localTestRegistry.ts +++ b/packages/magnitude-test/src/worker/localTestRegistry.ts @@ -121,8 +121,7 @@ messageEmitter.on('message', async (message: TestWorkerIncomingMessage) => { postToParent({ type: 'test_result', testId: test.id, - result: finalResult ?? - { passed: false, failure: { message: "Test result doesn't exist" } }, + result: { ...finalResult, state: finalState } }); } catch (error) { postToParent({