diff --git a/src/index.ts b/src/index.ts index b1e0dba..3b78b46 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,11 +39,13 @@ program .option('--scenario ', 'Scenario to test') .option('--suite ', 'Run a suite of tests in parallel (e.g., "auth")') .option('--timeout ', 'Timeout in milliseconds', '30000') + .option('-o, --output-dir ', 'Save results to this directory') .option('--verbose', 'Show verbose output') .action(async (options) => { try { const timeout = parseInt(options.timeout, 10); const verbose = options.verbose ?? false; + const outputDir = options.outputDir; // Handle suite mode if (options.suite) { @@ -78,7 +80,8 @@ program const result = await runConformanceTest( options.command, scenarioName, - timeout + timeout, + outputDir ); return { scenario: scenarioName, @@ -163,7 +166,7 @@ program // If no command provided, run in interactive mode if (!validated.command) { - await runInteractiveMode(validated.scenario, verbose); + await runInteractiveMode(validated.scenario, verbose, outputDir); process.exit(0); } @@ -171,7 +174,8 @@ program const result = await runConformanceTest( validated.command, validated.scenario, - timeout + timeout, + outputDir ); const { overallFailure } = printClientResults( @@ -209,6 +213,7 @@ program 'Suite to run: "active" (default, excludes pending), "all", or "pending"', 'active' ) + .option('-o, --output-dir ', 'Save results to this directory') .option('--verbose', 'Show verbose output (JSON instead of pretty print)') .action(async (options) => { try { @@ -216,12 +221,14 @@ program const validated = ServerOptionsSchema.parse(options); const verbose = options.verbose ?? false; + const outputDir = options.outputDir; // If a single scenario is specified, run just that one if (validated.scenario) { const result = await runServerConformanceTest( validated.url, - validated.scenario + validated.scenario, + outputDir ); const { failed } = printServerResults( @@ -259,7 +266,8 @@ program try { const result = await runServerConformanceTest( validated.url, - scenarioName + scenarioName, + outputDir ); allResults.push({ scenario: scenarioName, checks: result.checks }); } catch (error) { diff --git a/src/runner/client.ts b/src/runner/client.ts index 7db5461..27a8b6e 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -3,7 +3,7 @@ import { promises as fs } from 'fs'; import path from 'path'; import { ConformanceCheck } from '../types'; import { getScenario } from '../scenarios'; -import { ensureResultsDir, createResultDir, formatPrettyChecks } from './utils'; +import { createResultDir, formatPrettyChecks } from './utils'; export interface ClientExecutionResult { exitCode: number; @@ -91,15 +91,19 @@ async function executeClient( export async function runConformanceTest( clientCommand: string, scenarioName: string, - timeout: number = 30000 + timeout: number = 30000, + outputDir?: string ): Promise<{ checks: ConformanceCheck[]; clientOutput: ClientExecutionResult; - resultDir: string; + resultDir?: string; }> { - await ensureResultsDir(); - const resultDir = createResultDir(scenarioName); - await fs.mkdir(resultDir, { recursive: true }); + let resultDir: string | undefined; + + if (outputDir) { + resultDir = createResultDir(outputDir, scenarioName); + await fs.mkdir(resultDir, { recursive: true }); + } // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; @@ -138,16 +142,24 @@ export async function runConformanceTest( const checks = scenario.getChecks(); - await fs.writeFile( - path.join(resultDir, 'checks.json'), - JSON.stringify(checks, null, 2) - ); + if (resultDir) { + await fs.writeFile( + path.join(resultDir, 'checks.json'), + JSON.stringify(checks, null, 2) + ); - await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout); + await fs.writeFile( + path.join(resultDir, 'stdout.txt'), + clientOutput.stdout + ); - await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr); + await fs.writeFile( + path.join(resultDir, 'stderr.txt'), + clientOutput.stderr + ); - console.error(`Results saved to ${resultDir}`); + console.error(`Results saved to ${resultDir}`); + } return { checks, @@ -244,11 +256,15 @@ export function printClientResults( export async function runInteractiveMode( scenarioName: string, - verbose: boolean = false + verbose: boolean = false, + outputDir?: string ): Promise { - await ensureResultsDir(); - const resultDir = createResultDir(scenarioName); - await fs.mkdir(resultDir, { recursive: true }); + let resultDir: string | undefined; + + if (outputDir) { + resultDir = createResultDir(outputDir, scenarioName); + await fs.mkdir(resultDir, { recursive: true }); + } // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; @@ -257,23 +273,29 @@ export async function runInteractiveMode( const urls = await scenario.start(); console.log(`Server URL: ${urls.serverUrl}`); - console.log('Press Ctrl+C to stop and save checks...'); + console.log('Press Ctrl+C to stop...'); const handleShutdown = async () => { console.log('\nShutting down...'); const checks = scenario.getChecks(); - await fs.writeFile( - path.join(resultDir, 'checks.json'), - JSON.stringify(checks, null, 2) - ); + + if (resultDir) { + await fs.writeFile( + path.join(resultDir, 'checks.json'), + JSON.stringify(checks, null, 2) + ); + } if (verbose) { console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`); } else { console.log(`\nChecks:\n${formatPrettyChecks(checks)}`); } - console.log(`\nChecks saved to ${resultDir}/checks.json`); + + if (resultDir) { + console.log(`\nChecks saved to ${resultDir}/checks.json`); + } await scenario.stop(); process.exit(0); diff --git a/src/runner/index.ts b/src/runner/index.ts index d48a291..a39a86c 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -15,7 +15,6 @@ export { // Export utilities export { - ensureResultsDir, createResultDir, formatPrettyChecks, getStatusColor, diff --git a/src/runner/server.ts b/src/runner/server.ts index 18c8254..a7d8449 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'fs'; import path from 'path'; import { ConformanceCheck } from '../types'; import { getClientScenario } from '../scenarios'; -import { ensureResultsDir, createResultDir, formatPrettyChecks } from './utils'; +import { createResultDir, formatPrettyChecks } from './utils'; /** * Format markdown-style text for terminal output using ANSI codes @@ -19,15 +19,19 @@ function formatMarkdown(text: string): string { export async function runServerConformanceTest( serverUrl: string, - scenarioName: string + scenarioName: string, + outputDir?: string ): Promise<{ checks: ConformanceCheck[]; - resultDir: string; + resultDir?: string; scenarioDescription: string; }> { - await ensureResultsDir(); - const resultDir = createResultDir(scenarioName, 'server'); - await fs.mkdir(resultDir, { recursive: true }); + let resultDir: string | undefined; + + if (outputDir) { + resultDir = createResultDir(outputDir, scenarioName, 'server'); + await fs.mkdir(resultDir, { recursive: true }); + } // Scenario is guaranteed to exist by CLI validation const scenario = getClientScenario(scenarioName)!; @@ -38,12 +42,14 @@ export async function runServerConformanceTest( const checks = await scenario.run(serverUrl); - await fs.writeFile( - path.join(resultDir, 'checks.json'), - JSON.stringify(checks, null, 2) - ); + if (resultDir) { + await fs.writeFile( + path.join(resultDir, 'checks.json'), + JSON.stringify(checks, null, 2) + ); - console.log(`Results saved to ${resultDir}`); + console.log(`Results saved to ${resultDir}`); + } return { checks, diff --git a/src/runner/utils.ts b/src/runner/utils.ts index 6841bc6..14e9432 100644 --- a/src/runner/utils.ts +++ b/src/runner/utils.ts @@ -1,4 +1,3 @@ -import { promises as fs } from 'fs'; import path from 'path'; import { ConformanceCheck } from '../types'; @@ -51,14 +50,12 @@ export function formatPrettyChecks(checks: ConformanceCheck[]): string { .join('\n'); } -export async function ensureResultsDir(): Promise { - const resultsDir = path.join(process.cwd(), 'results'); - await fs.mkdir(resultsDir, { recursive: true }); - return resultsDir; -} - -export function createResultDir(scenario: string, prefix = ''): string { +export function createResultDir( + baseDir: string, + scenario: string, + prefix = '' +): string { const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const dirName = prefix ? `${prefix}-${scenario}` : scenario; - return path.join('results', `${dirName}-${timestamp}`); + return path.join(baseDir, `${dirName}-${timestamp}`); }