diff --git a/frontend/taskdeck-web/scripts/demo-director-presets.mjs b/frontend/taskdeck-web/scripts/demo-director-presets.mjs new file mode 100644 index 000000000..e6f11dc81 --- /dev/null +++ b/frontend/taskdeck-web/scripts/demo-director-presets.mjs @@ -0,0 +1,189 @@ +/** + * demo-director-presets.mjs + * + * Named configurations (presets) for common demo director proof modes. + * Each preset defines a scenario sequence, timing, and expected outcomes. + */ + +/** + * @typedef {object} DirectorPreset + * @property {string} id - Unique preset identifier + * @property {string} name - Human-readable name + * @property {string} description - What this preset demonstrates + * @property {string} scenario - Scenario ID to run + * @property {object} directorArgs - Override args for the demo director + * @property {object} expectations - Trace expectations for assertion + */ + +/** @type {Record} */ +const PRESETS = { + 'happy-path-capture': { + id: 'happy-path-capture', + name: 'Happy Path Capture', + description: + 'Runs the client-onboarding scenario end-to-end with LLM steps skipped. ' + + 'Validates that capture creation, board setup, and artifact generation complete without errors.', + scenario: 'client-onboarding', + directorArgs: { + skipLlm: true, + skipSeed: false, + turns: 0, + loop: 'mixed', + brain: 'heuristic', + intervalMs: 700, + }, + expectations: { + requiredSequence: [ + 'scenario.start', + 'scenario.step.ok', + ], + requiredEvents: ['scenario.start', 'scenario.end'], + allowedErrorTypes: [], + }, + }, + + 'review-approve-flow': { + id: 'review-approve-flow', + name: 'Review and Approve Flow', + description: + 'Runs the client-onboarding scenario with autopilot turns to exercise ' + + 'the full capture-triage-propose-review-apply loop. LLM steps skipped for determinism.', + scenario: 'client-onboarding', + directorArgs: { + skipLlm: true, + skipSeed: false, + turns: 6, + loop: 'queue', + brain: 'heuristic', + intervalMs: 500, + }, + expectations: { + requiredSequence: ['scenario.start'], + requiredEvents: ['scenario.start', 'scenario.end'], + allowedErrorTypes: [], + }, + }, + + 'error-recovery-demo': { + id: 'error-recovery-demo', + name: 'Error Recovery Demo', + description: + 'Runs the engineering-sprint scenario with aggressive autopilot timing to ' + + 'surface and recover from transient errors. Allows autopilot turn errors.', + scenario: 'engineering-sprint', + directorArgs: { + skipLlm: true, + skipSeed: false, + turns: 8, + loop: 'mixed', + brain: 'heuristic', + intervalMs: 300, + }, + expectations: { + requiredSequence: ['scenario.start'], + requiredEvents: ['scenario.start', 'scenario.end'], + allowedErrorTypes: ['autopilot.turn.error'], + }, + }, + + 'soak-baseline': { + id: 'soak-baseline', + name: 'Soak Baseline', + description: + 'Minimal preset for soak testing. Runs client-onboarding with no autopilot, ' + + 'suitable for repeated loop execution to detect drift.', + scenario: 'client-onboarding', + directorArgs: { + skipLlm: true, + skipSeed: true, + turns: 0, + loop: 'mixed', + brain: 'heuristic', + intervalMs: 700, + }, + expectations: { + requiredSequence: ['scenario.start'], + requiredEvents: ['scenario.start', 'scenario.end'], + allowedErrorTypes: [], + }, + }, +} + +/** + * Returns the list of all available preset IDs. + * @returns {string[]} + */ +export function listPresetIds() { + return Object.keys(PRESETS) +} + +/** + * Returns all presets as an array. + * @returns {DirectorPreset[]} + */ +export function listPresets() { + return Object.values(PRESETS) +} + +/** + * Loads a preset by its ID. + * @param {string} presetId + * @returns {DirectorPreset | null} + */ +export function loadPreset(presetId) { + const normalized = String(presetId || '').trim().toLowerCase() + return PRESETS[normalized] || null +} + +/** + * Loads a preset or throws if not found. + * @param {string} presetId + * @returns {DirectorPreset} + */ +export function requirePreset(presetId) { + const preset = loadPreset(presetId) + if (!preset) { + const available = listPresetIds().join(', ') + throw new Error( + `Unknown director preset: "${presetId}". Available presets: ${available}`, + ) + } + return preset +} + +/** + * Merges preset director args with any user overrides. + * User overrides take precedence. + * + * @param {DirectorPreset} preset + * @param {object} [overrides] + * @returns {object} Merged director args + */ +export function mergePresetArgs(preset, overrides = {}) { + return { + scenario: preset.scenario, + ...preset.directorArgs, + ...stripUndefined(overrides), + } +} + +/** + * Registers a custom preset at runtime (useful for testing or extensions). + * @param {DirectorPreset} preset + */ +export function registerPreset(preset) { + if (!preset?.id) { + throw new Error('Preset must have an id') + } + PRESETS[preset.id] = preset +} + +function stripUndefined(obj) { + const result = {} + for (const [key, value] of Object.entries(obj)) { + if (value !== undefined) { + result[key] = value + } + } + return result +} diff --git a/frontend/taskdeck-web/scripts/demo-report-html.mjs b/frontend/taskdeck-web/scripts/demo-report-html.mjs new file mode 100644 index 000000000..d6fbdc2dc --- /dev/null +++ b/frontend/taskdeck-web/scripts/demo-report-html.mjs @@ -0,0 +1,246 @@ +/** + * demo-report-html.mjs + * + * Generates a self-contained static HTML report from demo artifact bundles + * (run-summary.json, trace.ndjson, screenshots). No external dependencies. + */ + +import fs from 'node:fs/promises' +import path from 'node:path' + +/** + * @typedef {object} DemoReportInput + * @property {object} runSummary - Parsed run-summary.json + * @property {Array} traceEvents - Parsed NDJSON trace events + * @property {Array<{name: string, dataUrl: string}>} screenshots - Base64-encoded screenshots + */ + +/** + * Escapes HTML special characters to prevent injection. + * @param {string} text + * @returns {string} + */ +export function escapeHtml(text) { + const value = String(text ?? '') + return value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +/** + * Classifies a trace event into a pass/fail/info status for display. + * @param {object} event + * @returns {'pass' | 'fail' | 'skip' | 'info'} + */ +export function classifyEventStatus(event) { + const type = String(event?.type || '') + if (type.endsWith('.error')) return 'fail' + if (type.endsWith('.ok')) return 'pass' + if (type.endsWith('.skipped')) return 'skip' + return 'info' +} + +/** + * Extracts step-by-step trace rows for the report. + * @param {Array} events + * @returns {Array<{ts: string, type: string, status: string, label: string, detail: string}>} + */ +export function extractTraceSteps(events) { + return events.map((event) => { + const type = String(event?.type || 'unknown') + const status = classifyEventStatus(event) + const label = event?.stepLabel || event?.stepType || type + const detail = event?.error || event?.reason || event?.outcome || '' + return { + ts: event?.ts || '', + type, + status, + label: String(label), + detail: String(detail), + } + }) +} + +/** + * Reads a PNG file and returns a data-URL string, or null on failure. + * @param {string} filePath + * @returns {Promise} + */ +export async function screenshotToDataUrl(filePath) { + try { + const buffer = await fs.readFile(filePath) + return `data:image/png;base64,${buffer.toString('base64')}` + } catch { + return null + } +} + +/** + * Generates the self-contained HTML report string. + * @param {DemoReportInput} input + * @returns {string} + */ +export function generateHtmlReport({ runSummary, traceEvents, screenshots }) { + const scenario = escapeHtml(runSummary?.scenario || 'unknown') + const status = runSummary?.status || 'unknown' + const statusClass = status === 'ok' ? 'status-pass' : 'status-fail' + const runId = escapeHtml(runSummary?.runId || 'N/A') + const startedAt = escapeHtml(runSummary?.startedAt || '') + const endedAt = escapeHtml(runSummary?.endedAt || '') + const stats = runSummary?.stats || {} + + const steps = extractTraceSteps(traceEvents) + + const stepRows = steps + .map((step) => { + const statusBadge = + step.status === 'pass' + ? 'PASS' + : step.status === 'fail' + ? 'FAIL' + : step.status === 'skip' + ? 'SKIP' + : 'INFO' + + return ( + `` + + `${escapeHtml(step.ts)}` + + `${statusBadge}` + + `${escapeHtml(step.type)}` + + `${escapeHtml(step.label)}` + + `${escapeHtml(step.detail)}` + + `` + ) + }) + .join('\n') + + const screenshotHtml = + screenshots.length === 0 + ? '

No screenshots captured.

' + : screenshots + .map( + (s) => + `

${escapeHtml(s.name)}

` + + `${escapeHtml(s.name)}
`, + ) + .join('\n') + + return ` + + + + +Taskdeck Demo Report - ${scenario} + + + +
+
+

Taskdeck Demo Report

+
+
Scenario
${scenario}
+
Run ID
${runId}
+
Status
${escapeHtml(status.toUpperCase())}
+
Started
${startedAt}
+
Ended
${endedAt}
+
Events
${stats.events ?? 0}
+
Proposals
${stats.proposals ?? 0}
+
Captures
${stats.captures ?? 0}
+
+
+ +
+

Step-by-Step Trace

+ + + ${stepRows || ''} +
TimestampStatusTypeLabelDetail
No trace events.
+
+ +
+

Screenshots

+ ${screenshotHtml} +
+ +
Generated by Taskdeck Demo Director
+
+ +` +} + +/** + * Reads an artifact directory and produces a complete HTML report file. + * @param {string} artifactDir - Path to the demo-artifacts/run-xxx directory + * @param {string} outputPath - Where to write the HTML file + */ +export async function generateReportFromArtifacts(artifactDir, outputPath) { + const summaryPath = path.join(artifactDir, 'run-summary.json') + const tracePath = path.join(artifactDir, 'trace.ndjson') + const screenshotsDir = path.join(artifactDir, 'screenshots') + + const runSummary = JSON.parse(await fs.readFile(summaryPath, 'utf8')) + + let traceEvents = [] + try { + const raw = await fs.readFile(tracePath, 'utf8') + for (const line of raw.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + try { + traceEvents.push(JSON.parse(trimmed)) + } catch { + // skip malformed lines + } + } + } catch { + // trace file may not exist + } + + const screenshots = [] + try { + const files = await fs.readdir(screenshotsDir) + const pngs = files.filter((f) => f.toLowerCase().endsWith('.png')).sort() + for (const file of pngs) { + const dataUrl = await screenshotToDataUrl(path.join(screenshotsDir, file)) + if (dataUrl) { + screenshots.push({ name: file, dataUrl }) + } + } + } catch { + // screenshots dir may not exist + } + + const html = generateHtmlReport({ runSummary, traceEvents, screenshots }) + await fs.mkdir(path.dirname(outputPath), { recursive: true }) + await fs.writeFile(outputPath, html, 'utf8') + return outputPath +} diff --git a/frontend/taskdeck-web/scripts/demo-soak.mjs b/frontend/taskdeck-web/scripts/demo-soak.mjs new file mode 100644 index 000000000..43d7d826e --- /dev/null +++ b/frontend/taskdeck-web/scripts/demo-soak.mjs @@ -0,0 +1,218 @@ +/** + * demo-soak.mjs + * + * Long-run soak mode that runs director scenarios in a loop with configurable + * duration or iteration count. Tracks cumulative metrics and outputs a summary. + */ + +/** + * @typedef {object} SoakConfig + * @property {number} [maxIterations] - Max number of runs (0 = unlimited, use maxDurationMs) + * @property {number} [maxDurationMs] - Max total duration in ms (0 = unlimited, use maxIterations) + * @property {number} [cooldownMs] - Pause between runs in ms + */ + +/** + * @typedef {object} SoakIterationResult + * @property {number} iteration + * @property {'pass' | 'fail'} status + * @property {number} durationMs + * @property {number} eventCount + * @property {string | null} error + */ + +/** + * @typedef {object} SoakSummary + * @property {string} startedAt + * @property {string} endedAt + * @property {number} totalRuns + * @property {number} passCount + * @property {number} failCount + * @property {number} passRate - Percentage 0-100 + * @property {number} totalDurationMs + * @property {number} avgIterationMs + * @property {number} minIterationMs + * @property {number} maxIterationMs + * @property {number} timingDriftMs - Difference between fastest and slowest run + * @property {SoakIterationResult[]} iterations + * @property {object} memoryIndicators + */ + +/** + * Default soak configuration. + * @returns {SoakConfig} + */ +export function defaultSoakConfig() { + return { + maxIterations: 10, + maxDurationMs: 0, + cooldownMs: 500, + } +} + +/** + * Validates soak configuration. + * @param {SoakConfig} config + * @returns {SoakConfig} + */ +export function validateSoakConfig(config) { + const maxIterations = Number(config?.maxIterations ?? 10) + const maxDurationMs = Number(config?.maxDurationMs ?? 0) + const cooldownMs = Number(config?.cooldownMs ?? 500) + + if (maxIterations <= 0 && maxDurationMs <= 0) { + throw new Error( + 'Soak config requires at least one of maxIterations > 0 or maxDurationMs > 0', + ) + } + + if (maxIterations < 0 || !Number.isFinite(maxIterations)) { + throw new Error(`Invalid maxIterations: ${maxIterations}`) + } + + if (maxDurationMs < 0 || !Number.isFinite(maxDurationMs)) { + throw new Error(`Invalid maxDurationMs: ${maxDurationMs}`) + } + + if (cooldownMs < 0 || !Number.isFinite(cooldownMs)) { + throw new Error(`Invalid cooldownMs: ${cooldownMs}`) + } + + return { maxIterations, maxDurationMs, cooldownMs } +} + +/** + * Determines whether the soak loop should continue. + * @param {number} iteration - Current 0-based iteration index + * @param {number} elapsedMs - Time elapsed since soak start + * @param {SoakConfig} config + * @returns {boolean} + */ +export function shouldContinueSoak(iteration, elapsedMs, config) { + if (config.maxIterations > 0 && iteration >= config.maxIterations) { + return false + } + + if (config.maxDurationMs > 0 && elapsedMs >= config.maxDurationMs) { + return false + } + + return true +} + +/** + * Collects memory indicators from the Node.js process (if available). + * @returns {object} + */ +export function collectMemoryIndicators() { + if (typeof process !== 'undefined' && typeof process.memoryUsage === 'function') { + const mem = process.memoryUsage() + return { + heapUsedMB: Math.round((mem.heapUsed / 1024 / 1024) * 100) / 100, + heapTotalMB: Math.round((mem.heapTotal / 1024 / 1024) * 100) / 100, + rssMB: Math.round((mem.rss / 1024 / 1024) * 100) / 100, + externalMB: Math.round((mem.external / 1024 / 1024) * 100) / 100, + } + } + + return { heapUsedMB: 0, heapTotalMB: 0, rssMB: 0, externalMB: 0 } +} + +/** + * Builds a soak summary from accumulated iteration results. + * @param {string} startedAt - ISO timestamp + * @param {string} endedAt - ISO timestamp + * @param {SoakIterationResult[]} iterations + * @returns {SoakSummary} + */ +export function buildSoakSummary(startedAt, endedAt, iterations) { + const totalRuns = iterations.length + const passCount = iterations.filter((r) => r.status === 'pass').length + const failCount = totalRuns - passCount + const passRate = totalRuns > 0 ? Math.round((passCount / totalRuns) * 10000) / 100 : 0 + + let totalDurationMs = 0 + let minIterationMs = totalRuns > 0 ? Infinity : 0 + let maxIterationMs = 0 + for (const r of iterations) { + totalDurationMs += r.durationMs + if (r.durationMs < minIterationMs) minIterationMs = r.durationMs + if (r.durationMs > maxIterationMs) maxIterationMs = r.durationMs + } + // Normalize Infinity to 0 when there are no runs + if (minIterationMs === Infinity) minIterationMs = 0 + const avgIterationMs = totalRuns > 0 ? Math.round(totalDurationMs / totalRuns) : 0 + const timingDriftMs = maxIterationMs - minIterationMs + + return { + startedAt, + endedAt, + totalRuns, + passCount, + failCount, + passRate, + totalDurationMs, + avgIterationMs, + minIterationMs, + maxIterationMs, + timingDriftMs, + iterations, + memoryIndicators: collectMemoryIndicators(), + } +} + +/** + * Runs a soak loop using a provided runner function. + * + * @param {SoakConfig} config + * @param {(iteration: number) => Promise<{pass: boolean, eventCount: number, error?: string}>} runFn + * Function that executes one iteration. Must return pass/fail status and event count. + * @returns {Promise} + */ +export async function runSoak(config, runFn) { + const validated = validateSoakConfig(config) + const startedAt = new Date().toISOString() + const soakStartTime = Date.now() + const iterations = [] + + let iteration = 0 + while (shouldContinueSoak(iteration, Date.now() - soakStartTime, validated)) { + const iterStart = Date.now() + let status = 'pass' + let eventCount = 0 + let error = null + + try { + const result = await runFn(iteration) + status = result.pass ? 'pass' : 'fail' + eventCount = result.eventCount || 0 + error = result.error || null + } catch (err) { + status = 'fail' + error = String(err?.message || err) + } + + const durationMs = Date.now() - iterStart + iterations.push({ + iteration, + status, + durationMs, + eventCount, + error, + }) + + iteration++ + + // Cooldown between runs (skip after last) + if (validated.cooldownMs > 0 && shouldContinueSoak(iteration, Date.now() - soakStartTime, validated)) { + await sleep(validated.cooldownMs) + } + } + + const endedAt = new Date().toISOString() + return buildSoakSummary(startedAt, endedAt, iterations) +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/frontend/taskdeck-web/scripts/demo-trace-assertions.mjs b/frontend/taskdeck-web/scripts/demo-trace-assertions.mjs new file mode 100644 index 000000000..66bda6b15 --- /dev/null +++ b/frontend/taskdeck-web/scripts/demo-trace-assertions.mjs @@ -0,0 +1,220 @@ +/** + * demo-trace-assertions.mjs + * + * Assertion utilities for comparing demo trace output against known-good + * snapshots and validating structural expectations on trace events. + */ + +/** + * @typedef {'exact' | 'structural'} MatchMode + */ + +/** + * @typedef {object} TraceAssertionResult + * @property {boolean} pass + * @property {string[]} errors + */ + +/** + * Compares two trace snapshots using exact match mode. + * Events must match in count, order, and content (by JSON equality). + * + * @param {Array} actual - Actual trace events + * @param {Array} expected - Known-good trace events + * @returns {TraceAssertionResult} + */ +export function assertTraceExactMatch(actual, expected) { + const errors = [] + + if (!Array.isArray(actual)) { + return { pass: false, errors: ['Actual trace is not an array'] } + } + if (!Array.isArray(expected)) { + return { pass: false, errors: ['Expected trace is not an array'] } + } + + if (actual.length !== expected.length) { + errors.push( + `Event count mismatch: got ${actual.length}, expected ${expected.length}`, + ) + } + + const limit = Math.min(actual.length, expected.length) + for (let i = 0; i < limit; i++) { + const actualJson = JSON.stringify(actual[i]) + const expectedJson = JSON.stringify(expected[i]) + if (actualJson !== expectedJson) { + errors.push( + `Event at index ${i} differs:\n actual: ${actualJson.slice(0, 200)}\n expected: ${expectedJson.slice(0, 200)}`, + ) + } + } + + return { pass: errors.length === 0, errors } +} + +/** + * Compares two trace snapshots using structural/shape match mode. + * Validates that each event has the same `type` field in order, but + * ignores timestamps, IDs, and other volatile fields. + * + * @param {Array} actual + * @param {Array} expected + * @param {object} [options] + * @param {string[]} [options.shapeFields] - Fields to compare beyond `type` (default: ['type']) + * @returns {TraceAssertionResult} + */ +export function assertTraceStructuralMatch(actual, expected, options = {}) { + const errors = [] + const shapeFields = options.shapeFields || ['type'] + + if (!Array.isArray(actual)) { + return { pass: false, errors: ['Actual trace is not an array'] } + } + if (!Array.isArray(expected)) { + return { pass: false, errors: ['Expected trace is not an array'] } + } + + if (actual.length !== expected.length) { + errors.push( + `Event count mismatch: got ${actual.length}, expected ${expected.length}`, + ) + } + + const limit = Math.min(actual.length, expected.length) + for (let i = 0; i < limit; i++) { + for (const field of shapeFields) { + const actualValue = actual[i]?.[field] + const expectedValue = expected[i]?.[field] + if (actualValue !== expectedValue) { + errors.push( + `Event[${i}].${field}: got "${actualValue}", expected "${expectedValue}"`, + ) + } + } + } + + return { pass: errors.length === 0, errors } +} + +/** + * Validates that trace events contain a required ordered sequence of event types. + * The sequence must appear in order, but other events may appear between them. + * + * @param {Array} events + * @param {string[]} requiredSequence - Ordered list of event types that must appear + * @returns {TraceAssertionResult} + */ +export function assertTraceStepOrdering(events, requiredSequence) { + const errors = [] + + if (!Array.isArray(events)) { + return { pass: false, errors: ['Events is not an array'] } + } + if (!Array.isArray(requiredSequence) || requiredSequence.length === 0) { + return { pass: true, errors: [] } + } + + let seqIndex = 0 + for (const event of events) { + if (seqIndex >= requiredSequence.length) break + if (String(event?.type || '') === requiredSequence[seqIndex]) { + seqIndex++ + } + } + + if (seqIndex < requiredSequence.length) { + const missing = requiredSequence.slice(seqIndex) + errors.push( + `Required event sequence incomplete. Missing from index ${seqIndex}: [${missing.join(', ')}]`, + ) + } + + return { pass: errors.length === 0, errors } +} + +/** + * Validates that no events with error-typed suffixes exist in the trace, + * unless explicitly allowed. + * + * @param {Array} events + * @param {object} [options] + * @param {string[]} [options.allowedErrorTypes] - Error types to ignore + * @returns {TraceAssertionResult} + */ +export function assertNoUnexpectedErrors(events, options = {}) { + const errors = [] + const allowed = new Set(options.allowedErrorTypes || []) + + if (!Array.isArray(events)) { + return { pass: false, errors: ['Events is not an array'] } + } + + for (let i = 0; i < events.length; i++) { + const type = String(events[i]?.type || '') + if (type.endsWith('.error') && !allowed.has(type)) { + const detail = events[i]?.error || events[i]?.reason || '' + errors.push( + `Unexpected error event at index ${i}: ${type}${detail ? ` - ${String(detail).slice(0, 200)}` : ''}`, + ) + } + } + + return { pass: errors.length === 0, errors } +} + +/** + * Validates that required event types are present in the trace (in any order). + * + * @param {Array} events + * @param {string[]} requiredTypes - Event types that must appear at least once + * @returns {TraceAssertionResult} + */ +export function assertRequiredEventsPresent(events, requiredTypes) { + const errors = [] + + if (!Array.isArray(events)) { + return { pass: false, errors: ['Events is not an array'] } + } + + const presentTypes = new Set(events.map((e) => String(e?.type || ''))) + for (const requiredType of requiredTypes) { + if (!presentTypes.has(requiredType)) { + errors.push(`Required event type missing: ${requiredType}`) + } + } + + return { pass: errors.length === 0, errors } +} + +/** + * Runs a full assertion suite against a trace, combining ordering, errors, + * and required events checks. + * + * @param {Array} events + * @param {object} expectations + * @param {string[]} [expectations.requiredSequence] + * @param {string[]} [expectations.requiredEvents] + * @param {string[]} [expectations.allowedErrorTypes] + * @returns {TraceAssertionResult} + */ +export function assertTrace(events, expectations = {}) { + const allErrors = [] + + if (expectations.requiredSequence) { + const result = assertTraceStepOrdering(events, expectations.requiredSequence) + allErrors.push(...result.errors) + } + + if (expectations.requiredEvents) { + const result = assertRequiredEventsPresent(events, expectations.requiredEvents) + allErrors.push(...result.errors) + } + + const errorResult = assertNoUnexpectedErrors(events, { + allowedErrorTypes: expectations.allowedErrorTypes, + }) + allErrors.push(...errorResult.errors) + + return { pass: allErrors.length === 0, errors: allErrors } +} diff --git a/frontend/taskdeck-web/tests/demo-director-presets.spec.ts b/frontend/taskdeck-web/tests/demo-director-presets.spec.ts new file mode 100644 index 000000000..bb234e9b4 --- /dev/null +++ b/frontend/taskdeck-web/tests/demo-director-presets.spec.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest' + +import { + listPresetIds, + listPresets, + loadPreset, + requirePreset, + mergePresetArgs, + registerPreset, +} from '../scripts/demo-director-presets.mjs' + +describe('demo director presets', () => { + describe('listPresetIds', () => { + it('returns a non-empty array of preset IDs', () => { + const ids = listPresetIds() + expect(ids.length).toBeGreaterThan(0) + expect(ids).toContain('happy-path-capture') + expect(ids).toContain('review-approve-flow') + expect(ids).toContain('error-recovery-demo') + expect(ids).toContain('soak-baseline') + }) + }) + + describe('listPresets', () => { + it('returns preset objects with required fields', () => { + const presets = listPresets() + expect(presets.length).toBe(listPresetIds().length) + + for (const preset of presets) { + expect(preset).toHaveProperty('id') + expect(preset).toHaveProperty('name') + expect(preset).toHaveProperty('description') + expect(preset).toHaveProperty('scenario') + expect(preset).toHaveProperty('directorArgs') + expect(preset).toHaveProperty('expectations') + expect(typeof preset.id).toBe('string') + expect(typeof preset.scenario).toBe('string') + } + }) + }) + + describe('loadPreset', () => { + it('returns the preset for a valid ID', () => { + const preset = loadPreset('happy-path-capture') + expect(preset).not.toBeNull() + expect(preset.id).toBe('happy-path-capture') + expect(preset.scenario).toBe('client-onboarding') + }) + + it('returns null for unknown preset IDs', () => { + expect(loadPreset('nonexistent-preset')).toBeNull() + expect(loadPreset('')).toBeNull() + expect(loadPreset(null)).toBeNull() + }) + + it('normalizes case on lookup', () => { + const preset = loadPreset('Happy-Path-Capture') + // loadPreset lowercases the input, so mixed case finds the lowercase key + expect(preset).not.toBeNull() + expect(preset.id).toBe('happy-path-capture') + }) + }) + + describe('requirePreset', () => { + it('returns the preset for a valid ID', () => { + const preset = requirePreset('soak-baseline') + expect(preset.id).toBe('soak-baseline') + }) + + it('throws for unknown preset IDs with available list', () => { + expect(() => requirePreset('nope')).toThrow('Unknown director preset') + expect(() => requirePreset('nope')).toThrow('happy-path-capture') + }) + }) + + describe('mergePresetArgs', () => { + it('returns preset defaults when no overrides are given', () => { + const preset = loadPreset('happy-path-capture')! + const args = mergePresetArgs(preset) + + expect(args.scenario).toBe('client-onboarding') + expect(args.skipLlm).toBe(true) + expect(args.turns).toBe(0) + }) + + it('applies user overrides on top of preset defaults', () => { + const preset = loadPreset('happy-path-capture')! + const args = mergePresetArgs(preset, { turns: 5, intervalMs: 200 }) + + expect(args.turns).toBe(5) + expect(args.intervalMs).toBe(200) + // Preset defaults still present + expect(args.skipLlm).toBe(true) + expect(args.scenario).toBe('client-onboarding') + }) + + it('ignores undefined override values', () => { + const preset = loadPreset('happy-path-capture')! + const args = mergePresetArgs(preset, { turns: undefined }) + + expect(args.turns).toBe(0) // preset default + }) + }) + + describe('registerPreset', () => { + it('registers a custom preset that can be loaded', () => { + registerPreset({ + id: 'test-custom-preset', + name: 'Custom Test', + description: 'A test preset', + scenario: 'client-onboarding', + directorArgs: { skipLlm: true, turns: 1 }, + expectations: { requiredEvents: ['scenario.start'] }, + }) + + const loaded = loadPreset('test-custom-preset') + expect(loaded).not.toBeNull() + expect(loaded.name).toBe('Custom Test') + }) + + it('rejects presets without an id', () => { + expect(() => registerPreset({ name: 'No ID' } as any)).toThrow('Preset must have an id') + }) + }) + + describe('preset expectations are well-formed', () => { + it('all presets have valid expectation structures', () => { + for (const preset of listPresets()) { + const exp = preset.expectations + expect(exp).toBeDefined() + + if (exp.requiredSequence) { + expect(Array.isArray(exp.requiredSequence)).toBe(true) + for (const item of exp.requiredSequence) { + expect(typeof item).toBe('string') + } + } + + if (exp.requiredEvents) { + expect(Array.isArray(exp.requiredEvents)).toBe(true) + } + + if (exp.allowedErrorTypes) { + expect(Array.isArray(exp.allowedErrorTypes)).toBe(true) + } + } + }) + }) +}) diff --git a/frontend/taskdeck-web/tests/demo-preset-integration.spec.ts b/frontend/taskdeck-web/tests/demo-preset-integration.spec.ts new file mode 100644 index 000000000..60cb22a18 --- /dev/null +++ b/frontend/taskdeck-web/tests/demo-preset-integration.spec.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from 'vitest' + +import { requirePreset, mergePresetArgs } from '../scripts/demo-director-presets.mjs' +import { assertTrace } from '../scripts/demo-trace-assertions.mjs' +import { generateHtmlReport, extractTraceSteps } from '../scripts/demo-report-html.mjs' +import { buildSoakSummary } from '../scripts/demo-soak.mjs' + +describe('preset scenario integration', () => { + it('runs the happy-path-capture preset through the full assertion and reporting pipeline', () => { + // Load preset + const preset = requirePreset('happy-path-capture') + expect(preset.scenario).toBe('client-onboarding') + + // Merge args (no overrides) + const args = mergePresetArgs(preset) + expect(args.skipLlm).toBe(true) + expect(args.turns).toBe(0) + + // Simulate a successful trace that matches preset expectations + const simulatedTrace = [ + { type: 'scenario.start', ts: '2026-01-01T00:00:00Z' }, + { type: 'scenario.step.ok', ts: '2026-01-01T00:00:01Z', stepLabel: 'Create board', stepType: 'createBoard' }, + { type: 'scenario.step.ok', ts: '2026-01-01T00:00:02Z', stepLabel: 'Apply starter pack', stepType: 'applyStarterPack' }, + { type: 'scenario.step.ok', ts: '2026-01-01T00:00:03Z', stepLabel: 'Create capture', stepType: 'createCapture' }, + { type: 'scenario.step.skipped', ts: '2026-01-01T00:00:04Z', stepLabel: 'Triage capture', reason: 'requiresLlm' }, + { type: 'scenario.end', ts: '2026-01-01T00:00:05Z' }, + ] + + // Validate trace against preset expectations + const assertionResult = assertTrace(simulatedTrace, preset.expectations) + expect(assertionResult.pass).toBe(true) + expect(assertionResult.errors).toEqual([]) + + // Generate HTML report from the same data + const html = generateHtmlReport({ + runSummary: { + runId: 'integration-test-1', + scenario: preset.scenario, + status: 'ok', + startedAt: '2026-01-01T00:00:00Z', + endedAt: '2026-01-01T00:00:05Z', + stats: { events: simulatedTrace.length, proposals: 0, captures: 1 }, + }, + traceEvents: simulatedTrace, + screenshots: [], + }) + + expect(html).toContain('client-onboarding') + expect(html).toContain('Create board') + expect(html).toContain('PASS') + expect(html).toContain('SKIP') + + // Verify trace steps extraction + const steps = extractTraceSteps(simulatedTrace) + expect(steps).toHaveLength(6) + expect(steps[0].status).toBe('info') // scenario.start + expect(steps[1].status).toBe('pass') // step.ok + expect(steps[4].status).toBe('skip') // step.skipped + }) + + it('detects assertion failures when trace does not match preset expectations', () => { + const preset = requirePreset('happy-path-capture') + + // Simulate a trace with an unexpected error + const badTrace = [ + { type: 'scenario.start', ts: '2026-01-01T00:00:00Z' }, + { type: 'scenario.step.error', ts: '2026-01-01T00:00:01Z', error: 'board creation failed' }, + ] + + const result = assertTrace(badTrace, preset.expectations) + expect(result.pass).toBe(false) + expect(result.errors.length).toBeGreaterThan(0) + // Should flag missing scenario.end and the unexpected error + expect(result.errors.some((e: string) => e.includes('scenario.end'))).toBe(true) + expect(result.errors.some((e: string) => e.includes('scenario.step.error'))).toBe(true) + }) + + it('integrates soak summary with preset reporting', () => { + const iterations = [ + { iteration: 0, status: 'pass' as const, durationMs: 120, eventCount: 6, error: null }, + { iteration: 1, status: 'pass' as const, durationMs: 130, eventCount: 6, error: null }, + { iteration: 2, status: 'fail' as const, durationMs: 200, eventCount: 2, error: 'timeout' }, + ] + + const summary = buildSoakSummary('2026-01-01T00:00:00Z', '2026-01-01T00:01:00Z', iterations) + + // Verify the soak summary can feed into an HTML report + const html = generateHtmlReport({ + runSummary: { + runId: 'soak-integration-test', + scenario: 'client-onboarding', + status: summary.failCount > 0 ? 'error' : 'ok', + startedAt: summary.startedAt, + endedAt: summary.endedAt, + stats: { + events: summary.totalRuns, + proposals: 0, + captures: 0, + }, + }, + traceEvents: iterations.map((iter) => ({ + type: iter.status === 'pass' ? 'soak.iteration.ok' : 'soak.iteration.error', + ts: summary.startedAt, + error: iter.error, + })), + screenshots: [], + }) + + expect(html).toContain('soak-integration-test') + expect(html).toContain('status-fail') // because failCount > 0 + }) +}) diff --git a/frontend/taskdeck-web/tests/demo-report-html.spec.ts b/frontend/taskdeck-web/tests/demo-report-html.spec.ts new file mode 100644 index 000000000..1c49d0771 --- /dev/null +++ b/frontend/taskdeck-web/tests/demo-report-html.spec.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'vitest' + +import { + escapeHtml, + classifyEventStatus, + extractTraceSteps, + generateHtmlReport, +} from '../scripts/demo-report-html.mjs' + +describe('demo HTML report generator', () => { + describe('escapeHtml', () => { + it('escapes HTML special characters', () => { + expect(escapeHtml('')).toBe( + '<script>alert("xss")</script>', + ) + }) + + it('handles null and undefined gracefully', () => { + expect(escapeHtml(null)).toBe('') + expect(escapeHtml(undefined)).toBe('') + }) + + it('escapes ampersands and single quotes', () => { + expect(escapeHtml("Tom & Jerry's")).toBe('Tom & Jerry's') + }) + }) + + describe('classifyEventStatus', () => { + it('classifies .ok events as pass', () => { + expect(classifyEventStatus({ type: 'scenario.step.ok' })).toBe('pass') + }) + + it('classifies .error events as fail', () => { + expect(classifyEventStatus({ type: 'scenario.step.error' })).toBe('fail') + }) + + it('classifies .skipped events as skip', () => { + expect(classifyEventStatus({ type: 'scenario.step.skipped' })).toBe('skip') + }) + + it('classifies other events as info', () => { + expect(classifyEventStatus({ type: 'scenario.start' })).toBe('info') + expect(classifyEventStatus({})).toBe('info') + expect(classifyEventStatus(null)).toBe('info') + }) + }) + + describe('extractTraceSteps', () => { + it('maps events to step rows with correct status', () => { + const events = [ + { type: 'scenario.start', ts: '2026-01-01T00:00:00Z' }, + { type: 'scenario.step.ok', ts: '2026-01-01T00:00:01Z', stepLabel: 'Create board' }, + { type: 'scenario.step.error', ts: '2026-01-01T00:00:02Z', error: 'timeout' }, + ] + + const steps = extractTraceSteps(events) + expect(steps).toHaveLength(3) + expect(steps[0].status).toBe('info') + expect(steps[1].status).toBe('pass') + expect(steps[1].label).toBe('Create board') + expect(steps[2].status).toBe('fail') + expect(steps[2].detail).toBe('timeout') + }) + + it('handles empty events array', () => { + expect(extractTraceSteps([])).toEqual([]) + }) + }) + + describe('generateHtmlReport', () => { + it('produces valid self-contained HTML with scenario name and status', () => { + const html = generateHtmlReport({ + runSummary: { + runId: 'test-run-1', + scenario: 'client-onboarding', + status: 'ok', + startedAt: '2026-01-01T00:00:00Z', + endedAt: '2026-01-01T00:01:00Z', + stats: { events: 5, proposals: 2, captures: 1 }, + }, + traceEvents: [ + { type: 'scenario.start', ts: '2026-01-01T00:00:00Z' }, + { type: 'scenario.step.ok', ts: '2026-01-01T00:00:01Z', stepLabel: 'Create board' }, + ], + screenshots: [], + }) + + expect(html).toContain('') + expect(html).toContain('client-onboarding') + expect(html).toContain('test-run-1') + expect(html).toContain('status-pass') + expect(html).toContain('PASS') + expect(html).toContain('Create board') + expect(html).toContain('No screenshots captured.') + // Self-contained: no external link/script tags + expect(html).not.toContain(' { + const html = generateHtmlReport({ + runSummary: { runId: 'r1', scenario: 'test', status: 'ok', stats: {} }, + traceEvents: [], + screenshots: [ + { name: 'step1.png', dataUrl: 'data:image/png;base64,abc123' }, + ], + }) + + expect(html).toContain('step1.png') + expect(html).toContain('data:image/png;base64,abc123') + expect(html).not.toContain('No screenshots captured.') + }) + + it('renders fail status for error runs', () => { + const html = generateHtmlReport({ + runSummary: { runId: 'r2', scenario: 'test', status: 'error', stats: {} }, + traceEvents: [{ type: 'scenario.step.error', error: 'boom' }], + screenshots: [], + }) + + expect(html).toContain('status-fail') + expect(html).toContain('FAIL') + expect(html).toContain('boom') + }) + + it('escapes HTML in user-provided data to prevent injection', () => { + const html = generateHtmlReport({ + runSummary: { + runId: '', + scenario: '', + status: 'ok', + stats: {}, + }, + traceEvents: [], + screenshots: [], + }) + + expect(html).not.toContain('') + expect(html).toContain('<script>') + }) + }) +}) diff --git a/frontend/taskdeck-web/tests/demo-soak.spec.ts b/frontend/taskdeck-web/tests/demo-soak.spec.ts new file mode 100644 index 000000000..b5fabf11a --- /dev/null +++ b/frontend/taskdeck-web/tests/demo-soak.spec.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from 'vitest' + +import { + defaultSoakConfig, + validateSoakConfig, + shouldContinueSoak, + buildSoakSummary, + runSoak, +} from '../scripts/demo-soak.mjs' + +describe('demo soak mode', () => { + describe('defaultSoakConfig', () => { + it('returns sensible defaults', () => { + const config = defaultSoakConfig() + expect(config.maxIterations).toBe(10) + expect(config.maxDurationMs).toBe(0) + expect(config.cooldownMs).toBe(500) + }) + }) + + describe('validateSoakConfig', () => { + it('accepts valid config', () => { + const config = validateSoakConfig({ maxIterations: 5, maxDurationMs: 0, cooldownMs: 100 }) + expect(config.maxIterations).toBe(5) + }) + + it('rejects config with both limits at zero', () => { + expect(() => + validateSoakConfig({ maxIterations: 0, maxDurationMs: 0, cooldownMs: 0 }), + ).toThrow('at least one of maxIterations') + }) + + it('rejects negative maxIterations', () => { + expect(() => + validateSoakConfig({ maxIterations: -1, maxDurationMs: 1000, cooldownMs: 0 }), + ).toThrow('Invalid maxIterations') + }) + + it('rejects negative maxDurationMs', () => { + expect(() => + validateSoakConfig({ maxIterations: 1, maxDurationMs: -1, cooldownMs: 0 }), + ).toThrow('Invalid maxDurationMs') + }) + + it('rejects negative cooldownMs', () => { + expect(() => + validateSoakConfig({ maxIterations: 1, maxDurationMs: 0, cooldownMs: -1 }), + ).toThrow('Invalid cooldownMs') + }) + }) + + describe('shouldContinueSoak', () => { + it('stops when maxIterations is reached', () => { + const config = { maxIterations: 3, maxDurationMs: 0, cooldownMs: 0 } + expect(shouldContinueSoak(0, 0, config)).toBe(true) + expect(shouldContinueSoak(2, 0, config)).toBe(true) + expect(shouldContinueSoak(3, 0, config)).toBe(false) + }) + + it('stops when maxDurationMs is exceeded', () => { + const config = { maxIterations: 0, maxDurationMs: 5000, cooldownMs: 0 } + expect(shouldContinueSoak(0, 0, config)).toBe(true) + expect(shouldContinueSoak(0, 4999, config)).toBe(true) + expect(shouldContinueSoak(0, 5000, config)).toBe(false) + }) + + it('continues when neither limit is reached', () => { + const config = { maxIterations: 10, maxDurationMs: 60000, cooldownMs: 0 } + expect(shouldContinueSoak(5, 30000, config)).toBe(true) + }) + }) + + describe('buildSoakSummary', () => { + it('computes correct statistics from iteration results', () => { + const iterations = [ + { iteration: 0, status: 'pass', durationMs: 100, eventCount: 5, error: null }, + { iteration: 1, status: 'pass', durationMs: 150, eventCount: 6, error: null }, + { iteration: 2, status: 'fail', durationMs: 200, eventCount: 3, error: 'timeout' }, + ] + + const summary = buildSoakSummary('2026-01-01T00:00:00Z', '2026-01-01T00:01:00Z', iterations) + + expect(summary.totalRuns).toBe(3) + expect(summary.passCount).toBe(2) + expect(summary.failCount).toBe(1) + expect(summary.passRate).toBeCloseTo(66.67, 1) + expect(summary.totalDurationMs).toBe(450) + expect(summary.avgIterationMs).toBe(150) + expect(summary.minIterationMs).toBe(100) + expect(summary.maxIterationMs).toBe(200) + expect(summary.timingDriftMs).toBe(100) + expect(summary.iterations).toHaveLength(3) + expect(summary.memoryIndicators).toHaveProperty('heapUsedMB') + }) + + it('handles empty iterations', () => { + const summary = buildSoakSummary('2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', []) + + expect(summary.totalRuns).toBe(0) + expect(summary.passCount).toBe(0) + expect(summary.failCount).toBe(0) + expect(summary.passRate).toBe(0) + expect(summary.avgIterationMs).toBe(0) + expect(summary.minIterationMs).toBe(0) + expect(summary.maxIterationMs).toBe(0) + }) + + it('handles all-pass runs', () => { + const iterations = [ + { iteration: 0, status: 'pass', durationMs: 50, eventCount: 2, error: null }, + { iteration: 1, status: 'pass', durationMs: 60, eventCount: 3, error: null }, + ] + + const summary = buildSoakSummary('2026-01-01T00:00:00Z', '2026-01-01T00:01:00Z', iterations) + expect(summary.passRate).toBe(100) + expect(summary.failCount).toBe(0) + }) + }) + + describe('runSoak', () => { + it('executes the configured number of iterations', async () => { + const calls: number[] = [] + const summary = await runSoak( + { maxIterations: 3, maxDurationMs: 0, cooldownMs: 0 }, + async (iteration) => { + calls.push(iteration) + return { pass: true, eventCount: 2 } + }, + ) + + expect(calls).toEqual([0, 1, 2]) + expect(summary.totalRuns).toBe(3) + expect(summary.passCount).toBe(3) + expect(summary.passRate).toBe(100) + }) + + it('records failures from the runner function', async () => { + const summary = await runSoak( + { maxIterations: 2, maxDurationMs: 0, cooldownMs: 0 }, + async (iteration) => { + if (iteration === 1) { + return { pass: false, eventCount: 0, error: 'simulated failure' } + } + return { pass: true, eventCount: 5 } + }, + ) + + expect(summary.totalRuns).toBe(2) + expect(summary.passCount).toBe(1) + expect(summary.failCount).toBe(1) + expect(summary.iterations[1].error).toBe('simulated failure') + }) + + it('catches runner exceptions as failures', async () => { + const summary = await runSoak( + { maxIterations: 1, maxDurationMs: 0, cooldownMs: 0 }, + async () => { + throw new Error('unexpected crash') + }, + ) + + expect(summary.totalRuns).toBe(1) + expect(summary.failCount).toBe(1) + expect(summary.iterations[0].error).toBe('unexpected crash') + }) + + it('respects maxDurationMs to stop early', async () => { + let _iteration = 0 + const summary = await runSoak( + { maxIterations: 1000, maxDurationMs: 100, cooldownMs: 0 }, + async () => { + _iteration++ + // Simulate some work + await new Promise((resolve) => setTimeout(resolve, 30)) + return { pass: true, eventCount: 1 } + }, + ) + + // Should have stopped well before 1000 iterations due to time limit + expect(summary.totalRuns).toBeLessThan(1000) + expect(summary.totalRuns).toBeGreaterThan(0) + }) + + it('includes timing metrics in the summary', async () => { + const summary = await runSoak( + { maxIterations: 2, maxDurationMs: 0, cooldownMs: 0 }, + async () => { + return { pass: true, eventCount: 1 } + }, + ) + + expect(summary.startedAt).toBeTruthy() + expect(summary.endedAt).toBeTruthy() + expect(summary.avgIterationMs).toBeGreaterThanOrEqual(0) + expect(summary.timingDriftMs).toBeGreaterThanOrEqual(0) + }) + }) +}) diff --git a/frontend/taskdeck-web/tests/demo-trace-assertions.spec.ts b/frontend/taskdeck-web/tests/demo-trace-assertions.spec.ts new file mode 100644 index 000000000..76a763743 --- /dev/null +++ b/frontend/taskdeck-web/tests/demo-trace-assertions.spec.ts @@ -0,0 +1,180 @@ +import { describe, expect, it } from 'vitest' + +import { + assertTraceExactMatch, + assertTraceStructuralMatch, + assertTraceStepOrdering, + assertNoUnexpectedErrors, + assertRequiredEventsPresent, + assertTrace, +} from '../scripts/demo-trace-assertions.mjs' + +describe('demo trace assertions', () => { + describe('assertTraceExactMatch', () => { + it('passes when traces are identical', () => { + const events = [{ type: 'a', ts: '1' }, { type: 'b', ts: '2' }] + const result = assertTraceExactMatch(events, JSON.parse(JSON.stringify(events))) + expect(result.pass).toBe(true) + expect(result.errors).toEqual([]) + }) + + it('fails on event count mismatch', () => { + const result = assertTraceExactMatch([{ type: 'a' }], [{ type: 'a' }, { type: 'b' }]) + expect(result.pass).toBe(false) + expect(result.errors[0]).toContain('Event count mismatch') + }) + + it('fails on content mismatch at specific index', () => { + const result = assertTraceExactMatch( + [{ type: 'a', ts: '1' }], + [{ type: 'a', ts: '2' }], + ) + expect(result.pass).toBe(false) + expect(result.errors[0]).toContain('index 0') + }) + + it('rejects non-array inputs', () => { + expect(assertTraceExactMatch(null as any, []).pass).toBe(false) + expect(assertTraceExactMatch([], null as any).pass).toBe(false) + }) + }) + + describe('assertTraceStructuralMatch', () => { + it('passes when event types match in order regardless of other fields', () => { + const actual = [{ type: 'a', ts: 'now', id: '1' }, { type: 'b', ts: 'later', id: '2' }] + const expected = [{ type: 'a', ts: 'then', id: '99' }, { type: 'b', ts: 'whenever', id: '100' }] + const result = assertTraceStructuralMatch(actual, expected) + expect(result.pass).toBe(true) + }) + + it('fails when event types differ', () => { + const result = assertTraceStructuralMatch( + [{ type: 'a' }, { type: 'c' }], + [{ type: 'a' }, { type: 'b' }], + ) + expect(result.pass).toBe(false) + expect(result.errors[0]).toContain('Event[1].type') + }) + + it('supports custom shape fields', () => { + const actual = [{ type: 'a', stepType: 'create' }] + const expected = [{ type: 'a', stepType: 'delete' }] + const result = assertTraceStructuralMatch(actual, expected, { + shapeFields: ['type', 'stepType'], + }) + expect(result.pass).toBe(false) + expect(result.errors[0]).toContain('stepType') + }) + }) + + describe('assertTraceStepOrdering', () => { + it('passes when required sequence appears in order', () => { + const events = [ + { type: 'scenario.start' }, + { type: 'noise' }, + { type: 'scenario.step.ok' }, + { type: 'more.noise' }, + { type: 'scenario.end' }, + ] + const result = assertTraceStepOrdering(events, [ + 'scenario.start', + 'scenario.step.ok', + 'scenario.end', + ]) + expect(result.pass).toBe(true) + }) + + it('fails when sequence is incomplete', () => { + const events = [{ type: 'scenario.start' }, { type: 'scenario.end' }] + const result = assertTraceStepOrdering(events, [ + 'scenario.start', + 'scenario.step.ok', + 'scenario.end', + ]) + expect(result.pass).toBe(false) + expect(result.errors[0]).toContain('Missing from index 1') + }) + + it('passes with empty required sequence', () => { + expect(assertTraceStepOrdering([], []).pass).toBe(true) + }) + }) + + describe('assertNoUnexpectedErrors', () => { + it('passes when no error events exist', () => { + const events = [{ type: 'scenario.start' }, { type: 'scenario.step.ok' }] + expect(assertNoUnexpectedErrors(events).pass).toBe(true) + }) + + it('fails when unexpected error events exist', () => { + const events = [ + { type: 'scenario.start' }, + { type: 'scenario.step.error', error: 'timeout' }, + ] + const result = assertNoUnexpectedErrors(events) + expect(result.pass).toBe(false) + expect(result.errors[0]).toContain('scenario.step.error') + expect(result.errors[0]).toContain('timeout') + }) + + it('allows explicitly permitted error types', () => { + const events = [{ type: 'autopilot.turn.error' }] + const result = assertNoUnexpectedErrors(events, { + allowedErrorTypes: ['autopilot.turn.error'], + }) + expect(result.pass).toBe(true) + }) + }) + + describe('assertRequiredEventsPresent', () => { + it('passes when all required types are present', () => { + const events = [ + { type: 'scenario.start' }, + { type: 'scenario.step.ok' }, + { type: 'scenario.end' }, + ] + const result = assertRequiredEventsPresent(events, ['scenario.start', 'scenario.end']) + expect(result.pass).toBe(true) + }) + + it('fails when required type is missing', () => { + const events = [{ type: 'scenario.start' }] + const result = assertRequiredEventsPresent(events, ['scenario.start', 'scenario.end']) + expect(result.pass).toBe(false) + expect(result.errors[0]).toContain('scenario.end') + }) + }) + + describe('assertTrace (combined)', () => { + it('runs all checks and aggregates errors', () => { + const events = [ + { type: 'scenario.start' }, + { type: 'scenario.step.error', error: 'boom' }, + ] + + const result = assertTrace(events, { + requiredSequence: ['scenario.start', 'scenario.end'], + requiredEvents: ['scenario.end'], + allowedErrorTypes: [], + }) + + expect(result.pass).toBe(false) + // Should have errors from ordering, required events, and unexpected errors + expect(result.errors.length).toBeGreaterThanOrEqual(3) + }) + + it('passes a clean trace with valid expectations', () => { + const events = [ + { type: 'scenario.start' }, + { type: 'scenario.step.ok' }, + { type: 'scenario.end' }, + ] + + const result = assertTrace(events, { + requiredSequence: ['scenario.start', 'scenario.end'], + requiredEvents: ['scenario.start', 'scenario.end'], + }) + expect(result.pass).toBe(true) + }) + }) +})