From 7bcdaf24bad4b515f30e967c67535acd996611c7 Mon Sep 17 00:00:00 2001 From: William Weishuhn Date: Mon, 23 Mar 2026 14:08:46 -0700 Subject: [PATCH] fix: add telemetry completion events to run, diff, and watch commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These 3 commands account for 78% of all usage (813 run + 735 diff + 13 watch events today) but fire only command_run — never command_complete. This means we get zero enrichment data: no gate results, no tool counts, no health scores, no target IDs, no execution times. Now all three fire command_complete with full enrichment: - run: servers, tools, prompts, resources, gate, health, connect time - diff: gate, target, health score - watch: servers, tools, gate, health, check statuses Co-Authored-By: Claude Opus 4.6 (1M context) --- src/commands/diff.ts | 10 ++++++++++ src/commands/legacy.ts | 26 ++++++++++++++++++++++++++ src/commands/watch.ts | 15 +++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/src/commands/diff.ts b/src/commands/diff.ts index cda55cc..a96679c 100644 --- a/src/commands/diff.ts +++ b/src/commands/diff.ts @@ -4,6 +4,7 @@ import { diffArtifacts, readArtifact, } from "../index.js"; +import { buildEvent, recordEvent } from "../telemetry.js"; import { formatOutput, writeOutput } from "./helpers.js"; export function registerDiffCommands(program: Command): void { @@ -29,10 +30,19 @@ export function registerDiffCommands(program: Command): void { throw new Error("The diff command only accepts run artifacts."); } + const t0 = Date.now(); const artifact = diffArtifacts(baseArtifact, headArtifact); const output = formatOutput(artifact, options.format); await writeOutput(output, options.format, options.output); + recordEvent(buildEvent("command_complete", "diff", "cli", { + gateResult: artifact.gate, + executionMs: Date.now() - t0, + targetIds: [headArtifact.target.targetId], + healthScore: headArtifact.healthScore?.overall, + healthGrade: headArtifact.healthScore?.grade, + })); + if (options.failOnRegression && artifact.gate === "fail") { process.exitCode = 1; } diff --git a/src/commands/legacy.ts b/src/commands/legacy.ts index 9944b0b..2689582 100644 --- a/src/commands/legacy.ts +++ b/src/commands/legacy.ts @@ -6,7 +6,9 @@ import { runTarget, writeRunArtifact, } from "../index.js"; +import { appendHistory, buildHistoryEntry } from "../history.js"; import { defaultRunsDirectory } from "../storage.js"; +import { buildEvent, recordEvent } from "../telemetry.js"; import { ANSI, c, colorStatus, formatOutput, resolveTarget, writeOutput } from "./helpers.js"; import { runWatchMode } from "./watch.js"; @@ -26,10 +28,34 @@ export function registerLegacyCommands(program: Command): void { await runWatchMode(target, options.outDir, parseInt(options.interval, 10) || 30); return; } + const t0 = Date.now(); const artifact = await runTarget(target, { invokeTools: options.invokeTools }); const outPath = await writeRunArtifact(artifact, options.outDir); const summary = renderTerminal(artifact); process.stdout.write(`${summary}\nArtifact: ${outPath}\n`); + + // Track history + telemetry + await appendHistory(buildHistoryEntry(artifact)).catch(() => {}); + const toolCount = artifact.checks.find(ch => ch.id === "tools")?.evidence[0]?.itemCount ?? 0; + const promptCount = artifact.checks.find(ch => ch.id === "prompts")?.evidence[0]?.itemCount ?? 0; + const resourceCount = artifact.checks.find(ch => ch.id === "resources")?.evidence[0]?.itemCount ?? 0; + const checkStatuses: Record = {}; + for (const ch of artifact.checks) checkStatuses[ch.id] = ch.status; + recordEvent(buildEvent("command_complete", "run", "cli", { + serversScanned: 1, + toolsFound: toolCount, + promptsFound: promptCount, + resourcesFound: resourceCount, + gateResult: artifact.gate, + executionMs: Date.now() - t0, + targetIds: [target.targetId], + healthScore: artifact.healthScore?.overall, + healthGrade: artifact.healthScore?.grade, + connectMs: artifact.performanceMetrics?.connectMs, + checkStatuses, + fatalError: artifact.fatalError?.split("\n")[0], + })); + if (artifact.gate === "fail") { process.exitCode = 1; } diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 09e0142..e462c1f 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -9,6 +9,7 @@ import { renderWatchFirstRun, renderWatchNoChanges, renderWatchChanges } from ". import { isCI } from "../ci.js"; import { defaultRunsDirectory, findLatestArtifact, readArtifact } from "../storage.js"; import { appendHistory, buildHistoryEntry } from "../history.js"; +import { buildEvent, recordEvent } from "../telemetry.js"; import { ANSI, c, formatOutput, targetFromCommand } from "./helpers.js"; // ── One-shot mode ──────────────────────────────────────────────────────────── @@ -64,6 +65,20 @@ async function runWatchOneShot( process.stdout.write(renderWatchFirstRun(artifact) + "\n"); process.stdout.write(`${c(ANSI.dim, `Artifact: ${outPath}`)}\n`); + // Telemetry for watch one-shot + const toolCount = artifact.checks.find(ch => ch.id === "tools")?.evidence[0]?.itemCount ?? 0; + const checkStatuses: Record = {}; + for (const ch of artifact.checks) checkStatuses[ch.id] = ch.status; + recordEvent(buildEvent("command_complete", "watch", "cli", { + serversScanned: 1, + toolsFound: toolCount, + gateResult: artifact.gate, + targetIds: [target.targetId], + healthScore: artifact.healthScore?.overall, + healthGrade: artifact.healthScore?.grade, + checkStatuses, + })); + if (artifact.gate === "fail") { process.exitCode = 1; }