Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/commands/diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
diffArtifacts,
readArtifact,
} from "../index.js";
import { buildEvent, recordEvent } from "../telemetry.js";
import { formatOutput, writeOutput } from "./helpers.js";

export function registerDiffCommands(program: Command): void {
Expand All @@ -29,10 +30,19 @@ export function registerDiffCommands(program: Command): void {
throw new Error("The diff command only accepts run artifacts.");
}

const t0 = Date.now();
const artifact = diffArtifacts(baseArtifact, headArtifact);
const output = formatOutput(artifact, options.format);
await writeOutput(output, options.format, options.output);

recordEvent(buildEvent("command_complete", "diff", "cli", {
gateResult: artifact.gate,
executionMs: Date.now() - t0,
targetIds: [headArtifact.target.targetId],
healthScore: headArtifact.healthScore?.overall,
healthGrade: headArtifact.healthScore?.grade,
}));

if (options.failOnRegression && artifact.gate === "fail") {
process.exitCode = 1;
}
Expand Down
26 changes: 26 additions & 0 deletions src/commands/legacy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import {
runTarget,
writeRunArtifact,
} from "../index.js";
import { appendHistory, buildHistoryEntry } from "../history.js";
import { defaultRunsDirectory } from "../storage.js";
import { buildEvent, recordEvent } from "../telemetry.js";
import { ANSI, c, colorStatus, formatOutput, resolveTarget, writeOutput } from "./helpers.js";
import { runWatchMode } from "./watch.js";

Expand All @@ -26,10 +28,34 @@ export function registerLegacyCommands(program: Command): void {
await runWatchMode(target, options.outDir, parseInt(options.interval, 10) || 30);
return;
}
const t0 = Date.now();
const artifact = await runTarget(target, { invokeTools: options.invokeTools });
const outPath = await writeRunArtifact(artifact, options.outDir);
const summary = renderTerminal(artifact);
process.stdout.write(`${summary}\nArtifact: ${outPath}\n`);

// Track history + telemetry
await appendHistory(buildHistoryEntry(artifact)).catch(() => {});
const toolCount = artifact.checks.find(ch => ch.id === "tools")?.evidence[0]?.itemCount ?? 0;
const promptCount = artifact.checks.find(ch => ch.id === "prompts")?.evidence[0]?.itemCount ?? 0;
const resourceCount = artifact.checks.find(ch => ch.id === "resources")?.evidence[0]?.itemCount ?? 0;
const checkStatuses: Record<string, string> = {};
for (const ch of artifact.checks) checkStatuses[ch.id] = ch.status;
recordEvent(buildEvent("command_complete", "run", "cli", {
serversScanned: 1,
toolsFound: toolCount,
promptsFound: promptCount,
resourcesFound: resourceCount,
gateResult: artifact.gate,
executionMs: Date.now() - t0,
targetIds: [target.targetId],
healthScore: artifact.healthScore?.overall,
healthGrade: artifact.healthScore?.grade,
connectMs: artifact.performanceMetrics?.connectMs,
checkStatuses,
fatalError: artifact.fatalError?.split("\n")[0],
}));

if (artifact.gate === "fail") {
process.exitCode = 1;
}
Expand Down
15 changes: 15 additions & 0 deletions src/commands/watch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { renderWatchFirstRun, renderWatchNoChanges, renderWatchChanges } from ".
import { isCI } from "../ci.js";
import { defaultRunsDirectory, findLatestArtifact, readArtifact } from "../storage.js";
import { appendHistory, buildHistoryEntry } from "../history.js";
import { buildEvent, recordEvent } from "../telemetry.js";
import { ANSI, c, formatOutput, targetFromCommand } from "./helpers.js";

// ── One-shot mode ────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -64,6 +65,20 @@ async function runWatchOneShot(
process.stdout.write(renderWatchFirstRun(artifact) + "\n");
process.stdout.write(`${c(ANSI.dim, `Artifact: ${outPath}`)}\n`);

// Telemetry for watch one-shot
const toolCount = artifact.checks.find(ch => ch.id === "tools")?.evidence[0]?.itemCount ?? 0;
const checkStatuses: Record<string, string> = {};
for (const ch of artifact.checks) checkStatuses[ch.id] = ch.status;
recordEvent(buildEvent("command_complete", "watch", "cli", {
serversScanned: 1,
toolsFound: toolCount,
gateResult: artifact.gate,
targetIds: [target.targetId],
healthScore: artifact.healthScore?.overall,
healthGrade: artifact.healthScore?.grade,
checkStatuses,
}));

if (artifact.gate === "fail") {
process.exitCode = 1;
}
Expand Down
Loading