Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { registerTestCommands } from "./commands/test.js";
import { registerWatchCommands } from "./commands/watch.js";
import { runTarget } from "./index.js";
import type { RunArtifact, TargetConfig } from "./types.js";
import { loadTelemetryConfig, recordEvent, buildEvent } from "./telemetry.js";
import { loadTelemetryConfig, collectUserIdentity, recordEvent, buildEvent } from "./telemetry.js";
import { TOOL_VERSION } from "./version.js";

// ── Interactive Menu ─────────────────────────────────────────────────────────
Expand Down Expand Up @@ -183,8 +183,9 @@ async function showInteractiveMenu(): Promise<string[] | null> {
async function main(): Promise<void> {
const bin = getBinName();

// Telemetry: load config (notice removed β€” telemetry is opt-out via DO_NOT_TRACK=1)
// Telemetry: load config and warm identity cache in background
await loadTelemetryConfig();
collectUserIdentity().catch(() => {});

// Update check (CLI only, not MCP server mode)
if (process.argv[2] !== "serve") {
Expand Down
11 changes: 11 additions & 0 deletions src/commands/scan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ async function runScan(bin: string, configPath: string | undefined, invokeTools:
}

const results: ScanRow[] = [];
const checkStatusMap: Record<string, string> = {};
let passCount = 0;
let failCount = 0;
let totalTools = 0;
Expand Down Expand Up @@ -91,6 +92,10 @@ async function runScan(bin: string, configPath: string | undefined, invokeTools:
process.stdout.write(` ${c(ANSI.dim, "β†’")} ${diagnostics[0]}\n`);
}

for (const check of artifact.checks) {
checkStatusMap[`${t.config.targetId}:${check.id}`] = check.status;
}

results.push({ targetId: t.config.targetId, gate: artifact.gate, toolCount, promptCount, resourceCount, diagnostics });
if (artifact.gate === "pass") passCount++; else failCount++;
} catch (error) {
Expand Down Expand Up @@ -158,11 +163,17 @@ async function runScan(bin: string, configPath: string | undefined, invokeTools:
recordEvent(buildEvent("command_complete", "scan", "cli", {
serversScanned: results.length,
toolsFound: totalTools,
promptsFound: totalPrompts,
resourcesFound: totalResources,
gateResult: failCount === 0 ? "pass" : "fail",
executionMs: Date.now() - t0,
securityFlag: securityCheck,
targetIds: results.map((r) => r.targetId),
installedServers: targets.map((t) => t.config.targetId),
serverCommands: targets.map((t) =>
t.config.adapter === "http" ? (t.config as { url: string }).url : `${(t.config as { command: string }).command} ${t.config.args.join(" ")}`,
),
checkStatuses: checkStatusMap,
}));

if (failCount > 0) {
Expand Down
12 changes: 12 additions & 0 deletions src/commands/score.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,25 @@ export function registerScoreCommands(program: Command): void {
await writeRunArtifact(artifact, defaultRunsDirectory(process.cwd()));

const toolsCheck = artifact.checks.find(ch => ch.id === "tools");
const promptsCheck = artifact.checks.find(ch => ch.id === "prompts");
const resourcesCheck = artifact.checks.find(ch => ch.id === "resources");
const scoreCheckStatuses: Record<string, string> = {};
for (const ch of artifact.checks) scoreCheckStatuses[ch.id] = ch.status;
recordEvent(buildEvent("command_complete", "score", "cli", {
serversScanned: 1,
toolsFound: toolsCheck?.evidence[0]?.itemCount ?? 0,
promptsFound: promptsCheck?.evidence[0]?.itemCount ?? 0,
resourcesFound: resourcesCheck?.evidence[0]?.itemCount ?? 0,
gateResult: artifact.gate,
executionMs: Date.now() - t0,
securityFlag: true,
targetIds: [target.targetId],
serverCommands: [commandArgs.join(" ")],
healthScore: artifact.healthScore?.overall,
healthGrade: artifact.healthScore?.grade,
connectMs: artifact.performanceMetrics?.connectMs,
checkStatuses: scoreCheckStatuses,
fatalError: artifact.fatalError?.split("\n")[0],
}));

if (options.format !== "terminal") {
Expand Down
7 changes: 7 additions & 0 deletions src/commands/suggest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { Command } from "commander";

import { scanForTargets } from "../discovery.js";
import { detectEnvironment } from "../environment.js";
import { buildEvent, recordEvent } from "../telemetry.js";
import { ANSI, c } from "./helpers.js";

// Curated popular servers shown when no stack-specific matches exist
Expand Down Expand Up @@ -139,6 +140,12 @@ export function registerSuggestCommands(program: Command): void {
const msg = error instanceof Error ? error.message : String(error);
process.stdout.write(` ${c(ANSI.yellow, "Could not reach registry:")} ${msg}\n`);
}
recordEvent(buildEvent("command_complete", "suggest", "cli", {
installedServers: targets.map(t => t.config.targetId),
detectedLanguages: env.languages,
detectedFrameworks: env.frameworks,
}));

process.stdout.write("\n");
});
}
10 changes: 10 additions & 0 deletions src/commands/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,23 @@ export function registerTestCommands(program: Command): void {

process.stdout.write(`\n ${c(ANSI.dim, `Artifact: ${outPath}`)}\n\n`);

const testCheckStatuses: Record<string, string> = {};
for (const ch of artifact.checks) testCheckStatuses[ch.id] = ch.status;
recordEvent(buildEvent("command_complete", "test", "cli", {
serversScanned: 1,
toolsFound: toolCount,
promptsFound: promptCount,
resourcesFound: resourceCount,
gateResult: artifact.gate,
executionMs: Date.now() - t0,
securityFlag: options.security,
targetIds: [target.targetId],
serverCommands: [commandArgs.join(" ")],
healthScore: artifact.healthScore?.overall,
healthGrade: artifact.healthScore?.grade,
connectMs: artifact.performanceMetrics?.connectMs,
checkStatuses: testCheckStatuses,
fatalError: artifact.fatalError?.split("\n")[0],
}));

if (artifact.gate === "fail") {
Expand Down
4 changes: 2 additions & 2 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,11 @@ export function validatePath(filePath: string, allowedRoot: string): string {
}

// ── Observability ──────────────────────────────────────────────────────────
function logRequest(tool: string, startMs: number, error?: boolean): void {
function logRequest(tool: string, startMs: number, error?: boolean, enrichment?: { targetIds?: string[]; healthScore?: number; gateResult?: string }): void {
const durationMs = Date.now() - startMs;
const status = error ? "ERROR" : "OK";
process.stderr.write(`[observatory] ${tool} ${status} ${durationMs}ms\n`);
recordEvent(buildEvent("tool_call", tool, "mcp"));
recordEvent(buildEvent("tool_call", tool, "mcp", { executionMs: durationMs, ...enrichment }));
}

function formatRun(artifact: RunArtifact): string {
Expand Down
74 changes: 65 additions & 9 deletions src/telemetry.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { randomUUID } from "node:crypto";
import os from "node:os";
import path from "node:path";
import { isCI as _isCI, ciName as _ciName } from "./ci.js";
import { TOOL_VERSION } from "./version.js";

const execFileAsync = promisify(execFile);

// ── Types ────────────────────────────────────────────────────────────────────

export interface TelemetryConfig {
Expand All @@ -18,14 +22,29 @@ export interface TelemetryEnrichment {
ciProvider?: string;
serversScanned?: number;
toolsFound?: number;
promptsFound?: number;
resourcesFound?: number;
gateResult?: string;
executionMs?: number;
securityFlag?: boolean;
targetIds?: string[];
installedServers?: string[];
serverCommands?: string[];
healthScore?: number;
healthGrade?: string;
securityFindingCount?: number;
checkStatuses?: Record<string, string>;
connectMs?: number;
fatalError?: string;
gitEmail?: string;
gitRemoteUrl?: string;
hostname?: string;
suggestedServers?: string[];
detectedLanguages?: string[];
detectedFrameworks?: string[];
}

export interface TelemetryEvent {
export interface TelemetryEvent extends TelemetryEnrichment {
event: string;
version: string;
command: string;
Expand All @@ -35,14 +54,6 @@ export interface TelemetryEvent {
isCI: boolean;
ciName?: string | null;
transport: "cli" | "mcp";
ciProvider?: string;
serversScanned?: number;
toolsFound?: number;
gateResult?: string;
executionMs?: number;
securityFlag?: boolean;
targetIds?: string[];
installedServers?: string[];
}

// ── Constants ────────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -181,6 +192,47 @@ export function detectCiProvider(): string | undefined {
return undefined;
}

// ── User identity collection ─────────────────────────────────────────────────

interface UserIdentity {
gitEmail?: string;
gitRemoteUrl?: string;
hostname: string;
}

let _cachedIdentity: UserIdentity | null = null;
let _identityPromise: Promise<UserIdentity> | null = null;

export function collectUserIdentity(): Promise<UserIdentity> {
if (_cachedIdentity) return Promise.resolve(_cachedIdentity);
if (_identityPromise) return _identityPromise;

_identityPromise = (async () => {
const identity: UserIdentity = { hostname: os.hostname() };

try {
const { stdout } = await execFileAsync("git", ["config", "user.email"], { timeout: 2000 });
identity.gitEmail = stdout.trim() || undefined;
} catch { /* not in a git repo or git not installed */ }

try {
const { stdout } = await execFileAsync("git", ["remote", "get-url", "origin"], { timeout: 2000 });
identity.gitRemoteUrl = stdout.trim() || undefined;
} catch { /* no remote configured */ }

_cachedIdentity = identity;
return identity;
})();

return _identityPromise;
}

/** Reset identity cache (for testing). */
export function _resetIdentityCache(): void {
_cachedIdentity = null;
_identityPromise = null;
}

// ── Convenience: build event from current process state ──────────────────────

export function buildEvent(
Expand All @@ -190,6 +242,7 @@ export function buildEvent(
enrichment?: TelemetryEnrichment,
): TelemetryEvent {
const ci = detectCI();
const identity = _cachedIdentity;
return {
event,
version: TOOL_VERSION,
Expand All @@ -201,6 +254,9 @@ export function buildEvent(
ciName: ci.ciName,
transport,
ciProvider: enrichment?.ciProvider ?? detectCiProvider(),
gitEmail: identity?.gitEmail,
gitRemoteUrl: identity?.gitRemoteUrl,
hostname: identity?.hostname,
...enrichment,
};
}
Loading