diff --git a/.github/workflows/nightly-scan.template.yml b/.github/workflows/nightly-scan.template.yml new file mode 100644 index 0000000..a4cd89e --- /dev/null +++ b/.github/workflows/nightly-scan.template.yml @@ -0,0 +1,50 @@ +# MCP Observatory Nightly Scan +# Copy this file to .github/workflows/observatory-nightly.yml in your repo. +# It scans your MCP servers daily and opens an issue if regressions are found. +name: MCP Observatory Nightly Scan +on: + schedule: + - cron: '0 6 * * *' # 6 AM UTC daily + workflow_dispatch: {} + +jobs: + scan: + runs-on: ubuntu-latest + permissions: + issues: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Install MCP Observatory + run: npm install -g @kryptosai/mcp-observatory + - name: Run nightly scan + run: mcp-observatory scan --no-color + - name: Generate CI report + id: report + run: | + mcp-observatory ci-report --format json > /tmp/observatory-report.json + echo "has_regressions=$(node -e "console.log(JSON.parse(require('fs').readFileSync('/tmp/observatory-report.json','utf8')).hasRegressions)")" >> $GITHUB_OUTPUT + - name: Create or update issue + if: steps.report.outputs.has_regressions == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TITLE=$(node -e "console.log(JSON.parse(require('fs').readFileSync('/tmp/observatory-report.json','utf8')).title)") + BODY_FILE="/tmp/observatory-body.md" + node -e "process.stdout.write(JSON.parse(require('fs').readFileSync('/tmp/observatory-report.json','utf8')).body)" > "$BODY_FILE" + EXISTING=$(gh issue list --label "mcp-observatory" --state open --json number -q '.[0].number // empty' 2>/dev/null || echo "") + if [ -n "$EXISTING" ]; then + gh issue comment "$EXISTING" --body-file "$BODY_FILE" + else + gh issue create --title "$TITLE" --body-file "$BODY_FILE" --label "mcp-observatory" + fi + - name: Upload scan artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: observatory-nightly-${{ github.run_number }} + path: .mcp-observatory/runs/ + retention-days: 30 diff --git a/action/action.yml b/action/action.yml index a404bb7..6f7c04f 100644 --- a/action/action.yml +++ b/action/action.yml @@ -11,6 +11,9 @@ inputs: target: description: "Path to a target config JSON file (alternative to command)" required: false + targets: + description: "Path to MCP config file for multi-server matrix scan (scans all servers, generates matrix comment)" + required: false baseline: description: "Path to a baseline cassette file for regression verification" required: false @@ -27,11 +30,15 @@ inputs: required: false default: "true" comment-on-pr: - description: "Post a markdown report as a PR comment" + description: "Post a report as a PR comment" + required: false + default: "true" + set-status: + description: "Set a commit status check (green/red) on the HEAD SHA" required: false default: "true" github-token: - description: "GitHub token for PR comments" + description: "GitHub token for PR comments and commit statuses" required: false default: ${{ github.token }} node-version: @@ -65,57 +72,79 @@ runs: env: INPUT_COMMAND: ${{ inputs.command }} INPUT_TARGET: ${{ inputs.target }} + INPUT_TARGETS: ${{ inputs.targets }} INPUT_DEEP: ${{ inputs.deep }} INPUT_SECURITY: ${{ inputs.security }} run: | - # Build command as a bash array to prevent injection - CMD_ARRAY=() - - if [ -n "$INPUT_TARGET" ]; then - CMD_ARRAY=(mcp-observatory run --target "$INPUT_TARGET") - if [ "$INPUT_DEEP" = "true" ]; then - CMD_ARRAY+=(--invoke) - fi - elif [ -n "$INPUT_COMMAND" ]; then - CMD_ARRAY=(mcp-observatory test "$INPUT_COMMAND") - else - echo "::error::Either 'command' or 'target' input is required" - exit 1 - fi - - if [ "$INPUT_SECURITY" = "true" ]; then - CMD_ARRAY+=(--security) - fi - - CMD_ARRAY+=(--no-color) - - # Run and capture output ARTIFACT_DIR="${RUNNER_TEMP}/observatory" mkdir -p "$ARTIFACT_DIR" - set +e - "${CMD_ARRAY[@]}" 2>&1 | tee "${ARTIFACT_DIR}/output.txt" - EXIT_CODE=$? - set -e - - # Find the artifact file - ARTIFACT_PATH=$(find .mcp-observatory/runs -name "*.json" -type f 2>/dev/null | sort | tail -1) + # Multi-server matrix scan + if [ -n "$INPUT_TARGETS" ]; then + SCAN_ARRAY=(mcp-observatory scan --config "$INPUT_TARGETS") + if [ "$INPUT_SECURITY" = "true" ]; then + SCAN_ARRAY+=(--security) + fi + SCAN_ARRAY+=(--no-color) + + set +e + "${SCAN_ARRAY[@]}" 2>&1 | tee "${ARTIFACT_DIR}/output.txt" + set -e + + # Collect all artifacts and determine gate + GATE="pass" + for f in .mcp-observatory/runs/*.json; do + [ -f "$f" ] || continue + FILE_GATE=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$f','utf8')).gate)" 2>/dev/null || echo "unknown") + if [ "$FILE_GATE" = "fail" ]; then + GATE="fail" + fi + done + echo "gate=${GATE}" >> "$GITHUB_OUTPUT" + echo "artifact_path=${ARTIFACT_DIR}" >> "$GITHUB_OUTPUT" - if [ -n "$ARTIFACT_PATH" ]; then - cp "$ARTIFACT_PATH" "${ARTIFACT_DIR}/run.json" - echo "artifact_path=${ARTIFACT_DIR}/run.json" >> "$GITHUB_OUTPUT" + # Generate CI report for PR comment + mcp-observatory ci-report --format markdown --no-color > "${ARTIFACT_DIR}/report.md" 2>/dev/null || true - # Extract gate from artifact using node (avoid python dependency) - GATE=$(node -e "const fs=require('fs'); const d=JSON.parse(fs.readFileSync('${ARTIFACT_DIR}/run.json','utf8')); console.log(d.gate)" 2>/dev/null || echo "unknown") - echo "gate=${GATE}" >> "$GITHUB_OUTPUT" + # Single-server scan else - echo "gate=fail" >> "$GITHUB_OUTPUT" - echo "artifact_path=" >> "$GITHUB_OUTPUT" - fi + CMD_ARRAY=() + if [ -n "$INPUT_TARGET" ]; then + CMD_ARRAY=(mcp-observatory run --target "$INPUT_TARGET") + if [ "$INPUT_DEEP" = "true" ]; then + CMD_ARRAY+=(--invoke) + fi + elif [ -n "$INPUT_COMMAND" ]; then + CMD_ARRAY=(mcp-observatory test "$INPUT_COMMAND") + else + echo "::error::Either 'command', 'target', or 'targets' input is required" + exit 1 + fi - # Generate PR comment report - if [ -n "$ARTIFACT_PATH" ]; then - mcp-observatory report "$ARTIFACT_PATH" --format pr-comment --no-color > "${ARTIFACT_DIR}/report.md" 2>/dev/null || true + if [ "$INPUT_SECURITY" = "true" ]; then + CMD_ARRAY+=(--security) + fi + CMD_ARRAY+=(--no-color) + + set +e + "${CMD_ARRAY[@]}" 2>&1 | tee "${ARTIFACT_DIR}/output.txt" + set -e + + ARTIFACT_PATH=$(find .mcp-observatory/runs -name "*.json" -type f 2>/dev/null | sort | tail -1) + if [ -n "$ARTIFACT_PATH" ]; then + cp "$ARTIFACT_PATH" "${ARTIFACT_DIR}/run.json" + echo "artifact_path=${ARTIFACT_DIR}/run.json" >> "$GITHUB_OUTPUT" + GATE=$(node -e "const fs=require('fs'); const d=JSON.parse(fs.readFileSync('${ARTIFACT_DIR}/run.json','utf8')); console.log(d.gate)" 2>/dev/null || echo "unknown") + echo "gate=${GATE}" >> "$GITHUB_OUTPUT" + else + echo "gate=fail" >> "$GITHUB_OUTPUT" + echo "artifact_path=" >> "$GITHUB_OUTPUT" + fi + + # Generate PR comment report + if [ -n "$ARTIFACT_PATH" ]; then + mcp-observatory report "$ARTIFACT_PATH" --format pr-comment --no-color > "${ARTIFACT_DIR}/report.md" 2>/dev/null || true + fi fi - name: Verify against baseline @@ -126,15 +155,12 @@ runs: INPUT_COMMAND: ${{ inputs.command }} INPUT_TARGET: ${{ inputs.target }} run: | - # Build verify command as a bash array VERIFY_ARRAY=(mcp-observatory verify "$INPUT_BASELINE") - if [ -n "$INPUT_TARGET" ]; then VERIFY_ARRAY+=(--target "$INPUT_TARGET") elif [ -n "$INPUT_COMMAND" ]; then VERIFY_ARRAY+=("$INPUT_COMMAND") fi - VERIFY_ARRAY+=(--no-color) set +e @@ -146,6 +172,26 @@ runs: echo "::warning::Baseline verification detected changes" fi + - name: Set commit status + if: inputs.set-status == 'true' && github.event_name == 'pull_request' + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + GATE: ${{ steps.run.outputs.gate }} + run: | + STATE="success" + DESC="All clear" + if [ "$GATE" = "fail" ]; then + STATE="failure" + DESC="Issues detected" + fi + gh api "repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }}" \ + -f state="$STATE" \ + -f description="$DESC" \ + -f context="MCP Observatory" \ + -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + 2>/dev/null || echo "::warning::Could not set commit status" + - name: Comment on PR if: inputs.comment-on-pr == 'true' && github.event_name == 'pull_request' shell: bash @@ -161,7 +207,6 @@ runs: exit 0 fi - # pr-comment format already includes header and footer COMMENT_FILE="${RUNNER_TEMP}/observatory/comment.md" cp "$REPORT_FILE" "$COMMENT_FILE" diff --git a/src/ci-issue.ts b/src/ci-issue.ts new file mode 100644 index 0000000..e5cf972 --- /dev/null +++ b/src/ci-issue.ts @@ -0,0 +1,135 @@ +import { execFile } from "node:child_process"; +import { writeFile, unlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +/** + * Run a command via execFile and return { stdout, stderr }. + */ +function execCommand( + cmd: string, + args: string[], +): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + execFile(cmd, args, (error, stdout, stderr) => { + if (error) { + reject(error as Error); + } else { + resolve({ stdout: stdout ?? "", stderr: stderr ?? "" }); + } + }); + }); +} + +/** Executor type for dependency injection (testability). */ +export type CommandExecutor = ( + cmd: string, + args: string[], +) => Promise<{ stdout: string; stderr: string }>; + +/** + * Find an existing open issue with the given label. + * Returns the issue number or null if none found / gh unavailable. + */ +export async function findExistingIssue( + repo: string, + label: string, + exec: CommandExecutor = execCommand, +): Promise { + try { + const { stdout } = await exec("gh", [ + "issue", + "list", + "--repo", + repo, + "--label", + label, + "--state", + "open", + "--json", + "number", + "--limit", + "1", + ]); + const parsed: unknown = JSON.parse(stdout); + if (Array.isArray(parsed) && parsed.length > 0) { + const first = parsed[0] as Record; + if (typeof first["number"] === "number") { + return first["number"]; + } + } + return null; + } catch { + return null; + } +} + +/** + * Create a new issue or comment on an existing one. + * Uses --body-file to avoid shell injection. + * Returns the issue number. + */ +export async function createOrUpdateIssue(options: { + repo: string; + title: string; + body: string; + labels: string[]; + exec?: CommandExecutor; +}): Promise { + const exec = options.exec ?? execCommand; + const bodyFile = path.join( + tmpdir(), + `mcp-observatory-issue-${Date.now()}.md`, + ); + + try { + await writeFile(bodyFile, options.body, "utf8"); + + const existingNumber = await findExistingIssue( + options.repo, + options.labels[0] ?? "mcp-observatory", + exec, + ); + + if (existingNumber !== null) { + await exec("gh", [ + "issue", + "comment", + String(existingNumber), + "--repo", + options.repo, + "--body-file", + bodyFile, + ]); + return existingNumber; + } + + // Create new issue + const args = [ + "issue", + "create", + "--repo", + options.repo, + "--title", + options.title, + "--body-file", + bodyFile, + ]; + for (const label of options.labels) { + args.push("--label", label); + } + + const { stdout } = await exec("gh", args); + // gh issue create prints the URL, e.g. https://github.com/owner/repo/issues/42 + const match = /\/issues\/(\d+)/.exec(stdout.trim()); + if (match?.[1]) { + return parseInt(match[1], 10); + } + + throw new Error( + `Failed to parse issue number from gh output: ${stdout.trim()}`, + ); + } finally { + await unlink(bodyFile).catch(() => {}); + } +} diff --git a/src/ci.ts b/src/ci.ts index 9d2d600..ae217ec 100644 --- a/src/ci.ts +++ b/src/ci.ts @@ -2,3 +2,10 @@ import ci from "ci-info"; export const isCI: boolean = ci.isCI; export const ciName: string | null = ci.name; + +export function getGitHubContext(): { sha: string; repo: string } | null { + const sha = process.env["GITHUB_SHA"]; + const repo = process.env["GITHUB_REPOSITORY"]; + if (!sha || !repo) return null; + return { sha, repo }; +} diff --git a/src/cli.ts b/src/cli.ts index f31346b..f020168 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -15,6 +15,9 @@ import { registerSuggestCommands } from "./commands/suggest.js"; import { registerTelemetryCommands } from "./commands/telemetry.js"; import { registerTestCommands } from "./commands/test.js"; import { registerWatchCommands } from "./commands/watch.js"; +import { registerHistoryCommands } from "./commands/history.js"; +import { registerCiReportCommands } from "./commands/ci-report.js"; +import { registerLockCommands } from "./commands/lock.js"; import { runTarget } from "./index.js"; import type { RunArtifact, TargetConfig } from "./types.js"; import { loadTelemetryConfig, collectUserIdentity, recordEvent, buildEvent } from "./telemetry.js"; @@ -245,6 +248,9 @@ async function main(): Promise { registerScoreCommands(program); registerLegacyCommands(program); registerTelemetryCommands(program); + registerHistoryCommands(program); + registerCiReportCommands(program); + registerLockCommands(program); // ── smithery ───────────────────────────────────────────────────────── diff --git a/src/commands/ci-report.ts b/src/commands/ci-report.ts new file mode 100644 index 0000000..230907f --- /dev/null +++ b/src/commands/ci-report.ts @@ -0,0 +1,131 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import type { Command } from "commander"; +import type { RunArtifact } from "../types.js"; +import { validateRunArtifact } from "../validate.js"; +import { defaultRunsDirectory } from "../storage.js"; + +// ── Report shape ───────────────────────────────────────────────────────────── + +export interface CiReport { + title: string; + body: string; + labels: string[]; + hasRegressions: boolean; + serverCount: number; + failCount: number; +} + +// ── Build report from artifacts ────────────────────────────────────────────── + +export function buildCiReport(artifacts: RunArtifact[]): CiReport { + const today = new Date().toISOString().slice(0, 10); + const failing = artifacts.filter((a) => a.gate === "fail"); + const failCount = failing.length; + const hasRegressions = failCount > 0; + + const title = hasRegressions + ? `MCP Observatory: ${failCount} regression${failCount === 1 ? "" : "s"} detected (${today})` + : `MCP Observatory: all clear (${today})`; + + let body: string; + if (!hasRegressions) { + body = + artifacts.length === 0 + ? "No run artifacts found. Nothing to report." + : `All ${artifacts.length} server${artifacts.length === 1 ? "" : "s"} passed on ${today}.`; + } else { + const sections: string[] = []; + for (const artifact of failing) { + const targetId = artifact.target.targetId; + const lines: string[] = [`## ${targetId}`]; + + if (artifact.fatalError) { + lines.push("", `> **Fatal error:** ${artifact.fatalError.split("\n")[0]}`); + } + + const failingChecks = artifact.checks.filter( + (ch) => ch.status === "fail" || ch.status === "partial", + ); + for (const check of failingChecks) { + lines.push(`> **${check.id}:** ${check.message}`); + } + + if (failingChecks.length === 0 && !artifact.fatalError) { + lines.push("> Gate failed (no specific check failures recorded)."); + } + + sections.push(lines.join("\n")); + } + body = sections.join("\n\n"); + } + + return { + title, + body, + labels: ["mcp-observatory"], + hasRegressions, + serverCount: artifacts.length, + failCount, + }; +} + +// ── CLI command ────────────────────────────────────────────────────────────── + +export function registerCiReportCommands(program: Command): void { + program + .command("ci-report") + .description( + "Generate a CI report from run artifacts for GitHub issue creation.", + ) + .option( + "--artifacts-dir ", + "Directory containing run artifacts.", + defaultRunsDirectory(process.cwd()), + ) + .option("--format ", "Output format: json or markdown.", "json") + .option("--no-color", "Disable colored output.") + .action( + async (options: { artifactsDir: string; format: string }) => { + const artifacts = await loadArtifactsFromDir(options.artifactsDir); + const report = buildCiReport(artifacts); + + if (options.format === "markdown") { + process.stdout.write(report.body + "\n"); + } else { + process.stdout.write(JSON.stringify(report, null, 2) + "\n"); + } + + if (report.hasRegressions) { + process.exitCode = 1; + } + }, + ); +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +async function loadArtifactsFromDir(dir: string): Promise { + let entries: string[]; + try { + entries = await readdir(dir); + } catch { + return []; + } + + const jsonFiles = entries.filter((f) => f.endsWith(".json")).sort(); + const artifacts: RunArtifact[] = []; + + for (const file of jsonFiles) { + try { + const content = await readFile(path.join(dir, file), "utf8"); + const data: unknown = JSON.parse(content); + const artifact = validateRunArtifact(data); + artifacts.push(artifact); + } catch { + // Skip invalid files silently + } + } + + return artifacts; +} diff --git a/src/commands/history.ts b/src/commands/history.ts new file mode 100644 index 0000000..c8b35f2 --- /dev/null +++ b/src/commands/history.ts @@ -0,0 +1,62 @@ +import type { Command } from "commander"; +import { readHistory, getTrend, renderTrendLabel } from "../history.js"; +import { ANSI, c } from "./helpers.js"; + +export function registerHistoryCommands(program: Command): void { + program + .command("history") + .description("Show health score trends for your MCP servers.") + .option("--target ", "Filter to a specific server target ID.") + .option("--json", "Output raw JSON.", false) + .option("--no-color", "Disable colored output.") + .action(async (options: { target?: string; json: boolean }) => { + const history = await readHistory(); + + if (options.json) { + process.stdout.write(JSON.stringify(history, null, 2) + "\n"); + return; + } + + // Get unique target IDs preserving insertion order + let targetIds = [...new Set(history.entries.map((e) => e.targetId))]; + if (options.target) { + targetIds = targetIds.filter((id) => id === options.target); + } + + if (targetIds.length === 0) { + process.stdout.write( + "No history found. Run a scan or test to start tracking.\n", + ); + return; + } + + // Print header + process.stdout.write( + c(ANSI.bold, " Target") + + " " + + c(ANSI.bold, "Health") + + " " + + c(ANSI.bold, "Trend") + + "\n", + ); + + for (const id of targetIds) { + const trend = getTrend(id, history); + if (!trend) continue; + + const { current } = trend; + const gradeColor = + current.grade === "A" + ? ANSI.green + : current.grade === "F" + ? ANSI.red + : ANSI.yellow; + const label = renderTrendLabel(trend); + const paddedId = id.padEnd(30); + + process.stdout.write( + ` ${paddedId} ${c(gradeColor, current.grade)} (${current.healthScore}) ${label}\n`, + ); + } + }); +} diff --git a/src/commands/lock.ts b/src/commands/lock.ts new file mode 100644 index 0000000..55bfeb2 --- /dev/null +++ b/src/commands/lock.ts @@ -0,0 +1,152 @@ +import type { Command } from "commander"; + +import { scanForTargets } from "../discovery.js"; +import { + readLockFile, + writeLockFile, + buildServerLockEntry, + verifyAgainstLock, + mergeLockFile, +} from "../lockfile.js"; +import type { LockFileServerEntry } from "../lockfile.js"; +import { runTarget } from "../runner.js"; +import { ANSI, c, getBinName } from "./helpers.js"; + +// ── Register ──────────────────────────────────────────────────────────────── + +export function registerLockCommands(program: Command): void { + const lockCmd = program + .command("lock") + .description("Manage MCP server schema lock files."); + + lockCmd + .command("create", { isDefault: true }) + .description("Snapshot all MCP server schemas into a lock file.") + .option("--config ", "Path to MCP config file.") + .option("--update", "Merge with existing lock file instead of overwriting.", false) + .action(async (options: { config?: string; update: boolean }) => { + const targets = await scanForTargets(options.config); + + if (targets.length === 0) { + const bin = getBinName(); + process.stdout.write( + c(ANSI.yellow, " No MCP servers found.\n\n") + + c(ANSI.dim, " Looked in ~/.claude.json, Claude Desktop config, .mcp.json (+ parent dirs)\n\n") + + ` Test a specific server:\n` + + ` ${c(ANSI.dim, "$")} ${c(ANSI.cyan, `${bin} test npx -y @modelcontextprotocol/server-filesystem .`)}\n\n`, + ); + return; + } + + const entries: LockFileServerEntry[] = []; + + for (const t of targets) { + process.stdout.write(` ${c(ANSI.cyan, "⟳")} Locking ${t.config.targetId}...\n`); + try { + const artifact = await runTarget(t.config); + entries.push(buildServerLockEntry(artifact)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + process.stdout.write(` ${c(ANSI.red, "✗")} Failed to lock ${t.config.targetId}: ${msg}\n`); + } + } + + if (entries.length === 0) { + process.stdout.write(c(ANSI.red, "\n No servers could be locked.\n\n")); + process.exitCode = 1; + return; + } + + let lockFile; + if (options.update) { + try { + const existing = await readLockFile(); + lockFile = mergeLockFile(existing, entries); + } catch { + // No existing lock file — create fresh + lockFile = { + version: 1 as const, + lockedAt: new Date().toISOString(), + servers: entries, + }; + } + } else { + lockFile = { + version: 1 as const, + lockedAt: new Date().toISOString(), + servers: entries, + }; + } + + const lockPath = await writeLockFile(lockFile); + process.stdout.write( + `\n ${c(ANSI.green, "✓")} Locked ${entries.length} server${entries.length === 1 ? "" : "s"} to ${lockPath}\n\n`, + ); + }); + + lockCmd + .command("verify") + .description("Verify live servers match the lock file.") + .option("--config ", "Path to MCP config file.") + .action(async (options: { config?: string }) => { + let lock; + try { + lock = await readLockFile(); + } catch { + const bin = getBinName(); + process.stdout.write( + c(ANSI.red, " ✗ No lock file found.\n\n") + + ` Create one first:\n` + + ` ${c(ANSI.dim, "$")} ${c(ANSI.cyan, `${bin} lock`)}\n\n`, + ); + process.exitCode = 1; + return; + } + + const targets = await scanForTargets(options.config); + const lockMap = new Map( + lock.servers.map((s) => [s.targetId, s]), + ); + + let anyFailed = false; + + for (const t of targets) { + const lockEntry = lockMap.get(t.config.targetId); + if (!lockEntry) { + process.stdout.write( + ` ${c(ANSI.dim, "⊘")} ${t.config.targetId} ${c(ANSI.dim, "(not in lock file, skipping)")}\n`, + ); + continue; + } + + process.stdout.write(` ${c(ANSI.cyan, "⟳")} Verifying ${t.config.targetId}...\n`); + + try { + const artifact = await runTarget(t.config); + const result = verifyAgainstLock(lockEntry, artifact); + + if (result.passed) { + process.stdout.write(` ${c(ANSI.green, "✓")} ${t.config.targetId}\n`); + } else { + anyFailed = true; + process.stdout.write(` ${c(ANSI.red, "✗")} ${t.config.targetId}\n`); + for (const d of result.drift) { + process.stdout.write( + ` ${c(ANSI.yellow, "→")} ${d.category}/${d.name}: ${d.change}\n`, + ); + } + } + } catch (err) { + anyFailed = true; + const msg = err instanceof Error ? err.message : String(err); + process.stdout.write(` ${c(ANSI.red, "✗")} ${t.config.targetId}: ${msg}\n`); + } + } + + process.stdout.write("\n"); + + if (anyFailed) { + process.exitCode = 1; + } + }); +} diff --git a/src/commands/scan.ts b/src/commands/scan.ts index 6e216c8..713909a 100644 --- a/src/commands/scan.ts +++ b/src/commands/scan.ts @@ -5,6 +5,7 @@ import { scanForTargets } from "../discovery.js"; import { runTarget, } from "../index.js"; +import { appendHistory, buildHistoryEntry } from "../history.js"; import { buildEvent, recordEvent } from "../telemetry.js"; import { TOOL_VERSION } from "../version.js"; import { ANSI, LOGO, c, useColor } from "./helpers.js"; @@ -96,6 +97,9 @@ async function runScan(bin: string, configPath: string | undefined, invokeTools: checkStatusMap[`${t.config.targetId}:${check.id}`] = check.status; } + // Track history + await appendHistory(buildHistoryEntry(artifact)).catch(() => {}); + results.push({ targetId: t.config.targetId, gate: artifact.gate, toolCount, promptCount, resourceCount, diagnostics }); if (artifact.gate === "pass") passCount++; else failCount++; } catch (error) { diff --git a/src/commands/test.ts b/src/commands/test.ts index e8a7da7..466d2c0 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5,6 +5,7 @@ import { writeRunArtifact, } from "../index.js"; import { defaultRunsDirectory } from "../storage.js"; +import { appendHistory, buildHistoryEntry, getTrend, readHistory } from "../history.js"; import { buildEvent, recordEvent } from "../telemetry.js"; import { ANSI, c, targetFromCommand } from "./helpers.js"; @@ -42,9 +43,18 @@ export function registerTestCommands(program: Command): void { process.stdout.write(`\n ${c(ANSI.dim, `Artifact: ${outPath}`)}\n\n`); + // Track history + const historyEntry = buildHistoryEntry(artifact); + await appendHistory(historyEntry).catch(() => {}); + const history = await readHistory().catch(() => ({ version: 1 as const, entries: [] })); + const trend = getTrend(target.targetId, history); + const testCheckStatuses: Record = {}; for (const ch of artifact.checks) testCheckStatuses[ch.id] = ch.status; recordEvent(buildEvent("command_complete", "test", "cli", { + historyEntryCount: history.entries.length, + trendDirection: trend?.direction, + previousGrade: trend?.previous?.grade, serversScanned: 1, toolsFound: toolCount, promptsFound: promptCount, diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 0d4a6d7..09e0142 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -8,6 +8,7 @@ import { import { renderWatchFirstRun, renderWatchNoChanges, renderWatchChanges } from "../reporters/terminal.js"; import { isCI } from "../ci.js"; import { defaultRunsDirectory, findLatestArtifact, readArtifact } from "../storage.js"; +import { appendHistory, buildHistoryEntry } from "../history.js"; import { ANSI, c, formatOutput, targetFromCommand } from "./helpers.js"; // ── One-shot mode ──────────────────────────────────────────────────────────── @@ -22,6 +23,9 @@ async function runWatchOneShot( const artifact = await runTarget(target); const outPath = await writeRunArtifact(artifact, outDir); + // Track history + await appendHistory(buildHistoryEntry(artifact)).catch(() => {}); + // JSON mode bypasses compact rendering if (options.format === "json") { process.stdout.write(formatOutput(artifact, "json") + "\n"); diff --git a/src/commit-status.ts b/src/commit-status.ts new file mode 100644 index 0000000..a374d64 --- /dev/null +++ b/src/commit-status.ts @@ -0,0 +1,86 @@ +import type { RunArtifact } from "./types.js"; + +function getToolCount(artifact: RunArtifact): number { + return artifact.checks.find(c => c.id === "tools")?.evidence[0]?.itemCount ?? 0; +} + +function getPromptCount(artifact: RunArtifact): number { + return artifact.checks.find(c => c.id === "prompts")?.evidence[0]?.itemCount ?? 0; +} + +function countSecurityFindings(artifact: RunArtifact): number { + let count = 0; + for (const check of artifact.checks) { + if (check.id !== "security" && check.id !== "security-lite") continue; + for (const ev of check.evidence) { + if (!ev.diagnostics) continue; + for (const d of ev.diagnostics) { + if (/\[(high|medium)\]/i.test(d)) count++; + } + } + } + return count; +} + +function countFailingChecks(artifact: RunArtifact): number { + let count = 0; + for (const check of artifact.checks) { + if (check.id === "security" || check.id === "security-lite") continue; + if (check.status === "fail") count++; + } + return count; +} + +function countTotalIssues(artifact: RunArtifact): number { + return countSecurityFindings(artifact) + countFailingChecks(artifact); +} + +export function buildStatusDescription( + artifacts: RunArtifact[], +): { state: "success" | "failure"; description: string } { + const allPass = artifacts.every(a => a.gate === "pass"); + + if (allPass) { + if (artifacts.length === 1) { + const a = artifacts[0]!; + const tools = getToolCount(a); + const prompts = getPromptCount(a); + return { + state: "success", + description: `All clear (${tools} tools, ${prompts} prompts)`, + }; + } + const totalTools = artifacts.reduce((sum, a) => sum + getToolCount(a), 0); + return { + state: "success", + description: `All clear (${artifacts.length} servers, ${totalTools} tools)`, + }; + } + + // At least one failure + if (artifacts.length === 1) { + const a = artifacts[0]!; + const securityIssues = countSecurityFindings(a); + const failingChecks = countFailingChecks(a); + if (securityIssues > 0) { + return { state: "failure", description: `${securityIssues} security issues` }; + } + if (failingChecks > 0) { + return { state: "failure", description: `${failingChecks} failing checks` }; + } + return { state: "failure", description: "Issues detected" }; + } + + // Multiple artifacts, at least one failing + const failingServers = artifacts.filter(a => a.gate === "fail"); + const totalIssues = failingServers.reduce((sum, a) => sum + countTotalIssues(a), 0); + const issueCount = totalIssues > 0 ? totalIssues : failingServers.length; + return { + state: "failure", + description: `${issueCount} issues across ${failingServers.length} servers`, + }; +} + +export function buildCommitStatusContext(): string { + return "MCP Observatory"; +} diff --git a/src/history.ts b/src/history.ts new file mode 100644 index 0000000..102a11a --- /dev/null +++ b/src/history.ts @@ -0,0 +1,125 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { Gate, HealthGrade, RunArtifact } from "./types.js"; + +// ── Types ──────────────────────────────────────────────────────────────────── + +export interface HistoryEntry { + date: string; + targetId: string; + healthScore: number; + grade: HealthGrade; + toolCount: number; + promptCount: number; + resourceCount: number; + gate: Gate; +} + +export interface HistoryFile { + version: 1; + entries: HistoryEntry[]; +} + +export interface TrendInfo { + current: HistoryEntry; + previous?: HistoryEntry; + direction: "up" | "down" | "stable" | "new"; + delta: number; +} + +// ── Path helpers ───────────────────────────────────────────────────────────── + +export function defaultHistoryPath(cwd?: string): string { + return path.join(cwd ?? process.cwd(), ".mcp-observatory", "history.json"); +} + +// ── Read / Write ───────────────────────────────────────────────────────────── + +export async function readHistory(historyPath?: string): Promise { + const filePath = historyPath ?? defaultHistoryPath(); + try { + const raw = await readFile(filePath, "utf8"); + return JSON.parse(raw) as HistoryFile; + } catch { + return { version: 1, entries: [] }; + } +} + +export async function appendHistory( + entry: HistoryEntry, + historyPath?: string, +): Promise { + const filePath = historyPath ?? defaultHistoryPath(); + const history = await readHistory(filePath); + history.entries.push(entry); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(history, null, 2) + "\n", "utf8"); +} + +// ── Build entry from artifact ──────────────────────────────────────────────── + +function itemCountFromCheck( + checks: RunArtifact["checks"], + checkId: string, +): number { + const check = checks.find((c) => c.id === checkId); + if (!check || check.evidence.length === 0) return 0; + return check.evidence[0]!.itemCount ?? 0; +} + +export function buildHistoryEntry(artifact: RunArtifact): HistoryEntry { + return { + date: artifact.createdAt, + targetId: artifact.target.targetId, + healthScore: artifact.healthScore?.overall ?? 0, + grade: artifact.healthScore?.grade ?? "F", + toolCount: itemCountFromCheck(artifact.checks, "tools"), + promptCount: itemCountFromCheck(artifact.checks, "prompts"), + resourceCount: itemCountFromCheck(artifact.checks, "resources"), + gate: artifact.gate, + }; +} + +// ── Trend computation ──────────────────────────────────────────────────────── + +export function getTrend( + targetId: string, + history: HistoryFile, +): TrendInfo | null { + const matching = history.entries.filter((e) => e.targetId === targetId); + if (matching.length === 0) return null; + + const current = matching[matching.length - 1]!; + const previous = + matching.length >= 2 ? matching[matching.length - 2] : undefined; + + let direction: TrendInfo["direction"]; + if (!previous) { + direction = "new"; + } else if (current.healthScore > previous.healthScore) { + direction = "up"; + } else if (current.healthScore < previous.healthScore) { + direction = "down"; + } else { + direction = "stable"; + } + + const delta = current.healthScore - (previous?.healthScore ?? 0); + + return { current, previous, direction, delta }; +} + +// ── Rendering ──────────────────────────────────────────────────────────────── + +export function renderTrendLabel(trend: TrendInfo): string { + switch (trend.direction) { + case "new": + return "● first run"; + case "up": + return `↗ was ${trend.previous!.grade} (${trend.previous!.healthScore})`; + case "down": + return `↘ was ${trend.previous!.grade} (${trend.previous!.healthScore})`; + case "stable": + return "→ stable"; + } +} diff --git a/src/index.ts b/src/index.ts index ace09c2..9a4b092 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,12 @@ export { renderHtml } from "./reporters/html.js"; export { renderJUnit } from "./reporters/junit.js"; export { renderMarkdown } from "./reporters/markdown.js"; export { renderPrComment } from "./reporters/pr-comment.js"; +export { readHistory, appendHistory, buildHistoryEntry, getTrend, renderTrendLabel } from "./history.js"; +export { readLockFile, writeLockFile, buildServerLockEntry, verifyAgainstLock, mergeLockFile } from "./lockfile.js"; +export { renderMatrixComment, type MatrixRow } from "./reporters/pr-comment-matrix.js"; +export { buildStatusDescription, buildCommitStatusContext } from "./commit-status.js"; +export { buildCiReport } from "./commands/ci-report.js"; +export { findExistingIssue, createOrUpdateIssue } from "./ci-issue.js"; export { renderSarif } from "./reporters/sarif.js"; export { renderTerminal, renderWatchFirstRun, renderWatchNoChanges, renderWatchChanges } from "./reporters/terminal.js"; export { runTarget, runTargetRecording, type RunOptions, type RunResult } from "./runner.js"; diff --git a/src/lockfile.ts b/src/lockfile.ts new file mode 100644 index 0000000..0f1c9c4 --- /dev/null +++ b/src/lockfile.ts @@ -0,0 +1,236 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import type { RunArtifact } from "./types.js"; + +// ── Lock File Types ───────────────────────────────────────────────────────── + +export interface LockFileToolEntry { + name: string; + description?: string; + inputSchema: object; +} + +export interface LockFilePromptEntry { + name: string; + description?: string; + arguments?: object[]; +} + +export interface LockFileResourceEntry { + uri: string; + name: string; + description?: string; + mimeType?: string; +} + +export interface LockFileServerEntry { + targetId: string; + lockedAt: string; + serverName?: string; + serverVersion?: string; + tools: LockFileToolEntry[]; + prompts: LockFilePromptEntry[]; + resources: LockFileResourceEntry[]; +} + +export interface LockFile { + version: 1; + lockedAt: string; + servers: LockFileServerEntry[]; +} + +export interface LockDriftEntry { + targetId: string; + category: "tools" | "prompts" | "resources"; + name: string; + change: string; +} + +export interface LockVerifyResult { + targetId: string; + passed: boolean; + drift: LockDriftEntry[]; +} + +// ── Path helpers ──────────────────────────────────────────────────────────── + +export function defaultLockPath(cwd?: string): string { + return path.join(cwd ?? process.cwd(), ".mcp-observatory", "lock.json"); +} + +// ── Read / Write ──────────────────────────────────────────────────────────── + +export async function readLockFile(lockPath?: string): Promise { + const filePath = lockPath ?? defaultLockPath(); + const raw = await readFile(filePath, "utf8"); + return JSON.parse(raw) as LockFile; +} + +export async function writeLockFile( + lockFile: LockFile, + lockPath?: string, +): Promise { + const filePath = lockPath ?? defaultLockPath(); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(lockFile, null, 2) + "\n", "utf8"); + return filePath; +} + +// ── Build Lock Entry from RunArtifact ─────────────────────────────────────── + +export function buildServerLockEntry(artifact: RunArtifact): LockFileServerEntry { + const toolsCheck = artifact.checks.find((ch) => ch.id === "tools"); + const promptsCheck = artifact.checks.find((ch) => ch.id === "prompts"); + const resourcesCheck = artifact.checks.find((ch) => ch.id === "resources"); + + // Extract tools + const tools: LockFileToolEntry[] = []; + if (toolsCheck && toolsCheck.evidence.length > 0) { + const ev = toolsCheck.evidence[0]!; + const schemas = ev.schemas; + const identifiers = ev.identifiers ?? []; + + if (schemas && Object.keys(schemas).length > 0) { + for (const name of Object.keys(schemas)) { + tools.push({ name, inputSchema: schemas[name]! }); + } + } else { + // Fall back to identifiers only + for (const name of identifiers) { + tools.push({ name, inputSchema: {} }); + } + } + } + + // Extract prompts + const prompts: LockFilePromptEntry[] = []; + if (promptsCheck && promptsCheck.evidence.length > 0) { + const ev = promptsCheck.evidence[0]!; + const identifiers = ev.identifiers ?? []; + for (const name of identifiers) { + prompts.push({ name }); + } + } + + // Extract resources + const resources: LockFileResourceEntry[] = []; + if (resourcesCheck && resourcesCheck.evidence.length > 0) { + const ev = resourcesCheck.evidence[0]!; + const identifiers = ev.identifiers ?? []; + for (const name of identifiers) { + resources.push({ name, uri: name }); + } + } + + return { + targetId: artifact.target.targetId, + lockedAt: new Date().toISOString(), + serverName: artifact.target.serverName, + serverVersion: artifact.target.serverVersion, + tools, + prompts, + resources, + }; +} + +// ── Verify against lock ───────────────────────────────────────────────────── + +export function verifyAgainstLock( + lockEntry: LockFileServerEntry, + artifact: RunArtifact, +): LockVerifyResult { + const current = buildServerLockEntry(artifact); + const drift: LockDriftEntry[] = []; + const targetId = lockEntry.targetId; + + // Compare tools + const lockedToolNames = new Set(lockEntry.tools.map((t) => t.name)); + const currentToolNames = new Set(current.tools.map((t) => t.name)); + + for (const name of currentToolNames) { + if (!lockedToolNames.has(name)) { + drift.push({ targetId, category: "tools", name, change: "added" }); + } + } + for (const name of lockedToolNames) { + if (!currentToolNames.has(name)) { + drift.push({ targetId, category: "tools", name, change: "removed" }); + } + } + + // Compare tool schemas for tools present in both + const lockedToolMap = new Map(lockEntry.tools.map((t) => [t.name, t])); + for (const tool of current.tools) { + const locked = lockedToolMap.get(tool.name); + if (locked) { + const lockedSchema = JSON.stringify(locked.inputSchema); + const currentSchema = JSON.stringify(tool.inputSchema); + if (lockedSchema !== currentSchema) { + drift.push({ + targetId, + category: "tools", + name: tool.name, + change: "schema changed", + }); + } + } + } + + // Compare prompts + const lockedPromptNames = new Set(lockEntry.prompts.map((p) => p.name)); + const currentPromptNames = new Set(current.prompts.map((p) => p.name)); + + for (const name of currentPromptNames) { + if (!lockedPromptNames.has(name)) { + drift.push({ targetId, category: "prompts", name, change: "added" }); + } + } + for (const name of lockedPromptNames) { + if (!currentPromptNames.has(name)) { + drift.push({ targetId, category: "prompts", name, change: "removed" }); + } + } + + // Compare resources + const lockedResourceNames = new Set(lockEntry.resources.map((r) => r.name)); + const currentResourceNames = new Set(current.resources.map((r) => r.name)); + + for (const name of currentResourceNames) { + if (!lockedResourceNames.has(name)) { + drift.push({ targetId, category: "resources", name, change: "added" }); + } + } + for (const name of lockedResourceNames) { + if (!currentResourceNames.has(name)) { + drift.push({ targetId, category: "resources", name, change: "removed" }); + } + } + + return { targetId, passed: drift.length === 0, drift }; +} + +// ── Merge lock files ──────────────────────────────────────────────────────── + +export function mergeLockFile( + existing: LockFile, + newEntries: LockFileServerEntry[], +): LockFile { + const serverMap = new Map(); + + // Start with existing entries + for (const entry of existing.servers) { + serverMap.set(entry.targetId, entry); + } + + // Overwrite / add new entries + for (const entry of newEntries) { + serverMap.set(entry.targetId, entry); + } + + return { + version: 1, + lockedAt: new Date().toISOString(), + servers: Array.from(serverMap.values()), + }; +} diff --git a/src/reporters/pr-comment-matrix.ts b/src/reporters/pr-comment-matrix.ts new file mode 100644 index 0000000..0c514ea --- /dev/null +++ b/src/reporters/pr-comment-matrix.ts @@ -0,0 +1,149 @@ +import type { RunArtifact, TrendInfo } from "../types.js"; +import { findChecksByStatus } from "./common.js"; +import { + extractSecurityFindings, + extractQualityFindings, + countCapabilities, + blockquoteList, + prCommentFooter, + type ParsedFinding, +} from "./pr-comment.js"; + +// ── Types ─────────────────────────────────────────────────────────────────── + +export interface MatrixRow { + artifact: RunArtifact; + trend?: TrendInfo; +} + +// ── Helpers ───────────────────────────────────────────────────────────────── + +interface RowAnalysis { + row: MatrixRow; + security: ParsedFinding[]; + quality: ParsedFinding[]; + failingChecks: ReturnType; + highMedSecurity: ParsedFinding[]; + issueCount: number; + toolCount: number; +} + +function analyzeRow(row: MatrixRow): RowAnalysis { + const security = extractSecurityFindings(row.artifact.checks); + const quality = extractQualityFindings(row.artifact.checks); + const failingChecks = findChecksByStatus(row.artifact.checks, "fail") + .filter(c => c.id !== "security" && c.id !== "security-lite"); + const highMedSecurity = security.filter( + f => f.severity === "high" || f.severity === "medium", + ); + const issueCount = highMedSecurity.length + quality.length + failingChecks.length; + const caps = countCapabilities(row.artifact.checks); + return { row, security, quality, failingChecks, highMedSecurity, issueCount, toolCount: caps.tools }; +} + +function trendArrow(trend?: TrendInfo): string { + if (!trend || trend.direction === "new") return ""; + switch (trend.direction) { + case "up": + return " ↗"; + case "down": + return " ↘"; + case "stable": + return " →"; + } +} + +function healthCell(artifact: RunArtifact, trend?: TrendInfo): string { + const hs = artifact.healthScore; + const base = hs ? `**${hs.grade}** (${hs.overall})` : `**${artifact.gate}**`; + return base + trendArrow(trend); +} + +function issuesCell(analysis: RowAnalysis): string { + const { highMedSecurity, quality, failingChecks, issueCount } = analysis; + if (issueCount === 0) return "✅"; + + const hasSecurityOrFailing = highMedSecurity.length > 0 || failingChecks.length > 0; + const hasQuality = quality.length > 0; + + if (hasSecurityOrFailing && hasQuality) { + return `🔴 ${issueCount} issues`; + } + if (hasSecurityOrFailing) { + const count = highMedSecurity.length + failingChecks.length; + return `🔴 ${count} security`; + } + // quality only + return `⚠️ ${quality.length} quality`; +} + +function renderDetailsBlock(analysis: RowAnalysis): string | undefined { + const items: string[] = []; + + for (const f of analysis.highMedSecurity) { + items.push(`\`${f.severity}\` ${f.message}`); + } + for (const c of analysis.failingChecks) { + items.push(`\`fail\` **${c.id}**: ${c.message}`); + } + for (const f of analysis.quality) { + items.push(`\`${f.severity}\` ${f.message}`); + } + + if (items.length === 0) return undefined; + + const targetId = analysis.row.artifact.target.targetId; + const hasSecurityOrFailing = + analysis.highMedSecurity.length > 0 || analysis.failingChecks.length > 0; + const icon = hasSecurityOrFailing ? "🔴" : "⚠️"; + const lines: string[] = []; + lines.push(`
${icon} ${targetId} — ${items.length} issue${items.length === 1 ? "" : "s"}`); + lines.push(""); + lines.push(blockquoteList(items)); + lines.push(""); + lines.push("
"); + return lines.join("\n"); +} + +// ── Public API ────────────────────────────────────────────────────────────── + +export function renderMatrixComment(rows: MatrixRow[]): string { + if (rows.length === 0) { + return ["## 🔭 MCP Observatory — No servers scanned", "", "No servers were scanned in this run.", prCommentFooter()].join("\n"); + } + + const sections: string[] = []; + const analyses = rows.map(analyzeRow); + + // Header + sections.push( + `## 🔭 MCP Observatory — ${rows.length} server${rows.length === 1 ? "" : "s"} scanned`, + ); + sections.push(""); + + // Table header + sections.push("| Server | Health | Tools | Issues |"); + sections.push("| --- | --- | --- | --- |"); + + // Table rows + for (const analysis of analyses) { + const server = analysis.row.artifact.target.targetId; + const health = healthCell(analysis.row.artifact, analysis.row.trend); + const tools = String(analysis.toolCount); + const issues = issuesCell(analysis); + sections.push(`| ${server} | ${health} | ${tools} | ${issues} |`); + } + + // Details blocks for rows with issues + const detailBlocks = analyses + .map(renderDetailsBlock) + .filter((b): b is string => b !== undefined); + + if (detailBlocks.length > 0) { + sections.push(""); + sections.push(detailBlocks.join("\n\n")); + } + + sections.push(prCommentFooter()); + return sections.join("\n"); +} diff --git a/src/reporters/pr-comment.ts b/src/reporters/pr-comment.ts index 6fc69cc..af4d3bd 100644 --- a/src/reporters/pr-comment.ts +++ b/src/reporters/pr-comment.ts @@ -1,9 +1,9 @@ -import type { CheckResult, DiffArtifact, RunArtifact, SchemaDriftEntry } from "../types.js"; +import type { CheckResult, DiffArtifact, RunArtifact, SchemaDriftEntry, TrendInfo } from "../types.js"; import { findChecksByStatus } from "./common.js"; // ── Types ─────────────────────────────────────────────────────────────────── -interface ParsedFinding { +export interface ParsedFinding { severity: string; message: string; } @@ -13,9 +13,9 @@ interface ParsedFinding { const MAX_ITEMS_PER_SECTION = 5; const REPO_URL = "https://github.com/KryptosAI/mcp-observatory"; -// ── Extraction helpers ────────────────────────────────────────────────────── +// ── Extraction helpers (exported for matrix comment reuse) ────────────────── -function extractSecurityFindings(checks: CheckResult[]): ParsedFinding[] { +export function extractSecurityFindings(checks: CheckResult[]): ParsedFinding[] { const securityChecks = checks.filter(c => c.id === "security" || c.id === "security-lite"); const findings: ParsedFinding[] = []; for (const check of securityChecks) { @@ -31,7 +31,7 @@ function extractSecurityFindings(checks: CheckResult[]): ParsedFinding[] { return findings; } -function extractQualityFindings(checks: CheckResult[]): ParsedFinding[] { +export function extractQualityFindings(checks: CheckResult[]): ParsedFinding[] { const qualityChecks = checks.filter(c => c.id === "schema-quality"); const findings: ParsedFinding[] = []; for (const check of qualityChecks) { @@ -47,7 +47,7 @@ function extractQualityFindings(checks: CheckResult[]): ParsedFinding[] { return findings; } -function countCapabilities(checks: CheckResult[]): { tools: number; prompts: number; resources: number } { +export function countCapabilities(checks: CheckResult[]): { tools: number; prompts: number; resources: number } { const get = (id: string) => { const check = checks.find(c => c.id === id); return check?.evidence[0]?.itemCount ?? 0; @@ -63,7 +63,7 @@ function conformanceSummary(checks: CheckResult[]): string | undefined { // ── Formatting helpers ────────────────────────────────────────────────────── -function blockquoteList(items: string[], max = MAX_ITEMS_PER_SECTION): string { +export function blockquoteList(items: string[], max = MAX_ITEMS_PER_SECTION): string { const shown = items.slice(0, max); const lines = shown.map(item => `> ${item}`); const remaining = items.length - max; @@ -73,7 +73,7 @@ function blockquoteList(items: string[], max = MAX_ITEMS_PER_SECTION): string { return lines.join("\n"); } -function footer(): string { +export function prCommentFooter(): string { return [ "", "---", @@ -83,7 +83,7 @@ function footer(): string { // ── Run artifact rendering ────────────────────────────────────────────────── -function renderRunComment(artifact: RunArtifact): string { +function renderRunComment(artifact: RunArtifact, trend?: TrendInfo): string { const sections: string[] = []; const security = extractSecurityFindings(artifact.checks); const quality = extractQualityFindings(artifact.checks); @@ -138,10 +138,16 @@ function renderRunComment(artifact: RunArtifact): string { // Summary stats const caps = countCapabilities(artifact.checks); + const healthPart = artifact.healthScore + ? `Health: **${artifact.healthScore.grade}** (${artifact.healthScore.overall})` + : `Gate: **${artifact.gate}**`; + + const trendPart = trend && trend.direction !== "new" && trend.previous + ? ` ${trend.direction === "up" ? "↗" : trend.direction === "down" ? "↘" : "→"} was ${trend.previous.grade} (${trend.previous.healthScore})` + : ""; + const statsLine = [ - artifact.healthScore - ? `Health: **${artifact.healthScore.grade}** (${artifact.healthScore.overall})` - : `Gate: **${artifact.gate}**`, + healthPart + trendPart, `${caps.tools} tools`, `${caps.prompts} prompts`, `${caps.resources} resources`, @@ -151,7 +157,7 @@ function renderRunComment(artifact: RunArtifact): string { sections.push(`### 📊 Summary`); sections.push(`> ${statsLine}`); - sections.push(footer()); + sections.push(prCommentFooter()); return sections.join("\n"); } @@ -216,7 +222,7 @@ function renderDiffComment(artifact: DiffArtifact): string { sections.push(`### 📊 Summary`); sections.push(`> ${statsLine}`); - sections.push(footer()); + sections.push(prCommentFooter()); return sections.join("\n"); } @@ -232,8 +238,8 @@ function flattenDrift(drift: SchemaDriftEntry[]): string[] { // ── Public API ────────────────────────────────────────────────────────────── -export function renderPrComment(artifact: RunArtifact | DiffArtifact): string { +export function renderPrComment(artifact: RunArtifact | DiffArtifact, trend?: TrendInfo): string { return artifact.artifactType === "run" - ? renderRunComment(artifact) + ? renderRunComment(artifact, trend) : renderDiffComment(artifact); } diff --git a/src/telemetry.ts b/src/telemetry.ts index 174a678..51b5023 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -42,6 +42,26 @@ export interface TelemetryEnrichment { suggestedServers?: string[]; detectedLanguages?: string[]; detectedFrameworks?: string[]; + // Trend tracking + historyEntryCount?: number; + trendDirection?: string; + previousGrade?: string; + // Lock files + lockFileExists?: boolean; + lockServerCount?: number; + lockDriftDetected?: boolean; + lockDriftCount?: number; + // Matrix scanning + matrixServerCount?: number; + matrixFailCount?: number; + matrixPassCount?: number; + // Commit status + commitStatusSet?: boolean; + commitStatusState?: string; + // Nightly scans + nightlyScan?: boolean; + issueCreated?: boolean; + issueNumber?: number; } export interface TelemetryEvent extends TelemetryEnrichment { diff --git a/src/types.ts b/src/types.ts index d06510f..55cbbad 100644 --- a/src/types.ts +++ b/src/types.ts @@ -180,3 +180,78 @@ export interface DiffArtifact { schemaDrift?: SchemaDriftEntry[]; responseChanges?: ResponseChangeEntry[]; } + +// ── History / Trend Types ─────────────────────────────────────────────────── + +export interface HistoryEntry { + date: string; + targetId: string; + healthScore: number; + grade: HealthGrade; + toolCount: number; + promptCount: number; + resourceCount: number; + gate: Gate; +} + +export interface HistoryFile { + version: 1; + entries: HistoryEntry[]; +} + +export interface TrendInfo { + current: HistoryEntry; + previous?: HistoryEntry; + direction: "up" | "down" | "stable" | "new"; + delta: number; +} + +// ── Lock File Types ───────────────────────────────────────────────────────── + +export interface LockFileToolEntry { + name: string; + description?: string; + inputSchema: object; +} + +export interface LockFilePromptEntry { + name: string; + description?: string; + arguments?: object[]; +} + +export interface LockFileResourceEntry { + uri: string; + name: string; + description?: string; + mimeType?: string; +} + +export interface LockFileServerEntry { + targetId: string; + lockedAt: string; + serverName?: string; + serverVersion?: string; + tools: LockFileToolEntry[]; + prompts: LockFilePromptEntry[]; + resources: LockFileResourceEntry[]; +} + +export interface LockFile { + version: 1; + lockedAt: string; + servers: LockFileServerEntry[]; +} + +export interface LockDriftEntry { + targetId: string; + category: "tools" | "prompts" | "resources"; + name: string; + change: string; +} + +export interface LockVerifyResult { + targetId: string; + passed: boolean; + drift: LockDriftEntry[]; +} diff --git a/tests/ci-issue.test.ts b/tests/ci-issue.test.ts new file mode 100644 index 0000000..0336d2a --- /dev/null +++ b/tests/ci-issue.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it, vi } from "vitest"; + +// Mock fs/promises to avoid real file writes +vi.mock("node:fs/promises", () => ({ + writeFile: vi.fn(async () => {}), + unlink: vi.fn(async () => {}), +})); + +import type { CommandExecutor } from "../src/ci-issue.js"; +import { findExistingIssue, createOrUpdateIssue } from "../src/ci-issue.js"; + +function createMockExec(): CommandExecutor & { + calls: Array<{ cmd: string; args: string[] }>; + mockResolvedValueOnce(value: { stdout: string; stderr: string }): void; + mockRejectedValueOnce(error: Error): void; +} { + const queue: Array< + | { type: "resolve"; value: { stdout: string; stderr: string } } + | { type: "reject"; error: Error } + > = []; + const calls: Array<{ cmd: string; args: string[] }> = []; + + const fn = (cmd: string, args: string[]): Promise<{ stdout: string; stderr: string }> => { + calls.push({ cmd, args }); + const next = queue.shift(); + if (!next) return Promise.reject(new Error("No more mock values queued")); + if (next.type === "reject") return Promise.reject(next.error); + return Promise.resolve(next.value); + }; + + fn.calls = calls; + fn.mockResolvedValueOnce = (value: { stdout: string; stderr: string }) => { + queue.push({ type: "resolve", value }); + }; + fn.mockRejectedValueOnce = (error: Error) => { + queue.push({ type: "reject", error }); + }; + + return fn; +} + +describe("findExistingIssue", () => { + it("returns issue number when found", async () => { + const exec = createMockExec(); + exec.mockResolvedValueOnce({ + stdout: JSON.stringify([{ number: 42 }]), + stderr: "", + }); + const result = await findExistingIssue("owner/repo", "mcp-observatory", exec); + expect(result).toBe(42); + }); + + it("returns null when no issues exist", async () => { + const exec = createMockExec(); + exec.mockResolvedValueOnce({ + stdout: JSON.stringify([]), + stderr: "", + }); + const result = await findExistingIssue("owner/repo", "mcp-observatory", exec); + expect(result).toBeNull(); + }); + + it("returns null on error", async () => { + const exec = createMockExec(); + exec.mockRejectedValueOnce(new Error("gh not found")); + const result = await findExistingIssue("owner/repo", "mcp-observatory", exec); + expect(result).toBeNull(); + }); +}); + +describe("createOrUpdateIssue", () => { + it("creates a new issue when none exists", async () => { + const exec = createMockExec(); + // findExistingIssue: no open issues + exec.mockResolvedValueOnce({ stdout: JSON.stringify([]), stderr: "" }); + // gh issue create: returns URL + exec.mockResolvedValueOnce({ + stdout: "https://github.com/owner/repo/issues/99\n", + stderr: "", + }); + + const result = await createOrUpdateIssue({ + repo: "owner/repo", + title: "Test issue", + body: "Test body", + labels: ["mcp-observatory"], + exec, + }); + + expect(result).toBe(99); + expect(exec.calls).toHaveLength(2); + // Second call should be gh issue create + expect(exec.calls[1]!.args).toContain("create"); + }); + + it("comments on existing issue when one is found", async () => { + const exec = createMockExec(); + // findExistingIssue: returns issue 7 + exec.mockResolvedValueOnce({ + stdout: JSON.stringify([{ number: 7 }]), + stderr: "", + }); + // gh issue comment: succeeds + exec.mockResolvedValueOnce({ stdout: "", stderr: "" }); + + const result = await createOrUpdateIssue({ + repo: "owner/repo", + title: "Test issue", + body: "Updated body", + labels: ["mcp-observatory"], + exec, + }); + + expect(result).toBe(7); + expect(exec.calls).toHaveLength(2); + // Second call should be gh issue comment + expect(exec.calls[1]!.args).toContain("comment"); + expect(exec.calls[1]!.args).toContain("7"); + }); +}); diff --git a/tests/ci-report.test.ts b/tests/ci-report.test.ts new file mode 100644 index 0000000..59dfe72 --- /dev/null +++ b/tests/ci-report.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; + +import { buildCiReport } from "../src/commands/ci-report.js"; +import type { RunArtifact, CheckResult } from "../src/types.js"; + +function makeCheck(id: string, status: string, message = `${id} ${status}`): CheckResult { + return { + id: id as CheckResult["id"], + capability: id as CheckResult["capability"], + status: status as CheckResult["status"], + durationMs: 100, + message, + evidence: [], + }; +} + +function makeArtifact( + targetId: string, + gate: "pass" | "fail", + checks: CheckResult[] = [], +): RunArtifact { + return { + artifactType: "run", + schemaVersion: "1.0.0", + gate, + runId: `run-${targetId}`, + createdAt: "2026-03-23T00:00:00Z", + toolVersion: "0.1.0", + target: { + targetId, + adapter: "local-process", + command: "node", + args: ["server.js"], + }, + environment: { platform: "linux", nodeVersion: "22.0.0" }, + summary: { + gate, + total: checks.length, + pass: checks.filter((c) => c.status === "pass").length, + fail: checks.filter((c) => c.status === "fail").length, + partial: 0, + unsupported: 0, + flaky: 0, + skipped: 0, + }, + checks, + }; +} + +describe("buildCiReport", () => { + it("returns all-clear for empty artifacts", () => { + const report = buildCiReport([]); + expect(report.hasRegressions).toBe(false); + expect(report.title).toContain("all clear"); + expect(report.serverCount).toBe(0); + expect(report.failCount).toBe(0); + }); + + it("returns all-clear when all artifacts pass", () => { + const artifacts = [ + makeArtifact("server-a", "pass", [makeCheck("tools", "pass")]), + makeArtifact("server-b", "pass", [makeCheck("tools", "pass")]), + ]; + const report = buildCiReport(artifacts); + expect(report.hasRegressions).toBe(false); + expect(report.title).toContain("all clear"); + expect(report.serverCount).toBe(2); + expect(report.failCount).toBe(0); + }); + + it("detects a single regression", () => { + const artifacts = [ + makeArtifact("server-a", "pass", [makeCheck("tools", "pass")]), + makeArtifact("server-b", "fail", [ + makeCheck("tools", "fail", "tools listing returned empty"), + ]), + ]; + const report = buildCiReport(artifacts); + expect(report.hasRegressions).toBe(true); + expect(report.title).toContain("1 regression"); + expect(report.failCount).toBe(1); + expect(report.serverCount).toBe(2); + }); + + it("detects multiple regressions", () => { + const artifacts = [ + makeArtifact("server-a", "fail", [makeCheck("tools", "fail")]), + makeArtifact("server-b", "fail", [makeCheck("prompts", "fail")]), + makeArtifact("server-c", "pass", [makeCheck("tools", "pass")]), + ]; + const report = buildCiReport(artifacts); + expect(report.hasRegressions).toBe(true); + expect(report.title).toContain("2 regressions"); + expect(report.failCount).toBe(2); + }); + + it("includes failing server target IDs in the body", () => { + const artifacts = [ + makeArtifact("my-failing-server", "fail", [ + makeCheck("tools", "fail", "tools returned empty"), + makeCheck("conformance", "partial", "missing initialize"), + ]), + ]; + const report = buildCiReport(artifacts); + expect(report.body).toContain("my-failing-server"); + expect(report.body).toContain("tools"); + expect(report.body).toContain("conformance"); + }); + + it("always includes mcp-observatory label", () => { + const report = buildCiReport([]); + expect(report.labels).toContain("mcp-observatory"); + + const report2 = buildCiReport([ + makeArtifact("server-a", "fail", [makeCheck("tools", "fail")]), + ]); + expect(report2.labels).toContain("mcp-observatory"); + }); +}); diff --git a/tests/commit-status.test.ts b/tests/commit-status.test.ts new file mode 100644 index 0000000..92ead68 --- /dev/null +++ b/tests/commit-status.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from "vitest"; +import { buildStatusDescription } from "../src/commit-status.js"; +import { makeArtifact } from "./fixtures/test-helpers.js"; +import type { CheckResult } from "../src/types.js"; + +function makeCheck(overrides: Partial = {}): CheckResult { + return { + id: "tools", + capability: "tools", + status: "pass", + durationMs: 10, + message: "OK", + evidence: [{ endpoint: "tools/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 3 }], + ...overrides, + }; +} + +describe("buildStatusDescription", () => { + it("single passing artifact returns success with All clear", () => { + const artifact = makeArtifact([ + makeCheck({ id: "tools", capability: "tools", evidence: [{ endpoint: "tools/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 5 }] }), + makeCheck({ id: "prompts", capability: "prompts", evidence: [{ endpoint: "prompts/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 2 }] }), + ]); + const result = buildStatusDescription([artifact]); + expect(result.state).toBe("success"); + expect(result.description).toContain("All clear"); + }); + + it("single failing artifact with non-security failure returns failing checks", () => { + const artifact = makeArtifact([ + makeCheck({ id: "tools", capability: "tools", status: "fail", message: "no tools" }), + makeCheck({ id: "prompts", capability: "prompts", status: "pass" }), + ]); + const result = buildStatusDescription([artifact]); + expect(result.state).toBe("failure"); + expect(result.description).toContain("failing"); + }); + + it("single artifact with security findings returns security issues", () => { + const artifact = makeArtifact([ + makeCheck({ id: "security", capability: "security", status: "fail", evidence: [{ endpoint: "security", advertised: true, responded: true, minimalShapePresent: true, diagnostics: ["[high] Bad stuff", "[medium] Sketchy thing"] }] }), + ]); + const result = buildStatusDescription([artifact]); + expect(result.state).toBe("failure"); + expect(result.description).toContain("security"); + }); + + it("multiple passing artifacts returns success with servers count", () => { + const a1 = makeArtifact([makeCheck()]); + const a2 = makeArtifact([makeCheck()]); + const result = buildStatusDescription([a1, a2]); + expect(result.state).toBe("success"); + expect(result.description).toContain("servers"); + }); + + it("multiple artifacts with one failing returns failure with across", () => { + const a1 = makeArtifact([makeCheck()]); + const a2 = makeArtifact([makeCheck({ id: "conformance", capability: "conformance", status: "fail" })]); + const result = buildStatusDescription([a1, a2]); + expect(result.state).toBe("failure"); + expect(result.description).toContain("across"); + }); + + it("tool count appears in single-server success description", () => { + const artifact = makeArtifact([ + makeCheck({ id: "tools", capability: "tools", evidence: [{ endpoint: "tools/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 7 }] }), + makeCheck({ id: "prompts", capability: "prompts", evidence: [{ endpoint: "prompts/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 4 }] }), + ]); + const result = buildStatusDescription([artifact]); + expect(result.description).toContain("7 tools"); + expect(result.description).toContain("4 prompts"); + }); +}); diff --git a/tests/history.test.ts b/tests/history.test.ts new file mode 100644 index 0000000..88c35f4 --- /dev/null +++ b/tests/history.test.ts @@ -0,0 +1,226 @@ +import { describe, it, expect } from "vitest"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import path from "node:path"; + +import { + readHistory, + appendHistory, + buildHistoryEntry, + getTrend, + renderTrendLabel, + type HistoryEntry, + type HistoryFile, + type TrendInfo, +} from "../src/history.js"; +import { makeArtifact } from "./fixtures/test-helpers.js"; + +function makeTempHistoryPath(): string { + return path.join(tmpdir(), `mcp-obs-test-${randomUUID()}`, "history.json"); +} + +function makeEntry(overrides: Partial = {}): HistoryEntry { + return { + date: "2026-03-21T00:00:00Z", + targetId: "test", + healthScore: 80, + grade: "B", + toolCount: 5, + promptCount: 2, + resourceCount: 1, + gate: "pass", + ...overrides, + }; +} + +describe("readHistory", () => { + it("returns empty history when file does not exist", async () => { + const result = await readHistory(makeTempHistoryPath()); + expect(result).toEqual({ version: 1, entries: [] }); + }); +}); + +describe("appendHistory", () => { + it("creates file and appends entry", async () => { + const histPath = makeTempHistoryPath(); + const entry = makeEntry(); + await appendHistory(entry, histPath); + const history = await readHistory(histPath); + expect(history.entries).toHaveLength(1); + expect(history.entries[0]).toEqual(entry); + }); + + it("appends to existing history", async () => { + const histPath = makeTempHistoryPath(); + await appendHistory(makeEntry({ healthScore: 70 }), histPath); + await appendHistory(makeEntry({ healthScore: 90 }), histPath); + const history = await readHistory(histPath); + expect(history.entries).toHaveLength(2); + expect(history.entries[0]!.healthScore).toBe(70); + expect(history.entries[1]!.healthScore).toBe(90); + }); +}); + +describe("buildHistoryEntry", () => { + it("extracts correct values from a RunArtifact", () => { + const artifact = makeArtifact([ + { + id: "tools", + capability: "tools", + status: "pass", + durationMs: 100, + message: "OK", + evidence: [ + { + endpoint: "tools/list", + advertised: true, + responded: true, + minimalShapePresent: true, + itemCount: 24, + }, + ], + }, + { + id: "prompts", + capability: "prompts", + status: "pass", + durationMs: 50, + message: "OK", + evidence: [ + { + endpoint: "prompts/list", + advertised: true, + responded: true, + minimalShapePresent: true, + itemCount: 3, + }, + ], + }, + { + id: "resources", + capability: "resources", + status: "pass", + durationMs: 50, + message: "OK", + evidence: [ + { + endpoint: "resources/list", + advertised: true, + responded: true, + minimalShapePresent: true, + itemCount: 5, + }, + ], + }, + ]); + artifact.healthScore = { overall: 95, grade: "A", dimensions: [] }; + + const entry = buildHistoryEntry(artifact); + expect(entry.date).toBe("2026-03-21T00:00:00Z"); + expect(entry.targetId).toBe("test"); + expect(entry.healthScore).toBe(95); + expect(entry.grade).toBe("A"); + expect(entry.toolCount).toBe(24); + expect(entry.promptCount).toBe(3); + expect(entry.resourceCount).toBe(5); + expect(entry.gate).toBe("pass"); + }); +}); + +describe("getTrend", () => { + it("returns null for empty history", () => { + const history: HistoryFile = { version: 1, entries: [] }; + expect(getTrend("test", history)).toBeNull(); + }); + + it('returns direction "new" for single entry', () => { + const history: HistoryFile = { + version: 1, + entries: [makeEntry({ healthScore: 80 })], + }; + const trend = getTrend("test", history)!; + expect(trend.direction).toBe("new"); + expect(trend.previous).toBeUndefined(); + expect(trend.delta).toBe(80); + }); + + it('returns direction "up" when score improved', () => { + const history: HistoryFile = { + version: 1, + entries: [ + makeEntry({ healthScore: 60 }), + makeEntry({ healthScore: 85 }), + ], + }; + const trend = getTrend("test", history)!; + expect(trend.direction).toBe("up"); + expect(trend.delta).toBe(25); + }); + + it('returns direction "down" when score decreased', () => { + const history: HistoryFile = { + version: 1, + entries: [ + makeEntry({ healthScore: 90 }), + makeEntry({ healthScore: 70 }), + ], + }; + const trend = getTrend("test", history)!; + expect(trend.direction).toBe("down"); + expect(trend.delta).toBe(-20); + }); + + it('returns direction "stable" when score unchanged', () => { + const history: HistoryFile = { + version: 1, + entries: [ + makeEntry({ healthScore: 80 }), + makeEntry({ healthScore: 80 }), + ], + }; + const trend = getTrend("test", history)!; + expect(trend.direction).toBe("stable"); + expect(trend.delta).toBe(0); + }); +}); + +describe("renderTrendLabel", () => { + it("formats new direction", () => { + const trend: TrendInfo = { + current: makeEntry(), + direction: "new", + delta: 80, + }; + expect(renderTrendLabel(trend)).toBe("● first run"); + }); + + it("formats up direction", () => { + const trend: TrendInfo = { + current: makeEntry({ healthScore: 90, grade: "A" }), + previous: makeEntry({ healthScore: 70, grade: "C" }), + direction: "up", + delta: 20, + }; + expect(renderTrendLabel(trend)).toBe("↗ was C (70)"); + }); + + it("formats down direction", () => { + const trend: TrendInfo = { + current: makeEntry({ healthScore: 60, grade: "D" }), + previous: makeEntry({ healthScore: 90, grade: "A" }), + direction: "down", + delta: -30, + }; + expect(renderTrendLabel(trend)).toBe("↘ was A (90)"); + }); + + it("formats stable direction", () => { + const trend: TrendInfo = { + current: makeEntry(), + previous: makeEntry(), + direction: "stable", + delta: 0, + }; + expect(renderTrendLabel(trend)).toBe("→ stable"); + }); +}); diff --git a/tests/lockfile.test.ts b/tests/lockfile.test.ts new file mode 100644 index 0000000..637b728 --- /dev/null +++ b/tests/lockfile.test.ts @@ -0,0 +1,305 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; + +import { + defaultLockPath, + readLockFile, + writeLockFile, + buildServerLockEntry, + verifyAgainstLock, + mergeLockFile, +} from "../src/lockfile.js"; +import type { LockFile, LockFileServerEntry } from "../src/lockfile.js"; +import { makeArtifact } from "./fixtures/test-helpers.js"; +import type { CheckResult } from "../src/types.js"; + +// ── Helpers ───────────────────────────────────────────────────────────────── + +function makeToolsCheck(overrides: Partial = {}): CheckResult { + return { + id: "tools", + capability: "tools", + status: "pass", + durationMs: 100, + message: "OK", + evidence: [{ + endpoint: "tools/list", + advertised: true, + responded: true, + minimalShapePresent: true, + itemCount: 2, + identifiers: ["echo", "greet"], + schemas: { + echo: { type: "object", properties: { message: { type: "string" } }, required: ["message"] }, + greet: { type: "object", properties: { name: { type: "string" } } }, + }, + ...overrides, + }], + }; +} + +function makePromptsCheck(): CheckResult { + return { + id: "prompts", + capability: "prompts", + status: "pass", + durationMs: 50, + message: "OK", + evidence: [{ + endpoint: "prompts/list", + advertised: true, + responded: true, + minimalShapePresent: true, + itemCount: 1, + identifiers: ["daily-brief"], + }], + }; +} + +function makeResourcesCheck(): CheckResult { + return { + id: "resources", + capability: "resources", + status: "pass", + durationMs: 50, + message: "OK", + evidence: [{ + endpoint: "resources/list", + advertised: true, + responded: true, + minimalShapePresent: true, + itemCount: 1, + identifiers: ["file:///data.json"], + }], + }; +} + +function fullArtifact() { + return makeArtifact([makeToolsCheck(), makePromptsCheck(), makeResourcesCheck()]); +} + +// ── Tests ─────────────────────────────────────────────────────────────────── + +describe("defaultLockPath", () => { + it("returns correct path relative to cwd", () => { + const result = defaultLockPath("/my/project"); + expect(result).toBe(path.join("/my/project", ".mcp-observatory", "lock.json")); + }); + + it("uses process.cwd() when no cwd provided", () => { + const result = defaultLockPath(); + expect(result).toBe(path.join(process.cwd(), ".mcp-observatory", "lock.json")); + }); +}); + +describe("writeLockFile + readLockFile", () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await mkdtemp(path.join(os.tmpdir(), "lockfile-test-")); + }); + + afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }); + }); + + it("round-trips correctly", async () => { + const lockPath = path.join(tmpDir, ".mcp-observatory", "lock.json"); + const lockFile: LockFile = { + version: 1, + lockedAt: "2026-03-23T00:00:00Z", + servers: [ + { + targetId: "test-server", + lockedAt: "2026-03-23T00:00:00Z", + tools: [{ name: "echo", inputSchema: { type: "object" } }], + prompts: [{ name: "daily-brief" }], + resources: [{ name: "file:///data.json", uri: "file:///data.json" }], + }, + ], + }; + + await writeLockFile(lockFile, lockPath); + const read = await readLockFile(lockPath); + expect(read).toEqual(lockFile); + }); + + it("readLockFile throws when file does not exist", async () => { + const lockPath = path.join(tmpDir, "nonexistent", "lock.json"); + await expect(readLockFile(lockPath)).rejects.toThrow(); + }); +}); + +describe("buildServerLockEntry", () => { + it("extracts tools with schemas from artifact evidence", () => { + const artifact = fullArtifact(); + const entry = buildServerLockEntry(artifact); + + expect(entry.targetId).toBe("test"); + expect(entry.tools).toHaveLength(2); + expect(entry.tools[0]!.name).toBe("echo"); + expect(entry.tools[0]!.inputSchema).toEqual({ + type: "object", + properties: { message: { type: "string" } }, + required: ["message"], + }); + expect(entry.tools[1]!.name).toBe("greet"); + expect(entry.prompts).toHaveLength(1); + expect(entry.prompts[0]!.name).toBe("daily-brief"); + expect(entry.resources).toHaveLength(1); + expect(entry.resources[0]!.name).toBe("file:///data.json"); + expect(entry.resources[0]!.uri).toBe("file:///data.json"); + }); + + it("handles artifact with no schemas (falls back to identifiers)", () => { + const artifact = makeArtifact([ + makeToolsCheck({ schemas: undefined }), + makePromptsCheck(), + makeResourcesCheck(), + ]); + const entry = buildServerLockEntry(artifact); + + expect(entry.tools).toHaveLength(2); + expect(entry.tools[0]!.name).toBe("echo"); + expect(entry.tools[0]!.inputSchema).toEqual({}); + expect(entry.tools[1]!.name).toBe("greet"); + expect(entry.tools[1]!.inputSchema).toEqual({}); + }); +}); + +describe("verifyAgainstLock", () => { + it("passes when schemas match", () => { + const artifact = fullArtifact(); + const lockEntry = buildServerLockEntry(artifact); + const result = verifyAgainstLock(lockEntry, artifact); + + expect(result.passed).toBe(true); + expect(result.drift).toHaveLength(0); + }); + + it("detects added tool", () => { + const artifact = fullArtifact(); + const lockEntry = buildServerLockEntry(artifact); + + // Remove a tool from the lock so the live version has an "extra" + lockEntry.tools = lockEntry.tools.filter((t) => t.name !== "greet"); + + const result = verifyAgainstLock(lockEntry, artifact); + expect(result.passed).toBe(false); + expect(result.drift).toContainEqual( + expect.objectContaining({ category: "tools", name: "greet", change: "added" }), + ); + }); + + it("detects removed tool", () => { + const artifact = makeArtifact([ + makeToolsCheck({ + identifiers: ["echo"], + schemas: { + echo: { type: "object", properties: { message: { type: "string" } }, required: ["message"] }, + }, + itemCount: 1, + }), + makePromptsCheck(), + makeResourcesCheck(), + ]); + + // Lock has both echo + greet, but live only has echo + const lockEntry = buildServerLockEntry(fullArtifact()); + const result = verifyAgainstLock(lockEntry, artifact); + + expect(result.passed).toBe(false); + expect(result.drift).toContainEqual( + expect.objectContaining({ category: "tools", name: "greet", change: "removed" }), + ); + }); + + it("detects changed tool schema", () => { + const artifact = fullArtifact(); + const lockEntry = buildServerLockEntry(artifact); + + // Mutate the locked schema to differ + lockEntry.tools[0]!.inputSchema = { type: "object", properties: { msg: { type: "number" } } }; + + const result = verifyAgainstLock(lockEntry, artifact); + expect(result.passed).toBe(false); + expect(result.drift).toContainEqual( + expect.objectContaining({ category: "tools", name: "echo", change: "schema changed" }), + ); + }); + + it("detects added/removed prompts", () => { + const artifact = fullArtifact(); + const lockEntry = buildServerLockEntry(artifact); + + // Add an extra prompt to the lock that doesn't exist in live + lockEntry.prompts.push({ name: "weekly-report" }); + + const result = verifyAgainstLock(lockEntry, artifact); + expect(result.passed).toBe(false); + expect(result.drift).toContainEqual( + expect.objectContaining({ category: "prompts", name: "weekly-report", change: "removed" }), + ); + }); +}); + +describe("mergeLockFile", () => { + it("replaces existing server entry", () => { + const existing: LockFile = { + version: 1, + lockedAt: "2026-03-22T00:00:00Z", + servers: [ + { + targetId: "server-a", + lockedAt: "2026-03-22T00:00:00Z", + tools: [{ name: "old-tool", inputSchema: {} }], + prompts: [], + resources: [], + }, + ], + }; + + const replacement: LockFileServerEntry = { + targetId: "server-a", + lockedAt: "2026-03-23T00:00:00Z", + tools: [{ name: "new-tool", inputSchema: { type: "object" } }], + prompts: [], + resources: [], + }; + + const merged = mergeLockFile(existing, [replacement]); + expect(merged.servers).toHaveLength(1); + expect(merged.servers[0]!.tools[0]!.name).toBe("new-tool"); + }); + + it("adds new server entry", () => { + const existing: LockFile = { + version: 1, + lockedAt: "2026-03-22T00:00:00Z", + servers: [ + { + targetId: "server-a", + lockedAt: "2026-03-22T00:00:00Z", + tools: [], + prompts: [], + resources: [], + }, + ], + }; + + const newEntry: LockFileServerEntry = { + targetId: "server-b", + lockedAt: "2026-03-23T00:00:00Z", + tools: [{ name: "b-tool", inputSchema: {} }], + prompts: [], + resources: [], + }; + + const merged = mergeLockFile(existing, [newEntry]); + expect(merged.servers).toHaveLength(2); + expect(merged.servers.map((s) => s.targetId)).toContain("server-a"); + expect(merged.servers.map((s) => s.targetId)).toContain("server-b"); + }); +}); diff --git a/tests/pr-comment-matrix.test.ts b/tests/pr-comment-matrix.test.ts new file mode 100644 index 0000000..4e8c631 --- /dev/null +++ b/tests/pr-comment-matrix.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; + +import { renderMatrixComment } from "../src/reporters/pr-comment-matrix.js"; +import type { TrendInfo } from "../src/types.js"; +import { makeArtifact } from "./fixtures/test-helpers.js"; + +function makeTrend(direction: TrendInfo["direction"], grade = "B", score = 80): TrendInfo { + return { + current: { date: "2026-03-23", targetId: "test", healthScore: 90, grade: "A", toolCount: 5, promptCount: 0, resourceCount: 0, gate: "pass" }, + previous: { date: "2026-03-22", targetId: "test", healthScore: score, grade: grade as TrendInfo["current"]["grade"], toolCount: 5, promptCount: 0, resourceCount: 0, gate: "pass" }, + direction, + delta: direction === "up" ? 10 : direction === "down" ? -10 : 0, + }; +} + +describe("renderMatrixComment", () => { + it("shows 'No servers scanned' for empty rows", () => { + const out = renderMatrixComment([]); + expect(out).toContain("No servers scanned"); + expect(out).toContain("MCP Observatory"); + }); + + it("shows table with checkmark for single passing server", () => { + const artifact = makeArtifact([ + { id: "tools", capability: "tools", status: "pass", durationMs: 100, message: "OK", evidence: [{ endpoint: "tools/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 5 }] }, + ]); + artifact.healthScore = { overall: 95, grade: "A", dimensions: [] }; + const out = renderMatrixComment([{ artifact }]); + expect(out).toContain("1 server scanned"); + expect(out).toContain("| test |"); + expect(out).toContain("**A** (95)"); + expect(out).toContain("| 5 |"); + expect(out).toContain("✅"); + expect(out).not.toContain("
"); + }); + + it("shows correct table for multiple servers with mixed results", () => { + const passing = makeArtifact([ + { id: "tools", capability: "tools", status: "pass", durationMs: 100, message: "OK", evidence: [{ endpoint: "tools/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 10 }] }, + ]); + passing.target = { ...passing.target, targetId: "server-a" }; + passing.healthScore = { overall: 95, grade: "A", dimensions: [] }; + + const failing = makeArtifact([ + { id: "tools", capability: "tools", status: "pass", durationMs: 100, message: "OK", evidence: [{ endpoint: "tools/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 3 }] }, + { id: "security", capability: "security", status: "fail", durationMs: 50, message: "1 finding", evidence: [{ endpoint: "security/scan", advertised: true, responded: true, minimalShapePresent: true, diagnostics: ['[high] Tool "exec" allows command injection'] }] }, + ]); + failing.target = { ...failing.target, targetId: "server-b" }; + failing.healthScore = { overall: 40, grade: "D", dimensions: [] }; + + const out = renderMatrixComment([ + { artifact: passing }, + { artifact: failing }, + ]); + expect(out).toContain("2 servers scanned"); + expect(out).toContain("| server-a |"); + expect(out).toContain("| server-b |"); + expect(out).toContain("✅"); + expect(out).toContain("🔴"); + }); + + it("adds details block with red icon for security findings", () => { + const artifact = makeArtifact([ + { id: "security", capability: "security", status: "fail", durationMs: 50, message: "2 findings", evidence: [{ endpoint: "security/scan", advertised: true, responded: true, minimalShapePresent: true, diagnostics: ['[high] Tool "exec" allows command injection', '[medium] Tool "read" exposes file system'] }] }, + ]); + artifact.target = { ...artifact.target, targetId: "risky-server" }; + const out = renderMatrixComment([{ artifact }]); + expect(out).toContain("
"); + expect(out).toContain("🔴 risky-server — 2 issues"); + expect(out).toContain("`high`"); + expect(out).toContain("command injection"); + expect(out).toContain("`medium`"); + expect(out).toContain("
"); + }); + + it("adds details block with warning icon for quality-only findings", () => { + const artifact = makeArtifact([ + { id: "schema-quality", capability: "schema-quality", status: "partial", durationMs: 30, message: "1 issue", evidence: [{ endpoint: "schema-quality", advertised: true, responded: true, minimalShapePresent: true, diagnostics: ['[warning] tool "search": Missing description'] }] }, + ]); + artifact.target = { ...artifact.target, targetId: "quality-server" }; + const out = renderMatrixComment([{ artifact }]); + expect(out).toContain("
"); + expect(out).toContain("⚠️ quality-server — 1 issue"); + expect(out).toContain("`warning`"); + expect(out).toContain("Missing description"); + expect(out).toContain("⚠️ 1 quality"); + }); + + it("shows trend arrow in health column when trend is provided", () => { + const artifact = makeArtifact([ + { id: "tools", capability: "tools", status: "pass", durationMs: 100, message: "OK", evidence: [{ endpoint: "tools/list", advertised: true, responded: true, minimalShapePresent: true, itemCount: 5 }] }, + ]); + artifact.healthScore = { overall: 90, grade: "A", dimensions: [] }; + + const upOut = renderMatrixComment([{ artifact, trend: makeTrend("up") }]); + expect(upOut).toContain("**A** (90) ↗"); + + const downOut = renderMatrixComment([{ artifact, trend: makeTrend("down") }]); + expect(downOut).toContain("**A** (90) ↘"); + + const stableOut = renderMatrixComment([{ artifact, trend: makeTrend("stable") }]); + expect(stableOut).toContain("**A** (90) →"); + + // "new" direction should NOT show an arrow + const newOut = renderMatrixComment([{ artifact, trend: makeTrend("new") }]); + expect(newOut).toContain("**A** (90)"); + expect(newOut).not.toContain("↗"); + expect(newOut).not.toContain("↘"); + expect(newOut).not.toContain("→"); + }); + + it("always includes footer", () => { + const out = renderMatrixComment([{ artifact: makeArtifact([]) }]); + expect(out).toContain("MCP Observatory"); + expect(out).toContain("KryptosAI/mcp-observatory"); + }); + + it("has correct column headers in the table", () => { + const artifact = makeArtifact([]); + const out = renderMatrixComment([{ artifact }]); + expect(out).toContain("| Server | Health | Tools | Issues |"); + }); +});