feat: 5 CI features — lock files, matrix comments, commit status, nightly scans, trends

.github/workflows/nightly-scan.template.yml

@@ -0,0 +1,50 @@
+# MCP Observatory Nightly Scan
+# Copy this file to .github/workflows/observatory-nightly.yml in your repo.
+# It scans your MCP servers daily and opens an issue if regressions are found.
+name: MCP Observatory Nightly Scan
+on:
+  schedule:
+    - cron: '0 6 * * *'  # 6 AM UTC daily
+  workflow_dispatch: {}
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - name: Install MCP Observatory
+        run: npm install -g @kryptosai/mcp-observatory
+      - name: Run nightly scan
+        run: mcp-observatory scan --no-color
+      - name: Generate CI report
+        id: report
+        run: |
+          mcp-observatory ci-report --format json > /tmp/observatory-report.json
+          echo "has_regressions=$(node -e "console.log(JSON.parse(require('fs').readFileSync('/tmp/observatory-report.json','utf8')).hasRegressions)")" >> $GITHUB_OUTPUT
+      - name: Create or update issue
+        if: steps.report.outputs.has_regressions == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TITLE=$(node -e "console.log(JSON.parse(require('fs').readFileSync('/tmp/observatory-report.json','utf8')).title)")
+          BODY_FILE="/tmp/observatory-body.md"
+          node -e "process.stdout.write(JSON.parse(require('fs').readFileSync('/tmp/observatory-report.json','utf8')).body)" > "$BODY_FILE"
+          EXISTING=$(gh issue list --label "mcp-observatory" --state open --json number -q '.[0].number // empty' 2>/dev/null || echo "")
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body-file "$BODY_FILE"
+          else
+            gh issue create --title "$TITLE" --body-file "$BODY_FILE" --label "mcp-observatory"
+          fi
+      - name: Upload scan artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: observatory-nightly-${{ github.run_number }}
+          path: .mcp-observatory/runs/
+          retention-days: 30

action/action.yml

@@ -11,6 +11,9 @@ inputs:
   target:
     description: "Path to a target config JSON file (alternative to command)"
     required: false
+  targets:
+    description: "Path to MCP config file for multi-server matrix scan (scans all servers, generates matrix comment)"
+    required: false
   baseline:
     description: "Path to a baseline cassette file for regression verification"
     required: false
@@ -27,11 +30,15 @@ inputs:
     required: false
     default: "true"
   comment-on-pr:
-    description: "Post a markdown report as a PR comment"
+    description: "Post a report as a PR comment"
+    required: false
+    default: "true"
+  set-status:
+    description: "Set a commit status check (green/red) on the HEAD SHA"
     required: false
     default: "true"
   github-token:
-    description: "GitHub token for PR comments"
+    description: "GitHub token for PR comments and commit statuses"
     required: false
     default: ${{ github.token }}
   node-version:
@@ -65,57 +72,79 @@ runs:
       env:
         INPUT_COMMAND: ${{ inputs.command }}
         INPUT_TARGET: ${{ inputs.target }}
+        INPUT_TARGETS: ${{ inputs.targets }}
         INPUT_DEEP: ${{ inputs.deep }}
         INPUT_SECURITY: ${{ inputs.security }}
       run: |
-        # Build command as a bash array to prevent injection
-        CMD_ARRAY=()
-
-        if [ -n "$INPUT_TARGET" ]; then
-          CMD_ARRAY=(mcp-observatory run --target "$INPUT_TARGET")
-          if [ "$INPUT_DEEP" = "true" ]; then
-            CMD_ARRAY+=(--invoke)
-          fi
-        elif [ -n "$INPUT_COMMAND" ]; then
-          CMD_ARRAY=(mcp-observatory test "$INPUT_COMMAND")
-        else
-          echo "::error::Either 'command' or 'target' input is required"
-          exit 1
-        fi
-
-        if [ "$INPUT_SECURITY" = "true" ]; then
-          CMD_ARRAY+=(--security)
-        fi
-
-        CMD_ARRAY+=(--no-color)
-
-        # Run and capture output
         ARTIFACT_DIR="${RUNNER_TEMP}/observatory"
         mkdir -p "$ARTIFACT_DIR"
 
-        set +e
-        "${CMD_ARRAY[@]}" 2>&1 | tee "${ARTIFACT_DIR}/output.txt"
-        EXIT_CODE=$?
-        set -e
-
-        # Find the artifact file
-        ARTIFACT_PATH=$(find .mcp-observatory/runs -name "*.json" -type f 2>/dev/null | sort | tail -1)
+        # Multi-server matrix scan
+        if [ -n "$INPUT_TARGETS" ]; then
+          SCAN_ARRAY=(mcp-observatory scan --config "$INPUT_TARGETS")
+          if [ "$INPUT_SECURITY" = "true" ]; then
+            SCAN_ARRAY+=(--security)
+          fi
+          SCAN_ARRAY+=(--no-color)
+
+          set +e
+          "${SCAN_ARRAY[@]}" 2>&1 | tee "${ARTIFACT_DIR}/output.txt"
+          set -e
+
+          # Collect all artifacts and determine gate
+          GATE="pass"
+          for f in .mcp-observatory/runs/*.json; do
+            [ -f "$f" ] || continue
+            FILE_GATE=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$f','utf8')).gate)" 2>/dev/null || echo "unknown")
+            if [ "$FILE_GATE" = "fail" ]; then
+              GATE="fail"
+            fi
+          done
+          echo "gate=${GATE}" >> "$GITHUB_OUTPUT"
+          echo "artifact_path=${ARTIFACT_DIR}" >> "$GITHUB_OUTPUT"
 
-        if [ -n "$ARTIFACT_PATH" ]; then
-          cp "$ARTIFACT_PATH" "${ARTIFACT_DIR}/run.json"
-          echo "artifact_path=${ARTIFACT_DIR}/run.json" >> "$GITHUB_OUTPUT"
+          # Generate CI report for PR comment
+          mcp-observatory ci-report --format markdown --no-color > "${ARTIFACT_DIR}/report.md" 2>/dev/null || true
 
-          # Extract gate from artifact using node (avoid python dependency)
-          GATE=$(node -e "const fs=require('fs'); const d=JSON.parse(fs.readFileSync('${ARTIFACT_DIR}/run.json','utf8')); console.log(d.gate)" 2>/dev/null || echo "unknown")
-          echo "gate=${GATE}" >> "$GITHUB_OUTPUT"
+        # Single-server scan
         else
-          echo "gate=fail" >> "$GITHUB_OUTPUT"
-          echo "artifact_path=" >> "$GITHUB_OUTPUT"
-        fi
+          CMD_ARRAY=()
+          if [ -n "$INPUT_TARGET" ]; then
+            CMD_ARRAY=(mcp-observatory run --target "$INPUT_TARGET")
+            if [ "$INPUT_DEEP" = "true" ]; then
+              CMD_ARRAY+=(--invoke)
+            fi
+          elif [ -n "$INPUT_COMMAND" ]; then
+            CMD_ARRAY=(mcp-observatory test "$INPUT_COMMAND")
+          else
+            echo "::error::Either 'command', 'target', or 'targets' input is required"
+            exit 1
+          fi
 
-        # Generate PR comment report
-        if [ -n "$ARTIFACT_PATH" ]; then
-          mcp-observatory report "$ARTIFACT_PATH" --format pr-comment --no-color > "${ARTIFACT_DIR}/report.md" 2>/dev/null || true
+          if [ "$INPUT_SECURITY" = "true" ]; then
+            CMD_ARRAY+=(--security)
+          fi
+          CMD_ARRAY+=(--no-color)
+
+          set +e
+          "${CMD_ARRAY[@]}" 2>&1 | tee "${ARTIFACT_DIR}/output.txt"
+          set -e
+
+          ARTIFACT_PATH=$(find .mcp-observatory/runs -name "*.json" -type f 2>/dev/null | sort | tail -1)
+          if [ -n "$ARTIFACT_PATH" ]; then
+            cp "$ARTIFACT_PATH" "${ARTIFACT_DIR}/run.json"
+            echo "artifact_path=${ARTIFACT_DIR}/run.json" >> "$GITHUB_OUTPUT"
+            GATE=$(node -e "const fs=require('fs'); const d=JSON.parse(fs.readFileSync('${ARTIFACT_DIR}/run.json','utf8')); console.log(d.gate)" 2>/dev/null || echo "unknown")
+            echo "gate=${GATE}" >> "$GITHUB_OUTPUT"
+          else
+            echo "gate=fail" >> "$GITHUB_OUTPUT"
+            echo "artifact_path=" >> "$GITHUB_OUTPUT"
+          fi
+
+          # Generate PR comment report
+          if [ -n "$ARTIFACT_PATH" ]; then
+            mcp-observatory report "$ARTIFACT_PATH" --format pr-comment --no-color > "${ARTIFACT_DIR}/report.md" 2>/dev/null || true
+          fi
         fi
 
     - name: Verify against baseline
@@ -126,15 +155,12 @@ runs:
         INPUT_COMMAND: ${{ inputs.command }}
         INPUT_TARGET: ${{ inputs.target }}
       run: |
-        # Build verify command as a bash array
         VERIFY_ARRAY=(mcp-observatory verify "$INPUT_BASELINE")
-
         if [ -n "$INPUT_TARGET" ]; then
           VERIFY_ARRAY+=(--target "$INPUT_TARGET")
         elif [ -n "$INPUT_COMMAND" ]; then
           VERIFY_ARRAY+=("$INPUT_COMMAND")
         fi
-
         VERIFY_ARRAY+=(--no-color)
 
         set +e
@@ -146,6 +172,26 @@ runs:
           echo "::warning::Baseline verification detected changes"
         fi
 
+    - name: Set commit status
+      if: inputs.set-status == 'true' && github.event_name == 'pull_request'
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.github-token }}
+        GATE: ${{ steps.run.outputs.gate }}
+      run: |
+        STATE="success"
+        DESC="All clear"
+        if [ "$GATE" = "fail" ]; then
+          STATE="failure"
+          DESC="Issues detected"
+        fi
+        gh api "repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }}" \
+          -f state="$STATE" \
+          -f description="$DESC" \
+          -f context="MCP Observatory" \
+          -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+          2>/dev/null || echo "::warning::Could not set commit status"
+
     - name: Comment on PR
       if: inputs.comment-on-pr == 'true' && github.event_name == 'pull_request'
       shell: bash
@@ -161,7 +207,6 @@ runs:
           exit 0
         fi
 
-        # pr-comment format already includes header and footer
         COMMENT_FILE="${RUNNER_TEMP}/observatory/comment.md"
         cp "$REPORT_FILE" "$COMMENT_FILE"
 

src/ci-issue.ts

@@ -0,0 +1,135 @@
+import { execFile } from "node:child_process";
+import { writeFile, unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+/**
+ * Run a command via execFile and return { stdout, stderr }.
+ */
+function execCommand(
+  cmd: string,
+  args: string[],
+): Promise<{ stdout: string; stderr: string }> {
+  return new Promise((resolve, reject) => {
+    execFile(cmd, args, (error, stdout, stderr) => {
+      if (error) {
+        reject(error as Error);
+      } else {
+        resolve({ stdout: stdout ?? "", stderr: stderr ?? "" });
+      }
+    });
+  });
+}
+
+/** Executor type for dependency injection (testability). */
+export type CommandExecutor = (
+  cmd: string,
+  args: string[],
+) => Promise<{ stdout: string; stderr: string }>;
+
+/**
+ * Find an existing open issue with the given label.
+ * Returns the issue number or null if none found / gh unavailable.
+ */
+export async function findExistingIssue(
+  repo: string,
+  label: string,
+  exec: CommandExecutor = execCommand,
+): Promise<number | null> {
+  try {
+    const { stdout } = await exec("gh", [
+      "issue",
+      "list",
+      "--repo",
+      repo,
+      "--label",
+      label,
+      "--state",
+      "open",
+      "--json",
+      "number",
+      "--limit",
+      "1",
+    ]);
+    const parsed: unknown = JSON.parse(stdout);
+    if (Array.isArray(parsed) && parsed.length > 0) {
+      const first = parsed[0] as Record<string, unknown>;
+      if (typeof first["number"] === "number") {
+        return first["number"];
+      }
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Create a new issue or comment on an existing one.
+ * Uses --body-file to avoid shell injection.
+ * Returns the issue number.
+ */
+export async function createOrUpdateIssue(options: {
+  repo: string;
+  title: string;
+  body: string;
+  labels: string[];
+  exec?: CommandExecutor;
+}): Promise<number> {
+  const exec = options.exec ?? execCommand;
+  const bodyFile = path.join(
+    tmpdir(),
+    `mcp-observatory-issue-${Date.now()}.md`,
+  );
+
+  try {
+    await writeFile(bodyFile, options.body, "utf8");
+
+    const existingNumber = await findExistingIssue(
+      options.repo,
+      options.labels[0] ?? "mcp-observatory",
+      exec,
+    );
+
+    if (existingNumber !== null) {
+      await exec("gh", [
+        "issue",
+        "comment",
+        String(existingNumber),
+        "--repo",
+        options.repo,
+        "--body-file",
+        bodyFile,
+      ]);
+      return existingNumber;
+    }
+
+    // Create new issue
+    const args = [
+      "issue",
+      "create",
+      "--repo",
+      options.repo,
+      "--title",
+      options.title,
+      "--body-file",
+      bodyFile,
+    ];
+    for (const label of options.labels) {
+      args.push("--label", label);
+    }
+
+    const { stdout } = await exec("gh", args);
+    // gh issue create prints the URL, e.g. https://github.com/owner/repo/issues/42
+    const match = /\/issues\/(\d+)/.exec(stdout.trim());
+    if (match?.[1]) {
+      return parseInt(match[1], 10);
+    }
+
+    throw new Error(
+      `Failed to parse issue number from gh output: ${stdout.trim()}`,
+    );
+  } finally {
+    await unlink(bodyFile).catch(() => {});
+  }
+}

src/ci.ts

@@ -2,3 +2,10 @@ import ci from "ci-info";
 
 export const isCI: boolean = ci.isCI;
 export const ciName: string | null = ci.name;
+
+export function getGitHubContext(): { sha: string; repo: string } | null {
+  const sha = process.env["GITHUB_SHA"];
+  const repo = process.env["GITHUB_REPOSITORY"];
+  if (!sha || !repo) return null;
+  return { sha, repo };
+}