harness_audit.py

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import os
import re
import sys
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path

try:
    import tomllib
except ModuleNotFoundError:  # pragma: no cover
    tomllib = None


CONTEXT_NAMES = {
    "AGENTS.md",
    "AGENTS.override.md",
    "CLAUDE.md",
    "CLAUDE.local.md",
    "GEMINI.md",
    "COPILOT-INSTRUCTIONS.md",
    ".cursorrules",
    ".windsurfrules",
    ".clinerules",
}

SPECIAL_CONTEXT_PATHS = [
    Path(".github/copilot-instructions.md"),
]

DOC_HINTS = [
    "docs",
    "architecture",
    "adr",
    "decision",
    "design",
    "spec",
    "plan",
    "runbook",
    "playbook",
]

SKILL_DIR_HINTS = [
    Path(".codex/skills"),
    Path(".claude/skills"),
    Path(".claude/agents"),
    Path(".github/workflows"),
]

IGNORE_DIRS = {
    ".git",
    ".worktrees",
    ".wrangler",
    ".npm-cache",
    ".npm-cache-local",
    "node_modules",
    "dist",
    "build",
    "coverage",
    "playwright-report",
    "test-results",
    "tmp",
    "deploy_tmp",
}

TEST_KEYS = ("test", "check", "verify", "e2e", "unit", "integration", "coverage")
BUILD_KEYS = ("build", "dev", "preview", "start")
LINT_KEYS = ("lint", "typecheck", "format", "ruff", "mypy")


@dataclass
class ContextFile:
    path: str
    lines: int
    size_bytes: int
    signals: list[str]


def rel(root: Path, path: Path) -> str:
    try:
        return path.relative_to(root).as_posix()
    except ValueError:
        return path.as_posix()


def read_text(path: Path) -> str:
    try:
        return path.read_text(encoding="utf-8")
    except UnicodeDecodeError:
        return path.read_text(encoding="utf-8", errors="replace")


def walk_files(root: Path):
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [name for name in dirnames if name not in IGNORE_DIRS]
        base = Path(dirpath)
        for filename in filenames:
            yield base / filename


def iter_context_files(root: Path) -> list[Path]:
    found: set[Path] = set()
    for path in walk_files(root):
        if path.name in CONTEXT_NAMES:
            found.add(path)
    for special in SPECIAL_CONTEXT_PATHS:
        candidate = root / special
        if candidate.is_file():
            found.add(candidate)
    return sorted(found)


def classify_context_file(root: Path, path: Path, max_context_lines: int) -> ContextFile:
    text = read_text(path)
    lines = text.count("\n") + (1 if text else 0)
    size_bytes = path.stat().st_size
    lowered = text.lower()
    signals: list[str] = []

    if path.parent == root:
        signals.append("root")
    if lines > max_context_lines or size_bytes > 12_000:
        signals.append("bloated")
    if "auto-generated" in lowered or "generated by" in lowered or "do not edit" in lowered:
        signals.append("generated")
    if "build" in lowered and "test" in lowered:
        signals.append("commands")
    if "docs/" in lowered or "@docs/" in lowered or "see " in lowered:
        signals.append("routes")
    if "todo" in lowered or "tbd" in lowered:
        signals.append("todo")
    if not signals:
        signals.append("plain")

    return ContextFile(path=rel(root, path), lines=lines, size_bytes=size_bytes, signals=signals)


def discover_docs(root: Path) -> list[str]:
    hits: set[str] = set()
    for path in walk_files(root):
        parts = [part.lower() for part in path.parts]
        if any(hint in part for hint in DOC_HINTS for part in parts):
            hits.add(rel(root, path))
    return sorted(hits)[:40]


def parse_package_json(root: Path) -> dict[str, str]:
    path = root / "package.json"
    if not path.is_file():
        return {}
    try:
        data = json.loads(read_text(path))
    except json.JSONDecodeError:
        return {}
    scripts = data.get("scripts")
    if isinstance(scripts, dict):
        return {str(k): str(v) for k, v in scripts.items()}
    return {}


def parse_pyproject(root: Path) -> dict[str, object]:
    path = root / "pyproject.toml"
    if not path.is_file() or tomllib is None:
        return {}
    try:
        data = tomllib.loads(read_text(path))
    except Exception:
        return {}
    return data


def parse_makefile(root: Path) -> list[str]:
    path = root / "Makefile"
    if not path.is_file():
        return []
    targets: list[str] = []
    for line in read_text(path).splitlines():
        if line.startswith("\t") or line.startswith("#"):
            continue
        match = re.match(r"^([A-Za-z0-9_.-]+):(?:\s|$)", line)
        if match:
            targets.append(match.group(1))
    return targets[:30]


def extract_named_scripts(scripts: dict[str, str], keys: tuple[str, ...]) -> dict[str, str]:
    picked: dict[str, str] = {}
    for name, command in scripts.items():
        lowered = name.lower()
        if any(key in lowered for key in keys):
            picked[name] = command
    return picked


def detect_ci(root: Path) -> list[str]:
    workflow_dir = root / ".github" / "workflows"
    if not workflow_dir.is_dir():
        return []
    return sorted(rel(root, path) for path in workflow_dir.iterdir() if path.is_file())


def detect_browser_surface(root: Path, scripts: dict[str, str]) -> list[str]:
    hits: list[str] = []
    for name in scripts:
        lowered = name.lower()
        if any(token in lowered for token in ("playwright", "cypress", "puppeteer", "selenium", "e2e")):
            hits.append(f"package.json:{name}")
    for candidate in ("playwright.config.ts", "playwright.config.js", "cypress.config.ts", "cypress.config.js"):
        if (root / candidate).is_file():
            hits.append(candidate)
    return sorted(set(hits))


def detect_skill_surfaces(root: Path) -> list[str]:
    hits: list[str] = []
    for hint in SKILL_DIR_HINTS:
        candidate = root / hint
        if candidate.exists():
            hits.append(rel(root, candidate))
    return hits


def detect_risks(
    contexts: list[ContextFile],
    docs: list[str],
    build_scripts: dict[str, str],
    test_scripts: dict[str, str],
    lint_scripts: dict[str, str],
    ci_files: list[str],
    skill_surfaces: list[str],
    browser_surfaces: list[str],
) -> list[str]:
    risks: list[str] = []
    root_contexts = [ctx for ctx in contexts if "root" in ctx.signals]

    if not root_contexts:
        risks.append("No root context file found. Add a short router file with project identity, commands, key docs, and repo-specific invariants.")
    if any("bloated" in ctx.signals for ctx in root_contexts):
        risks.append("Root context file is bloated. Cut it down and move detail into docs or skills.")
    if any("generated" in ctx.signals for ctx in root_contexts):
        risks.append("Root context appears auto-generated. Replace generic summaries with minimal, human-written instructions.")
    if not docs:
        risks.append("No obvious docs system of record found. Add versioned docs for architecture, plans, and decisions.")
    if not build_scripts or not test_scripts:
        risks.append("Build or test commands are unclear. Make them explicit in repo docs and root context.")
    if not lint_scripts:
        risks.append("No obvious lint or typecheck commands detected. Add at least one mechanical quality gate.")
    if not ci_files:
        risks.append("No CI workflows detected. Add a build/test gate before granting more autonomy.")
    if not skill_surfaces:
        risks.append("No skill or agent surface detected. Move repeated workflows into skills instead of bloating root context.")
    if not browser_surfaces:
        risks.append("No browser or manual testing surface detected. If the repo ships UI, add one.")

    return risks


def recommend_actions(risks: list[str]) -> list[str]:
    actions: list[str] = []

    if any("root context" in risk.lower() for risk in risks):
        actions.append("Shrink or create the root context file first. Keep only one-line identity, commands, key docs, and non-obvious invariants.")
    if any("docs system of record" in risk.lower() for risk in risks):
        actions.append("Create or repair `docs/` as the system of record for architecture, plans, and decisions.")
    if any("build or test commands" in risk.lower() for risk in risks):
        actions.append("Write explicit build/test/lint/typecheck commands into the repo map and verify they actually work.")
    if any("ci workflows" in risk.lower() for risk in risks):
        actions.append("Add a minimal CI gate before enabling more autonomous or background agent flows.")
    if any("skill or agent surface" in risk.lower() for risk in risks):
        actions.append("Extract repeated workflows into skills, scripts, or hooks instead of repeating them in prompts.")
    if any("browser or manual testing" in risk.lower() for risk in risks):
        actions.append("Add a browser or manual testing path for behavioral changes.")

    return actions[:5]


def build_report(root: Path, max_context_lines: int) -> dict[str, object]:
    contexts = [classify_context_file(root, path, max_context_lines) for path in iter_context_files(root)]
    docs = discover_docs(root)
    package_scripts = parse_package_json(root)
    pyproject = parse_pyproject(root)
    make_targets = parse_makefile(root)
    build_scripts = extract_named_scripts(package_scripts, BUILD_KEYS)
    test_scripts = extract_named_scripts(package_scripts, TEST_KEYS)
    lint_scripts = extract_named_scripts(package_scripts, LINT_KEYS)
    ci_files = detect_ci(root)
    browser_surfaces = detect_browser_surface(root, package_scripts)
    skill_surfaces = detect_skill_surfaces(root)
    risks = detect_risks(
        contexts=contexts,
        docs=docs,
        build_scripts=build_scripts,
        test_scripts=test_scripts,
        lint_scripts=lint_scripts,
        ci_files=ci_files,
        skill_surfaces=skill_surfaces,
        browser_surfaces=browser_surfaces,
    )
    actions = recommend_actions(risks)
    root_contexts = [ctx for ctx in contexts if "root" in ctx.signals]

    return {
        "generated": datetime.now(timezone.utc).isoformat(),
        "repository": str(root),
        "summary": {
            "root_context": [ctx.path for ctx in root_contexts],
            "context_files": len(contexts),
            "docs_hits": len(docs),
            "ci_workflows": len(ci_files),
            "skill_or_agent_surfaces": len(skill_surfaces),
            "build_scripts": len(build_scripts),
            "test_scripts": len(test_scripts),
            "lint_or_typecheck_scripts": len(lint_scripts),
            "browser_or_manual_testing_surfaces": len(browser_surfaces),
            "max_context_lines": max_context_lines,
        },
        "context_files": [asdict(ctx) for ctx in contexts[:40]],
        "commands": {
            "build": build_scripts,
            "test": test_scripts,
            "lint": lint_scripts,
            "pyproject_keys": sorted(pyproject.keys()),
            "make_targets": make_targets,
        },
        "docs_and_surfaces": {
            "docs_hits": docs[:20],
            "skill_or_agent_surfaces": skill_surfaces,
            "ci_workflows": ci_files,
            "browser_or_manual_testing": browser_surfaces,
        },
        "risks": risks,
        "immediate_actions": actions,
    }


def print_section(title: str) -> None:
    print()
    print(f"## {title}")


def print_markdown(report: dict[str, object]) -> None:
    summary = report["summary"]
    commands = report["commands"]
    docs_and_surfaces = report["docs_and_surfaces"]

    print("# Harness Audit")
    print()
    print(f"- Generated: {report['generated']}")
    print(f"- Repository: `{report['repository']}`")
    print(f"- Context files: `{summary['context_files']}`")
    print(f"- Docs hits: `{summary['docs_hits']}`")
    print(f"- CI workflows: `{summary['ci_workflows']}`")
    print(f"- Skill or agent surfaces: `{summary['skill_or_agent_surfaces']}`")

    print_section("Summary")
    root_context = summary["root_context"] or ["none"]
    print(f"- Root context: `{', '.join(root_context)}`")
    print(f"- Build scripts detected: `{summary['build_scripts']}`")
    print(f"- Test scripts detected: `{summary['test_scripts']}`")
    print(f"- Lint or typecheck scripts detected: `{summary['lint_or_typecheck_scripts']}`")
    print(f"- Browser or manual testing surfaces: `{summary['browser_or_manual_testing_surfaces']}`")
    print(f"- Context line budget: `{summary['max_context_lines']}`")

    print_section("Context Files")
    context_files = report["context_files"]
    if not context_files:
        print("- None found")
    else:
        print("| Path | Lines | Size | Signals |")
        print("| --- | ---: | ---: | --- |")
        for ctx in context_files:
            signals = ", ".join(ctx["signals"])
            print(f"| `{ctx['path']}` | {ctx['lines']} | {ctx['size_bytes']} B | {signals} |")

    print_section("Commands")
    print("### package.json")
    for group_name, key in (("Build", "build"), ("Test", "test"), ("Lint", "lint")):
        group = commands[key]
        if not group:
            continue
        print(f"- {group_name}:")
        for name, command in sorted(group.items()):
            print(f"  - `{name}` -> `{command}`")

    if commands["pyproject_keys"]:
        print("### pyproject.toml")
        print(f"- Top-level keys: `{', '.join(commands['pyproject_keys'])}`")

    if commands["make_targets"]:
        print("### Makefile")
        print(f"- Targets: `{', '.join(commands['make_targets'])}`")

    print_section("Docs And Agent Surfaces")
    docs_hits = docs_and_surfaces["docs_hits"]
    if docs_hits:
        print("- Docs hits:")
        for item in docs_hits:
            print(f"  - `{item}`")
    else:
        print("- No obvious docs system of record detected")

    for label, key in (
        ("Skill or agent surfaces", "skill_or_agent_surfaces"),
        ("CI workflows", "ci_workflows"),
        ("Browser or manual testing", "browser_or_manual_testing"),
    ):
        items = docs_and_surfaces[key]
        if not items:
            continue
        print(f"- {label}:")
        for item in items:
            print(f"  - `{item}`")

    print_section("Risks")
    risks = report["risks"]
    if not risks:
        print("- No obvious harness risks detected")
    else:
        for risk in risks:
            print(f"- {risk}")

    print_section("Immediate Actions")
    actions = report["immediate_actions"]
    if not actions:
        print("- No immediate actions suggested")
    else:
        for idx, action in enumerate(actions, start=1):
            print(f"{idx}. {action}")


def main() -> int:
    parser = argparse.ArgumentParser(description="Audit a repository for coding-agent harness health.")
    parser.add_argument("repo", nargs="?", default=".", help="Path to the repository root.")
    parser.add_argument("--json", action="store_true", help="Emit JSON instead of Markdown.")
    parser.add_argument("--max-context-lines", type=int, default=120, help="Line budget before a context file is flagged as bloated.")
    args = parser.parse_args()

    root = Path(args.repo).resolve()
    if not root.exists():
        print(f"Repository path does not exist: {root}", file=sys.stderr)
        return 1

    report = build_report(root, max_context_lines=args.max_context_lines)
    if args.json:
        print(json.dumps(report, ensure_ascii=False, indent=2))
    else:
        print_markdown(report)

    return 0


if __name__ == "__main__":
    raise SystemExit(main())