diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 78df6c1a..1aa7b227 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -6,7 +6,10 @@ on: - main jobs: + lint: + uses: ./.github/workflows/lint.yaml docker: + needs: lint runs-on: ubuntu-latest permissions: packages: write diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..144ef918 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,28 @@ +name: lint + +on: + push: + branches: + - main + pull_request: + workflow_call: + +jobs: + script-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/restore@v4 + with: + key: script-lint-${{ github.ref_name }}- + restore-keys: | + script-lint-main- + path: ~/.cache/ystack + - name: Script lint + run: bin/y-script-lint --fail=degrade bin/ + env: + Y_SCRIPT_LINT_BRANCH: ${{ github.ref_name }} + - uses: actions/cache/save@v4 + with: + key: script-lint-${{ github.ref_name }}-${{ github.run_id }} + path: ~/.cache/ystack diff --git a/Y_SCRIPT_AUTHORING.md b/Y_SCRIPT_AUTHORING.md new file mode 100644 index 00000000..d6a3612a --- /dev/null +++ b/Y_SCRIPT_AUTHORING.md @@ -0,0 +1,384 @@ +# Y-Script Authoring Guide + +Scripts in `bin/` follow the `y-` prefix convention for PATH discoverability via tab completion. +This guide covers conventions for writing new scripts and improving existing ones, +with emphasis on making scripts useful to both humans and AI agents. + +The y-* convention spans multiple repositories. +Each repo's `bin/` is added to PATH, and scripts can call each other across repos. + +## OS Compatibility + +Scripts must work on: +- macOS 14.8+ (Sonoma) — BSD userland, no GNU coreutils +- Debian Trixie+ (13) +- Ubuntu 24.04+ (Noble) + +macOS ships BSD versions of core utilities. Key incompatibilities: +- `sed -i ''` on macOS vs `sed -i` on Linux — avoid `sed -i` entirely, use a temp file or `ed` +- `stat -f %m` on macOS vs `stat -c %Y` on Linux +- `date -Iseconds` (GNU ISO 8601) unavailable on macOS, use `date -u +%Y-%m-%dT%H:%M:%SZ` +- `date -d` (GNU parse) vs `date -v` (BSD adjust) +- `readlink -f` unavailable on macOS without coreutils +- `grep -P` (PCRE) unavailable on macOS, use `grep -E` (extended regex) + +When a portable solution is impractical, use platform detection: + +```bash +case "$(uname -s)" in + Darwin) ;; # macOS-specific + Linux) ;; # Linux-specific +esac +``` + +## Quick Reference: Making a Compliant Script + +`y-script-lint` validates scripts statically (no execution). To pass all checks: + +### Bash + +```bash +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +YHELP='y-example - Bump image tags in kustomization files + +Usage: y-example IMAGE TAG PATH [--dry-run] + +Arguments: + IMAGE Container image name (e.g. yolean/node-kafka) + TAG Image tag (e.g. a git commit sha) + PATH File or directory to update + +Options: + --dry-run Show what would change without writing + +Environment: + REGISTRY Override default registry (default: docker.io) + +Dependencies: + +Exit codes: + 0 Success + 1 Missing or invalid arguments + 2 Image not found in registry +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +# Validate +[ -z "$1" ] && echo "First arg must be an image like yolean/node-kafka" >&2 && exit 1 + +# Main logic here +``` + +### Node.js + +```javascript +#!/usr/bin/env node + +const YHELP = `y-example - One line description + +Usage: y-example [options] + +Options: + --output json Output as JSON (default: human-readable) + +Environment: + MY_VAR Description (default: value) + +Dependencies: +`; + +if (process.argv[2] === 'help' || process.argv[2] === '--help') { + console.log(YHELP.trim()); + process.exit(0); +} + +// Main logic here +``` + +### TypeScript + +```typescript +#!/usr/bin/env -S node --experimental-strip-types + +const YHELP = `y-example - One line description +...same structure as Node.js... +`; + +if (process.argv[2] === 'help' || process.argv[2] === '--help') { + console.log(YHELP.trim()); + process.exit(0); +} +``` + +### What y-script-lint checks (all static, never executes your script) + +| Check | FAIL or WARN | Shell | Node.js | How to pass | +|-------|-------------|-------|---------|-------------| +| Shebang | FAIL | `#!/usr/bin/env bash` or `#!/bin/sh` | `#!/usr/bin/env node` or `*-strip-types` | First line | +| `set -eo pipefail` | FAIL | Required | n/a | Second or third line | +| DEBUG pattern | WARN | `[ -z "$DEBUG" ] \|\| set -x` | n/a | Second line | +| Help handler | WARN | `"$1" = "help"` in case/if | `process.argv` includes `help` | See templates above | +| No `npx` | FAIL | Not in non-comment lines | Not in non-comment lines | Use project deps | +| No `eval` | FAIL | Not in non-comment lines | No `eval(` calls | Avoid eval | +| shellcheck | FAIL | `--severity=error` | n/a | Fix shellcheck errors | + +### The help text format + +``` +y-name - One sentence summary + +Usage: y-name [options] ARGS + +Arguments: + ... + +Options: + ... + +Environment: + ... + +Dependencies: + +Exit codes: + 0 Success + 1 Usage error +``` + +Rules: +- First line: `y-name - description` (the "index line", used by tooling for discovery) +- `Dependencies:` section is maintained by tooling — leave it empty in new scripts +- Print help to stdout, exit 0 +- Keep it factual and compact — no ASCII art, no long examples + +### The `help` subcommand pattern + +New scripts use `help` as a positional subcommand (first argument): + +```bash +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; # backwards compat +esac +``` + +```javascript +if (process.argv[2] === 'help' || process.argv[2] === '--help') { +``` + +This is preferred over `--help`-only because: +- It reads naturally: `y-crane help`, `y-cluster-provision-k3d help` +- It follows the subcommand pattern used by git, docker, kubectl +- The help check happens before argument parsing, before prerequisite checks, + before any side effects +- `--help` is accepted for backwards compatibility but not documented in new scripts + +Existing scripts use various patterns (`--help`, `-h|--help`, `[[ "$1" =~ help$ ]]`). +`y-script-lint` detects all of these. When touching an existing script, migrate to the +`help` subcommand pattern. + +### The YHELP variable pattern + +Store help text in a variable named `YHELP` (bash) or `YHELP` (JS/TS const): + +```bash +YHELP='y-name - description +... +' +``` + +```javascript +const YHELP = `y-name - description +... +`; +``` + +This enables `y-script-lint` to extract the summary and dependencies by reading +the source file — no execution required. The variable name `YHELP` is the convention +that tooling looks for. + +## General Conventions + +### Header + +Every shell script must start with a shebang and standard preamble: + +```bash +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail +``` + +Use `#!/bin/sh` only for POSIX-portable scripts with no bashisms. + +Node.js scripts use `#!/usr/bin/env node`. +TypeScript scripts use `#!/usr/bin/env -S node --experimental-strip-types` (Node 22.6+). + +### Structured Output for Agents + +When a script's output will be consumed by agents, prefer machine-readable formats: + +```bash +# For listing/query commands, support --output json +if [ "$OUTPUT_FORMAT" = "json" ]; then + echo "{\"image\":\"$IMAGE\",\"digest\":\"$DIGEST\",\"bumped\":$BUMPED_COUNT}" +else + echo "Got $DIGEST for $IMAGE_URL" +fi +``` + +- Token efficiency: Keep output minimal. Avoid banners, progress bars, repeated info. +- Dry run: Mutating commands should support `--dry-run`. +- Deterministic output: Same inputs produce same output format. +- Error output to stderr: `echo "ERROR: message" >&2`. Structured results go to stdout. + +### Naming + +- Prefix: `y-` for all scripts (enforced by `y-check-bin-executables-are-named-ydash` in checkit) +- Hierarchical naming: `y-{domain}-{action}` (e.g. `y-cluster-provision-k3d`, `y-image-bump`) +- Wrapper scripts for binaries: `y-{toolname}` (e.g. `y-crane`, `y-kubectl`) +- Keep names descriptive enough that tab-completing `y-cluster-` reveals related commands + +### Argument Parsing + +Use `--flag=value` style for named parameters. Handle help first, then flags: + +```bash +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +while [ $# -gt 0 ]; do + case "$1" in + --context=*) CTX="${1#*=}"; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --) shift; break ;; + -*) echo "Unknown flag: $1" >&2; exit 1 ;; + *) break ;; + esac +done +``` + +Validate required arguments early with clear error messages: + +```bash +[ -z "$1" ] && echo "First arg must be an image like yolean/node-kafka" >&2 && exit 1 +``` + +### Exit Codes + +| Range | Meaning | Example | +|-------|---------|---------| +| 0 | Success | | +| 1 | Usage error / missing args | Missing required IMAGE arg | +| 2-9 | Domain-specific operational errors | Image not found, registry unreachable | +| 10-19 | Precondition failures | Wrong tool version, missing prerequisite | +| 90-99 | Convention/policy violations | Used by checkit for monorepo policy checks | + +### Error Handling + +- Use `set -eo pipefail` always +- Send error messages to stderr: `echo "ERROR: message" >&2` +- Use meaningful exit codes (document non-zero codes in help) +- Use `trap cleanup EXIT` when temporary files or state changes need reversal +- Validate prerequisites early: `command -v y-crane >/dev/null || { echo "ERROR: y-crane not found" >&2; exit 1; }` + +### Environment Variables + +- Document all env vars in help text +- Provide sensible defaults: `[ -z "$REGISTRY" ] && REGISTRY="docker.io"` +- Use `DEBUG` for `set -x` tracing (convention: `[ -z "$DEBUG" ] || set -x`) +- Never require secrets as positional args; use env vars or files + +## Shell Practices + +- Quote variables: `"$VAR"` not `$VAR` (shellcheck will catch this) +- Use `$(command)` not backticks +- Use `[[ ]]` for complex conditionals, `[ ]` for simple tests +- Declare function-local variables with `local` +- Use `mktemp` for temporary files, clean up with `trap` +- Prefer `printf` over `echo` for portable formatting + +### Shellcheck + +All shell scripts in `bin/` are linted by `y-script-lint` using `y-shellcheck`. +Current minimum severity: `error`. +Use inline directives sparingly and with justification: + +```bash +# shellcheck disable=SC2086 # intentional word splitting for BUILDCTL_OPTS +``` + +## Node.js Practices + +- Use project dependencies only (never `npx`) +- Exit with appropriate codes: `process.exit(1)` for errors +- Write errors to stderr: `console.error("ERROR: ...")` +- For JSON output, use `JSON.stringify(result, null, 2)` for human mode, compact for `--output json` + +## Cross-Repo Dependencies + +### The $YBIN Pattern (checkit) + +Checkit scripts use `YBIN="$(dirname $0)"` and call siblings via `$YBIN/y-other-script`. +This makes dependencies traceable by static analysis (grep for `$YBIN/y-`) +but limits calls to within the same repo's bin/. + +### PATH-based calls (ystack, bots) + +Ystack and bots scripts call y-* commands via PATH, allowing cross-repo invocation. +This is more flexible but makes dependency tracing harder — the indexer +must scan all repos' bin/ directories to resolve a dependency. + +Prefer PATH-based calls for cross-repo dependencies (e.g. checkit calling `y-crane` from ystack). +Use `$YBIN/` only when you specifically need to call a sibling script +and want to avoid PATH ambiguity. + +## Discoverability and Indexing + +### How y-script-lint discovers help text + +`y-script-lint` reads script source files and extracts information statically: + +1. Help handler detection: Greps for known patterns (`"$1" = "help"`, `--help` in case, + `process.argv.includes`). Detects existing scripts regardless of which pattern they use. +2. Summary and dependencies (planned): Parse the `YHELP` variable from source to extract + the first line (index line) and `Dependencies:` section without executing the script. + +Scripts are never executed during lint. This means: +- Lint is fast (~0.2s per script) +- No side effects, no network requests, no prerequisite checks +- Works on any OS without sandbox concerns +- Works on scripts that require specific env (KUBECONFIG, docker, etc.) + +### Dependency Graph Discovery + +Two complementary approaches: + +1. Declared dependencies (preferred): +Parse the `Dependencies:` section from the `YHELP` variable in source. + +2. Static analysis (works on existing scripts without changes): +Grep script source for `y-*` invocations. For checkit-style `$YBIN/y-*` calls +this is straightforward. For PATH-based calls, look for `y-[a-z]` patterns +that aren't inside comments or strings. + +### Agent-Oriented Conventions + +When agents use y-* scripts, token cost and parse reliability matter: + +1. Concise help: Keep help text factual and compact. +2. Structured errors: Prefix errors with `ERROR:` so agents can regex-match failures. +3. JSON mode: For commands that list or query, support `--output json`. +4. Idempotency: Where possible, make scripts safe to re-run. Document when a script is NOT idempotent. +5. Predictable exit codes: 0 = success, 1 = usage error, 2+ = domain-specific. +6. No interactive prompts: Never prompt for input. Use flags, env vars, or fail with a clear message. +7. Minimal side effects: Scripts should do one thing. diff --git a/bin/y-help b/bin/y-help new file mode 100755 index 00000000..9b864502 --- /dev/null +++ b/bin/y-help @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +YHELP='y-help - List y-* scripts with their help summaries + +Usage: y-help [help] [FILTER] + +Arguments: + FILTER Only show scripts matching this substring + +Reads ~/.cache/ystack/y-script-lint.json index. +Run y-script-lint first to generate or update the index. + +Dependencies: +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +FILTER="${1:-}" + +command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required" >&2; exit 1; } + +INDEX="${HOME}/.cache/ystack/y-script-lint.json" + +if [ ! -f "$INDEX" ]; then + echo "No index found at $INDEX. Run y-script-lint first." >&2 + exit 1 +fi + +jq -r --arg filter "$FILTER" ' + .scripts | to_entries[] + | select(.key | contains($filter)) + | select(.value.skipped != true) + | .key as $name + | .value.help_line as $help + | .value.lint_ok as $ok + | [$name, ($help // "no help section"), (if $ok then "" else "NOLINT" end)] + | @tsv +' "$INDEX" | while IFS=$'\t' read -r name help nolint; do + if [ -n "$nolint" ]; then + printf " %-40s %s %s\n" "$name" "$help" "$nolint" + else + printf " %-40s %s\n" "$name" "$help" + fi +done diff --git a/bin/y-script-lint b/bin/y-script-lint new file mode 100755 index 00000000..67b58644 --- /dev/null +++ b/bin/y-script-lint @@ -0,0 +1,528 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +trap 'trap - INT; kill -INT -$$' INT + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +AUTHORING_GUIDE="$(cd "$SCRIPT_DIR/.." && pwd)/Y_SCRIPT_AUTHORING.md" +AUTHORING_GUIDE_DISPLAY="~/${AUTHORING_GUIDE#"$HOME/"}" + +YHELP='y-script-lint - Static analysis and convention checks for y-* scripts + +Usage: y-script-lint [help] [options] [DIR...] + +Arguments: + DIR Directory to scan (default: all PATH dirs containing y-* scripts) + +Options: + --check Alias for --fail=all + --fail=all Exit non-zero if any script fails + --fail=degrade Exit non-zero only if a script degraded vs baseline + --dependencies-add Detect y-* invocations and add to Dependencies section in YHELP + +Degradation baselines (in ~/.cache/ystack/): + y-script-lint.main.json Saved on main branch builds + y-script-lint.branch.json Saved on branch builds + Branch baseline preferred, falls back to main. + First build with no baseline skips comparison. + On main: result always saves as main baseline. + On branch: result saves as branch baseline on success. + +All checks are static (source parsing only, no script execution). + +Environment: + Y_SCRIPT_LINT_BRANCH Branch name (default: git current branch, "main" = main baseline) + +Dependencies: + y-bin-download + y-shellcheck + +Exit codes: + 0 All scripts passed (or no degradation in degrade mode) + 1 Usage error + 2 Lint failures detected +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +FAIL_MODE="" +DEPS_ADD=false +JSON_OUTPUT=true +SINGLE_FILE="" +TARGET_DIRS=() + +while [ $# -gt 0 ]; do + case "$1" in + --check) FAIL_MODE=all; shift ;; + --fail=all) FAIL_MODE=all; shift ;; + --fail=degrade) FAIL_MODE=degrade; shift ;; + --fail=*) echo "ERROR: unknown fail mode: ${1#*=} (use: all, degrade)" >&2; exit 1 ;; + --dependencies-add) DEPS_ADD=true; shift ;; + --) shift; break ;; + -*) echo "Unknown flag: $1" >&2; exit 1 ;; + *) TARGET_DIRS+=("$1"); shift ;; + esac +done + +# Detect single-script mode: one arg that is a file or a bare script name +if [ ${#TARGET_DIRS[@]} -eq 1 ]; then + arg="${TARGET_DIRS[0]}" + if [ -f "$arg" ]; then + # Explicit path to a file + SINGLE_FILE="$(cd "$(dirname "$arg")" && pwd)/$(basename "$arg")" + elif [ -d "$arg" ]; then + : # directory, handled below + elif [[ "$arg" != */* ]]; then + # Bare name like "y-turbo" — search PATH + found="" + IFS=: read -ra path_entries <<< "$PATH" + for dir in "${path_entries[@]}"; do + if [ -x "$dir/$arg" ] && [ -f "$dir/$arg" ]; then + found="$dir/$arg" + break + fi + done + [ -z "$found" ] && { echo "ERROR: $arg not found in PATH" >&2; exit 1; } + SINGLE_FILE="$found" + else + # Path with / but not a file + echo "ERROR: file not found: $arg" >&2; exit 1 + fi +fi + +if [ -n "$SINGLE_FILE" ]; then + JSON_OUTPUT=false + RESOLVED_DIRS=() +else + if [ ${#TARGET_DIRS[@]} -eq 0 ]; then + IFS=: read -ra path_entries <<< "$PATH" + for dir in "${path_entries[@]}"; do + [ -d "$dir" ] || continue + for f in "$dir"/y-*; do + if [ -x "$f" ] && [ -f "$f" ]; then + TARGET_DIRS+=("$dir") + break + fi + done + done + [ ${#TARGET_DIRS[@]} -eq 0 ] && { echo "ERROR: no y-* scripts found in PATH" >&2; exit 1; } + fi + + RESOLVED_DIRS=() + for dir in "${TARGET_DIRS[@]}"; do + [ -d "$dir" ] || { echo "ERROR: directory not found: $dir" >&2; exit 1; } + RESOLVED_DIRS+=("$(cd "$dir" && pwd)") + done +fi + +# --- Language detection --- + +detect_language() { + local shebang + shebang=$(head -1 "$1" 2>/dev/null) || true + case "$shebang" in + '#!/usr/bin/env bash'|'#!/bin/bash') echo "bash" ;; + '#!/bin/sh') echo "sh" ;; + *node*experimental-strip-types*) echo "typescript" ;; + *node*) echo "node" ;; + *python*) echo "python" ;; + *y-bin-download*) echo "config" ;; + *) case "$1" in *.js|*.ts) echo "library" ;; *) echo "unknown" ;; esac ;; + esac +} + +is_shell() { [ "$1" = "bash" ] || [ "$1" = "sh" ]; } +is_node() { [ "$1" = "node" ] || [ "$1" = "typescript" ]; } + +# --- Static checks --- + +check_header_pipefail() { grep -qE '^set -e(o pipefail)?$' "$1" 2>/dev/null; } +check_header_debug() { grep -qE '^\[ -z "\$DEBUG" \] \|\| set -x' "$1" 2>/dev/null; } + +check_help_handler() { + local file="$1" lang="$2" + if is_shell "$lang"; then + grep -qE '(\-h\|--help|"\$1" = "help"|"\$1" = "--help"|\$1 =~ help|^\s*help\))' "$file" 2>/dev/null + elif is_node "$lang"; then + grep -qE "process\.argv\.includes\(['\"](-h|--help)['\"]" "$file" 2>/dev/null || + grep -qE "process\.argv\[2\] === ['\"]help['\"]" "$file" 2>/dev/null + else + return 1 + fi +} + +check_no_npx() { + local file="$1" lang="$2" + if is_node "$lang"; then + ! grep -vE '^\s*//' "$file" 2>/dev/null | grep -qE '\bnpx\b' + else + ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_npx|"uses npx"|No npx)' | grep -qE '\bnpx\b' + fi +} + +check_no_eval() { + local file="$1" lang="$2" + if is_node "$lang"; then + ! grep -vE '^\s*//' "$file" 2>/dev/null | grep -qE '\beval\s*\(' + else + ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_eval|"uses eval"|No.*eval)' | grep -qE '\beval\b' + fi +} + +# Detect trivial binary wrapper: header + YBIN + y-bin-download + versioned exec, nothing else +is_bin_wrapper() { + local file="$1" + # Must be sh, exactly 8 lines, contain y-bin-download and "$@" + [ "$(wc -l < "$file")" -le 8 ] || return 1 + grep -qE '^version=\$\(y-bin-download ' "$file" 2>/dev/null || return 1 + grep -qE '"\$@"' "$file" 2>/dev/null || return 1 + # Must not have anything beyond the wrapper pattern (no conditionals, loops, functions) + ! grep -qE '^(if |for |while |case |function |\w+\(\))' "$file" 2>/dev/null +} + +HAS_SHELLCHECK=false +command -v y-shellcheck >/dev/null 2>&1 && HAS_SHELLCHECK=true + +run_shellcheck() { + [ "$HAS_SHELLCHECK" = "true" ] && y-shellcheck --severity=error "$1" >/dev/null 2>&1 +} + +# --- YHELP source parsing --- + +parse_yhelp_from_source() { + local file="$1" lang="$2" + if is_shell "$lang"; then + awk " + /^YHELP='/ { + sub(/^YHELP='/, \"\") + if (/'$/) { sub(/'$/, \"\"); print; next } + print + while (getline > 0) { + if (/'$/) { sub(/'$/, \"\"); print; exit } + print + } + } + " "$file" + elif is_node "$lang"; then + awk ' + /const YHELP = `/ { + sub(/.*const YHELP = `/, "") + if (/`;/) { sub(/`;.*/, ""); print; next } + if (/`/) { sub(/`.*/, ""); print; exit } + print + while (getline > 0) { + if (/`;/) { sub(/`;.*/, ""); print; exit } + if (/`/) { sub(/`.*/, ""); print; exit } + print + } + } + ' "$file" + fi +} + +parse_summary_from_yhelp() { echo "$1" | grep -m1 -vE '^\s*$'; } +parse_deps_from_yhelp() { echo "$1" | awk '/^Dependencies:/{f=1;next} f&&/^$/{exit} f&&/^[A-Z]/{exit} f{print}' | awk 'NF{print $1}' || true; } +yhelp_has_deps_section() { echo "$1" | grep -q '^Dependencies:' 2>/dev/null; } + +# --- Static dependency detection --- + +detect_y_invocations() { + local file="$1" lang="$2" self + self=$(basename "$file") + local content + if is_shell "$lang"; then + content=$(grep -vE '^\s*#' "$file" 2>/dev/null || true) + elif is_node "$lang"; then + content=$(grep -vE '^\s*//' "$file" 2>/dev/null || true) + else + content=$(cat "$file" 2>/dev/null || true) + fi + echo "$content" | grep -oE '\by-[a-z][a-z0-9_-]*' | grep -v "^${self}$" | sort -u || true +} + +# --- --dependencies-add --- + +add_deps_to_file() { + local file="$1" lang="$2" detected_deps="$3" + [ -z "$detected_deps" ] && return + + local yhelp + yhelp=$(parse_yhelp_from_source "$file" "$lang") + [ -z "$yhelp" ] && return + yhelp_has_deps_section "$yhelp" || return + + local existing_deps + existing_deps=$(parse_deps_from_yhelp "$yhelp") + + local new_deps=() + while IFS= read -r dep; do + [ -z "$dep" ] && continue + echo "$existing_deps" | grep -qxF "$dep" || new_deps+=("$dep") + done <<< "$detected_deps" + [ ${#new_deps[@]} -eq 0 ] && return + + # Find the line number of Dependencies: in the file, insert after it + local deps_line + deps_line=$(grep -n '^Dependencies:' "$file" | head -1 | cut -d: -f1) + [ -z "$deps_line" ] && return + + local tmpfile insert_file + tmpfile=$(mktemp) + insert_file=$(mktemp) + for dep in "${new_deps[@]}"; do echo " ${dep}" >> "$insert_file"; done + + head -n "$deps_line" "$file" > "$tmpfile" + cat "$insert_file" >> "$tmpfile" + tail -n +"$((deps_line + 1))" "$file" >> "$tmpfile" + + if ! cmp -s "$file" "$tmpfile"; then + cp "$tmpfile" "$file" + for dep in "${new_deps[@]}"; do echo " + $dep"; done + fi + rm -f "$tmpfile" "$insert_file" +} + +# --- Per-file lint --- + +lint_file() { + local file="$1" name + name=$(basename "$file") + lang=$(detect_language "$file") + + if [ "$lang" = "config" ] || [ "$lang" = "library" ]; then + SKIPPED=$((SKIPPED + 1)) + [ "$JSON_OUTPUT" = "true" ] && DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ + --arg name "$name" --arg lang "$lang" --arg parent "$CURRENT_PARENT" \ + --arg reporoot "$CURRENT_REPOROOT" \ + '{($name): {parent: $parent, reporoot: (if $reporoot == "" then null else $reporoot end), language: $lang, skipped: true}}')," + return + fi + + local loc errors=() + loc=$(wc -l < "$file") + local checks_shebang=true checks_header=true checks_debug=true + local checks_help_handler=false checks_shellcheck=null + local checks_no_npx=true checks_no_eval=true + local checks_help_format=null checks_deps_declared=null + local summary=null help_line=null deps_json="[]" detected_deps="" + + [ "$lang" = "unknown" ] && { checks_shebang=false; errors+=("unrecognized or missing shebang"); } + + if is_shell "$lang"; then + check_header_pipefail "$file" || { checks_header=false; errors+=("missing set -eo pipefail or set -e"); } + check_header_debug "$file" || { checks_debug=false; errors+=("missing DEBUG pattern: [ -z \"\\\$DEBUG\" ] || set -x"); } + else + checks_header=null; checks_debug=null + fi + + local bin_wrapper=false + is_shell "$lang" && is_bin_wrapper "$file" && bin_wrapper=true + + if [ "$bin_wrapper" = "true" ]; then + checks_help_handler=true + # Extract wrapped binary name for help_line + local wrapped_bin + wrapped_bin=$(grep 'y-bin-download' "$file" 2>/dev/null | sed 's/.*y-bin-download [^ ]* //;s/[)].*//') + [ -n "$wrapped_bin" ] && help_line="$wrapped_bin binary wrapper" + elif check_help_handler "$file" "$lang"; then + checks_help_handler=true + else + errors+=("no help handler found in source") + fi + check_no_npx "$file" "$lang" || { checks_no_npx=false; errors+=("uses npx"); } + check_no_eval "$file" "$lang" || { checks_no_eval=false; errors+=("uses eval"); } + + if is_shell "$lang" && [ "$HAS_SHELLCHECK" = "true" ]; then + run_shellcheck "$file" && checks_shellcheck=true || { checks_shellcheck=false; errors+=("shellcheck --severity=error failed"); } + fi + + # Parse YHELP from source + local yhelp + yhelp=$(parse_yhelp_from_source "$file" "$lang") + if [ -n "$yhelp" ]; then + local first_line + first_line=$(parse_summary_from_yhelp "$yhelp") + if echo "$first_line" | grep -qE "^${name} - .+"; then + checks_help_format=true; summary="$first_line" + help_line="${first_line#"${name} - "}" + else + checks_help_format=false; errors+=("YHELP first line does not match: $name - description") + fi + if yhelp_has_deps_section "$yhelp"; then + checks_deps_declared=true + local declared_deps + declared_deps=$(parse_deps_from_yhelp "$yhelp") + [ -n "$declared_deps" ] && deps_json=$(echo "$declared_deps" | jq -R . | jq -s .) + else + checks_deps_declared=false; errors+=("YHELP missing Dependencies: section") + fi + fi + + detected_deps=$(detect_y_invocations "$file" "$lang") + [ "$DEPS_ADD" = "true" ] && [ -n "$detected_deps" ] && add_deps_to_file "$file" "$lang" "$detected_deps" + + local detected_deps_json="[]" + [ -n "$detected_deps" ] && detected_deps_json=$(echo "$detected_deps" | jq -R . | jq -s .) + + # Pass/fail + local has_static_failure=false has_help_failure=false + { [ "$checks_shebang" = "false" ] || [ "$checks_header" = "false" ] \ + || [ "$checks_no_npx" = "false" ] || [ "$checks_no_eval" = "false" ] \ + || [ "$checks_shellcheck" = "false" ]; } && has_static_failure=true + { [ "$checks_help_handler" = "false" ] || [ "$checks_help_format" = "false" ]; } && has_help_failure=true + + if [ "$has_static_failure" = "true" ]; then + FAILED_STATIC=$((FAILED_STATIC + 1)); FAIL_LIST="$FAIL_LIST $name" + elif [ "$has_help_failure" = "true" ]; then + FAILED_HELP=$((FAILED_HELP + 1)) + else + PASSED=$((PASSED + 1)) + fi + + local type_label="$lang" + [ "$bin_wrapper" = "true" ] && type_label="$lang wrapper" + echo " $name $type_label ${loc}L" + for err in "${errors[@]}"; do + local level="WARN" + case "$err" in + "unrecognized or missing shebang"|"missing set -eo pipefail"*|"uses npx"|"uses eval"|"shellcheck "*) level="FAIL" ;; + esac + echo " $level $err" + done + + if [ "$JSON_OUTPUT" = "true" ]; then + local errors_json="[]" summary_json="null" help_line_json="null" + [ ${#errors[@]} -gt 0 ] && errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) + [ "$summary" != "null" ] && [ -n "$summary" ] && summary_json=$(echo "$summary" | jq -R .) + [ "$help_line" != "null" ] && [ -n "$help_line" ] && help_line_json=$(echo "$help_line" | jq -R .) + local checks_json + checks_json=$(jq -n \ + --argjson shebang "$checks_shebang" --argjson header "$checks_header" \ + --argjson debug "$checks_debug" --argjson help_handler "$checks_help_handler" \ + --argjson shellcheck "$checks_shellcheck" --argjson no_npx "$checks_no_npx" \ + --argjson no_eval "$checks_no_eval" --argjson help_format "$checks_help_format" \ + --argjson deps_declared "$checks_deps_declared" \ + '{shebang:$shebang,header:$header,debug:$debug,help_handler:$help_handler, + shellcheck:$shellcheck,no_npx:$no_npx,no_eval:$no_eval, + help_format:$help_format,deps_declared:$deps_declared}') + DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ + --arg name "$name" --arg lang "$lang" --arg parent "$CURRENT_PARENT" \ + --arg reporoot "$CURRENT_REPOROOT" \ + --argjson summary "$summary_json" --argjson help_line "$help_line_json" \ + --argjson deps "$deps_json" --argjson detected "$detected_deps_json" \ + --argjson checks "$checks_json" --argjson errors "$errors_json" \ + --argjson lint_ok "$([ "$has_static_failure" = "false" ] && [ "$has_help_failure" = "false" ] && echo true || echo false)" \ + '{($name):{parent:$parent,reporoot:(if $reporoot == "" then null else $reporoot end), + language:$lang,summary:$summary,help_line:$help_line,lint_ok:$lint_ok, + dependencies:$deps,detected_dependencies:$detected,checks:$checks,errors:$errors}}')," + fi +} + +# --- Main --- + +TOTAL=0 PASSED=0 FAILED_STATIC=0 FAILED_HELP=0 SKIPPED=0 +FAIL_LIST="" + +# Single-file mode: lint one script, exit with its result +if [ -n "$SINGLE_FILE" ]; then + TOTAL=1 + lint_file "$SINGLE_FILE" + if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then + echo "See $AUTHORING_GUIDE_DISPLAY" + [ "$FAILED_STATIC" -gt 0 ] && exit 2 + exit 1 + fi + exit 0 +fi + +SEEN_SCRIPTS="" +ALL_JSON_SCRIPTS="" + +for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do + dir_scripts=() + for file in "$TARGET_DIR"/y-*; do + [ -f "$file" ] && [ -x "$file" ] || continue + name=$(basename "$file") + case "$name" in *.yaml|*.yml|*.md|*.spec.js|*.spec.ts|*.test.js|*.test.ts|*-bin|*-dist) continue ;; esac + case "$SEEN_SCRIPTS" in *"|$name|"*) continue ;; esac + SEEN_SCRIPTS="${SEEN_SCRIPTS}|${name}|" + dir_scripts+=("$file") + done + [ ${#dir_scripts[@]} -eq 0 ] && continue + + echo "[y-script-lint] $TARGET_DIR (${#dir_scripts[@]} scripts)" + DIR_JSON_SCRIPTS="" + CURRENT_PARENT="$TARGET_DIR" + CURRENT_REPOROOT=$(git -C "$TARGET_DIR" rev-parse --show-toplevel 2>/dev/null || echo "") + for file in "${dir_scripts[@]}"; do + TOTAL=$((TOTAL + 1)) + lint_file "$file" + done + + ALL_JSON_SCRIPTS="${ALL_JSON_SCRIPTS}${DIR_JSON_SCRIPTS}" +done + +NOTES="" +[ "$HAS_SHELLCHECK" = "false" ] && NOTES="${NOTES}, no shellcheck" +echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" + +# Write index +CACHE_DIR="${HOME}/.cache/ystack" +mkdir -p "$CACHE_DIR" +INDEX_FILE="${CACHE_DIR}/y-script-lint.json" + +build_index() { + echo "[${ALL_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ + jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "3" \ + --argjson scripts "$(cat)" \ + '{generated:$generated,tool_version:$tool_version,scripts:$scripts}' +} + +if [ -n "$ALL_JSON_SCRIPTS" ] && command -v jq >/dev/null 2>&1; then + build_index > "$INDEX_FILE" + echo "[y-script-lint] Wrote $INDEX_FILE" +fi + +if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then + echo "See $AUTHORING_GUIDE_DISPLAY" +fi + +if [ "$FAIL_MODE" = "all" ] && [ "$FAILED_STATIC" -gt 0 ]; then + echo "FAILED:$FAIL_LIST" + exit 2 +fi + +if [ "$FAIL_MODE" = "degrade" ] && [ -f "$INDEX_FILE" ]; then + BRANCH="${Y_SCRIPT_LINT_BRANCH:-$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "")}" + MAIN_BASELINE="${CACHE_DIR}/y-script-lint.main.json" + BRANCH_BASELINE="${CACHE_DIR}/y-script-lint.branch.json" + + # Select best baseline: branch cache > main cache > none + BASELINE="" + [ -f "$BRANCH_BASELINE" ] && BASELINE="$BRANCH_BASELINE" + [ -z "$BASELINE" ] && [ -f "$MAIN_BASELINE" ] && BASELINE="$MAIN_BASELINE" + + if [ -z "$BASELINE" ]; then + echo "[y-script-lint] No baseline found, skipping degradation check" + else + echo "[y-script-lint] Comparing against $(basename "$BASELINE")" + "$SCRIPT_DIR/y-script-lint-compare" "$BASELINE" "$INDEX_FILE" + compare_exit=$? + if [ $compare_exit -ne 0 ]; then + exit $compare_exit + fi + fi + + # Save baseline for future runs + if [ "$BRANCH" = "main" ] || [ "$BRANCH" = "master" ]; then + cp "$INDEX_FILE" "$MAIN_BASELINE" + echo "[y-script-lint] Saved main baseline" + else + cp "$INDEX_FILE" "$BRANCH_BASELINE" + echo "[y-script-lint] Saved branch baseline" + fi +fi diff --git a/bin/y-script-lint-compare b/bin/y-script-lint-compare new file mode 100755 index 00000000..58ed680b --- /dev/null +++ b/bin/y-script-lint-compare @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +YHELP='y-script-lint-compare - Compare two lint index files for degradation + +Usage: y-script-lint-compare [help] OLD_INDEX NEW_INDEX + +A check is degraded if it passed (true) in OLD and fails (false) in NEW. +A new script (present in NEW but not OLD) must pass static checks +(shebang, header, shellcheck, no_npx, no_eval). Warn-level checks +(help_handler, help_format, etc.) are not enforced for new scripts. +Scripts deleted between OLD and NEW are ignored. + +Exit codes: + 0 No degradation + 2 Degradation or new script failure detected + +Dependencies: +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +[ -z "$1" ] || [ -z "$2" ] && { echo "Usage: y-script-lint-compare OLD_INDEX NEW_INDEX" >&2; exit 1; } +[ -f "$1" ] || { echo "ERROR: old index not found: $1" >&2; exit 1; } +[ -f "$2" ] || { echo "ERROR: new index not found: $2" >&2; exit 1; } + +OLD="$1" +NEW="$2" + +# jq does all the work: compare checks between old and new +RESULT=$(jq -n \ + --slurpfile old "$OLD" \ + --slurpfile new "$NEW" ' + + # Checks where false is a static failure (FAIL level) + def fail_checks: ["shebang","header","shellcheck","no_npx","no_eval"]; + # All checks including warn-level + def all_checks: ["shebang","header","debug","help_handler","shellcheck","no_npx","no_eval","help_format","deps_declared"]; + + ($old[0].scripts // {}) as $old_scripts | + ($new[0].scripts // {}) as $new_scripts | + + [ + $new_scripts | to_entries[] | + .key as $name | .value as $new_val | + select($new_val.skipped != true) | + if ($old_scripts[$name] // null) != null and ($old_scripts[$name].skipped != true) then + # Existing script: any check that passed before must still pass + ($old_scripts[$name].checks // {}) as $old_checks | + ($new_val.checks // {}) as $new_checks | + all_checks[] | + select($old_checks[.] == true and $new_checks[.] == false) | + {script: $name, check: ., type: "degraded"} + else + # New script: only static failures count (not warn-level checks) + ($new_val.checks // {}) as $new_checks | + fail_checks[] | + select($new_checks[.] == false) | + {script: $name, check: ., type: "new_failure"} + end + ] +') + +COUNT=$(echo "$RESULT" | jq 'length') + +if [ "$COUNT" -eq 0 ]; then + echo "[y-script-lint-compare] No degradation detected" + exit 0 +fi + +echo "[y-script-lint-compare] $COUNT issue(s) found:" +echo "$RESULT" | jq -r '.[] | " \(.type): \(.script) check \(.check)"' +exit 2