From bcdf8a8d213e469059b250b662ac0c210cc61537 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Mar 2026 06:37:12 +0100 Subject: [PATCH 01/28] explores how to improve the discoverability and bot-ability of y-* --- Y_SCRIPT_AUTHORING.md | 468 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100644 Y_SCRIPT_AUTHORING.md diff --git a/Y_SCRIPT_AUTHORING.md b/Y_SCRIPT_AUTHORING.md new file mode 100644 index 00000000..04961565 --- /dev/null +++ b/Y_SCRIPT_AUTHORING.md @@ -0,0 +1,468 @@ +# Y-Script Authoring Guide + +Scripts in `bin/` follow the `y-` prefix convention for PATH discoverability via tab completion. +This guide covers conventions for writing new scripts and improving existing ones, +with emphasis on making scripts useful to both humans and AI agents. + +The y-* convention spans multiple repos (ystack ~90 scripts, checkit ~152 scripts, bots ~10 scripts). +Each repo's `bin/` is added to PATH, and scripts can call each other across repos. + +## General Conventions + +### Header + +Every script must start with a shebang and standard preamble: + +```bash +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail +``` + +Use `#!/bin/sh` only for POSIX-portable scripts with no bashisms. + +### Help Text (Required) + +Every script MUST support `--help` / `-h` and print a structured help block. +The **first line** of help output is used by index scripts for discovery, +so it must be a concise one-line summary of what the script does. + +**Migration note**: Existing scripts use inconsistent help triggers: +`[ "$1" = "help" ]`, `--help`, or regex `[[ "$1" =~ help$ ]]`. +New scripts MUST use `--help` / `-h`. When touching existing scripts, migrate to `--help`/`-h` +while keeping `help` (no dashes) as a fallback for backwards compatibility. + +``` +y-example - Bump image tags in kustomization files + +Usage: y-example IMAGE TAG PATH [--dry-run] + +Arguments: + IMAGE Container image name (e.g. yolean/node-kafka) + TAG Image tag (e.g. a git commit sha) + PATH File or directory to update + +Options: + --dry-run Show what would change without writing + --help Show this help + +Environment: + REGISTRY Override default registry (default: docker.io) + +Dependencies: + y-crane Used to resolve image digests + yq Used for kustomization.yaml updates + +Exit codes: + 0 Success + 1 Missing or invalid arguments + 2 Image not found in registry +``` + +Key rules: +- First line: `y-name - One sentence description` (this is the "index line") +- List all positional arguments with descriptions +- List all flags/options +- List environment variables that affect behavior +- List y-* dependencies so tooling can build a dependency graph +- List non-obvious exit codes +- Print help to stdout (not stderr) so it can be piped/parsed +- Exit 0 after printing help + +### Structured Output for Agents + +When a script's output will be consumed by agents, prefer machine-readable formats: + +```bash +# For listing/query commands, support --output json +if [ "$OUTPUT_FORMAT" = "json" ]; then + echo "{\"image\":\"$IMAGE\",\"digest\":\"$DIGEST\",\"bumped\":$BUMPED_COUNT}" +else + echo "Got $DIGEST for $IMAGE_URL" +fi +``` + +Guidelines from [Rewrite your CLI for AI Agents](https://justin.poehnelt.com/posts/rewrite-your-cli-for-ai-agents/): + +- **Token efficiency**: Keep output minimal. Agents pay per token of context consumed. Avoid banners, decorative output, progress bars, or repeated information. +- **Field selection**: For commands that return rich data, consider `--fields` to let callers request only what they need. +- **Dry run**: Mutating commands should support `--dry-run` to let agents validate before acting. +- **Deterministic output**: Same inputs should produce same output format. Don't mix human commentary into structured output. +- **Error output to stderr**: Human-readable messages go to stderr. Machine-parseable results go to stdout. This lets agents reliably capture output while still showing errors. + +### Naming + +- Prefix: `y-` for all scripts (enforced by `y-check-bin-executables-are-named-ydash` in checkit) +- Hierarchical naming: `y-{domain}-{action}` (e.g. `y-cluster-provision-k3d`, `y-image-bump`) +- Wrapper scripts for binaries: `y-{toolname}` (e.g. `y-crane`, `y-kubectl`) +- Keep names descriptive enough that tab-completing `y-cluster-` reveals related commands +- Consistent suffixes for CLI wrappers that wrap versioned binaries: `y-{tool}-v{version}-bin` + +### Argument Parsing + +Use `--flag=value` style for named parameters. Parse with a while loop: + +```bash +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) show_help; exit 0 ;; + --context=*) CTX="${1#*=}"; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --) shift; break ;; + -*) echo "Unknown flag: $1" >&2; exit 1 ;; + *) break ;; + esac +done +``` + +Validate required arguments early with clear error messages: + +```bash +[ -z "$1" ] && echo "First arg must be an image like yolean/node-kafka" >&2 && exit 1 +``` + +### Exit Codes + +Reserve ranges for consistent meaning across all y-* scripts: + +| Range | Meaning | Example | +|-------|---------|---------| +| 0 | Success | | +| 1 | Usage error / missing args | Missing required IMAGE arg | +| 2-9 | Domain-specific operational errors | Image not found, registry unreachable | +| 10-19 | Precondition failures | Wrong tool version, missing prerequisite | +| 90-99 | Convention/policy violations | Used by checkit for monorepo policy checks | + +Document non-zero exit codes in the `Exit codes:` section of `--help`. + +### Error Handling + +- Use `set -eo pipefail` always +- Send error messages to stderr: `echo "ERROR: message" >&2` +- Use meaningful exit codes (document non-zero codes in help) +- Use `trap cleanup EXIT` when temporary files or state changes need reversal +- Validate prerequisites early: `command -v y-crane >/dev/null || { echo "ERROR: y-crane not found" >&2; exit 1; }` + +### Environment Variables + +- Document all env vars in `--help` output +- Provide sensible defaults: `[ -z "$REGISTRY" ] && REGISTRY="docker.io"` +- Use `DEBUG` for `set -x` tracing (convention: `[ -z "$DEBUG" ] || set -x`) +- Use `DEBUGDEBUG` for extra verbose output in complex scripts +- Never require secrets as positional args; use env vars or files + +### Dependencies Section in Help + +To enable automated dependency discovery, include a `Dependencies:` section in help output listing every `y-*` command and external tool the script calls: + +``` +Dependencies: + y-crane Resolve image digests + y-buildctl Run buildkit builds + yq YAML processing + jq JSON processing + git Version control queries +``` + +This lets an indexer script parse `--help` output to build a dependency graph across all `y-*` scripts. + +## Bash-Specific Conventions + +### Script Template + +```bash +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +show_help() { + cat <<'EOF' +y-example - One line description + +Usage: y-example [options] ARG + +Arguments: + ARG Description of argument + +Options: + --dry-run Preview changes without applying + -h,--help Show this help + +Environment: + MY_VAR Description (default: value) + +Dependencies: + y-crane Used to resolve digests +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) show_help; exit 0 ;; + --dry-run) DRY_RUN=true; shift ;; + --) shift; break ;; + -*) echo "Unknown flag: $1" >&2; exit 1 ;; + *) break ;; + esac +done + +# Validate +[ -z "$1" ] && echo "ARG is required. See --help" >&2 && exit 1 + +# Main logic here +``` + +### Shell Practices + +- Quote variables: `"$VAR"` not `$VAR` (shellcheck will catch this) +- Use `$(command)` not backticks +- Use `[[ ]]` for complex conditionals, `[ ]` for simple tests +- Declare function-local variables with `local` +- Use `mktemp` for temporary files, clean up with `trap` +- Prefer `printf` over `echo` for portable formatting + +### Shellcheck + +All bash scripts in `bin/` are linted by `test.sh` using `y-shellcheck`. +Current minimum severity: `error`. Fix all shellcheck warnings at the `warning` level or above. +Use inline directives sparingly and with justification: + +```bash +# shellcheck disable=SC2086 # intentional word splitting for BUILDCTL_OPTS +``` + +## Node.js-Specific Conventions + +### Script Template + +Use TypeScript with `--experimental-strip-types` (Node 22.6+): + +```typescript +#!/usr/bin/env -S node --experimental-strip-types + +const HELP = `y-example - One line description + +Usage: y-example [options] + +Options: + --output json Output as JSON (default: human-readable) + -h, --help Show this help + +Environment: + MY_VAR Description (default: value) + +Dependencies: + y-crane Used to resolve digests (via shell) +`; + +if (process.argv.includes('--help') || process.argv.includes('-h')) { + console.log(HELP.trim()); + process.exit(0); +} + +// Main logic here +``` + +### Node.js Practices + +- Use project dependencies only (never `npx`) +- Import types from typed packages (e.g. `@octokit/graphql-schema`) +- Exit with appropriate codes: `process.exit(1)` for errors +- Write errors to stderr: `console.error("ERROR: ...")` +- For JSON output, use `JSON.stringify(result, null, 2)` for human mode, compact for `--output json` + +## Cross-Repo Dependencies + +### The $YBIN Pattern (checkit) + +Checkit scripts use `YBIN="$(dirname $0)"` and call siblings via `$YBIN/y-other-script`. +This makes dependencies traceable by static analysis (grep for `$YBIN/y-`) +but limits calls to within the same repo's bin/. + +### PATH-based calls (ystack, bots) + +Ystack and bots scripts call y-* commands via PATH, allowing cross-repo invocation. +This is more flexible but makes dependency tracing harder — the indexer +must scan all repos' bin/ directories to resolve a dependency. + +### Recommendation + +Prefer PATH-based calls for cross-repo dependencies (e.g. checkit calling `y-crane` from ystack). +Use `$YBIN/` only when you specifically need to call a sibling script +and want to avoid PATH ambiguity. + +### Agent Output Modes + +Checkit's `y-test-fast` already supports `Y_BUILD_OUTPUT_MODE=agents` for reduced output. +Standardize this pattern: scripts with verbose human output should check for an env var +that switches to minimal, machine-parseable output. + +## Discoverability and Indexing + +### How index.sh Works + +The `index.sh` pattern (piloted in bots) iterates over `bin/*`, calls `--help` on each script, and displays the first line. This relies on: + +1. Every script supporting `--help` +2. The first line of help being a meaningful summary +3. Help printing to stdout + +### Dependency Graph Discovery + +Two complementary approaches: + +**1. Declared dependencies (preferred for new scripts):** +Parse the `Dependencies:` section from each script's `--help` output. + +**2. Static analysis (works on existing scripts without changes):** +Grep script source for `y-*` invocations. For checkit-style `$YBIN/y-*` calls +this is straightforward. For PATH-based calls, look for `y-[a-z]` patterns +that aren't inside comments or strings. + +Both approaches enable: + +- Visualizing which scripts depend on which +- Detecting circular dependencies +- Understanding the blast radius of changes to foundational scripts like `y-crane` +- Helping agents understand which scripts to use together +- Identifying cross-repo dependency chains (e.g. checkit → ystack → binary) + +### Agent-Oriented Conventions + +When agents use y-* scripts, token cost and parse reliability matter: + +1. **Concise help**: Keep `--help` output factual and compact. No ASCII art, no examples longer than one line each. +2. **Structured errors**: Prefix errors with `ERROR:` so agents can regex-match failures. +3. **JSON mode**: For commands that list or query, support `--output json` to avoid parsing human-formatted tables. +4. **Idempotency**: Where possible, make scripts safe to re-run. Document when a script is NOT idempotent. +5. **Predictable exit codes**: 0 = success, 1 = usage error, 2+ = domain-specific. Document in help. +6. **No interactive prompts**: Never prompt for input. Use flags, env vars, or fail with a clear message. +7. **Minimal side effects**: Scripts should do one thing. If a script has modes that do very different things, consider splitting it. + +## Tooling for Style and Security Checks + +### Current: shellcheck via test.sh + +`test.sh` runs `y-shellcheck --severity=error` on all shell scripts in `bin/`. + +### Recommended Additions + +Place these in ystack as scripts or CI steps: + +#### y-script-lint (slow, runs in CI or on demand) + +Scans a repo's `bin/` directory, executes `--help` on each y-* script, +and validates convention alignment. Writes results to a dotfile +that `y-script-index` can read without re-executing anything. + +**Two-phase approach — static checks never execute the script:** + +Phase 1 (static, safe): Read the script source and check: +- Shebang present and recognized (`bash`, `sh`, `node`) +- Standard header (`set -eo pipefail`, DEBUG pattern) +- Contains a help handler (recognizable `--help` / `-h` in a case/if pattern) +- No use of `npx` +- No `eval` on user input + +Scripts that fail phase 1 are recorded with `"help": false` and **never executed**. + +Phase 2 (sandboxed execution): For scripts that pass static screening, +run `--help` in a sandbox to extract the summary and declared dependencies: + +```bash +env -i \ + PATH="$PATH" \ + HOME="$(mktemp -d)" \ + timeout 5 \ + /bin/bash -c 'cd "$(mktemp -d)" && exec "$@"' _ "$script" --help +``` + +Sandbox properties: +- **Stripped env**: `env -i` clears credentials, cloud auth, KUBECONFIG etc. + Only PATH is preserved (needed to resolve y-* dependencies in help code). +- **Temp HOME**: Prevents reading credentials, ssh keys, cloud config files. +- **Temp working dir**: Prevents writes to the repo. +- **Timeout 5s**: Kills scripts that hang or do real work on `--help`. +- **No network** (Linux bots): Wrap with `unshare --net` where available. + +**Checks from the sandboxed run:** +- Exits 0 and produces output to stdout +- First help line matches `y-name - description` format +- `Dependencies:` section exists in help output + +**Output:** Writes `.y-script-lint.json` in the repo's `bin/` directory: + +```json +{ + "generated": "2026-03-14T12:00:00Z", + "scripts": { + "y-crane": { + "summary": "Crane binary wrapper for container image operations", + "dependencies": ["y-bin-download"], + "checks": {"help": true, "header": true, "deps_declared": true} + }, + "y-build": { + "summary": null, + "dependencies": [], + "checks": {"help": false, "header": true, "deps_declared": false}, + "errors": ["no --help support", "no Dependencies section"] + } + } +} +``` + +Scripts that fail the `help` check are included with `"summary": null` +so the lint result doubles as a backfill TODO list. + +The dotfile should be gitignored — it's a local cache, not source of truth. + +#### y-script-index (fast, safe for agents) + +Reads `.y-script-lint.json` from one or more repo `bin/` directories. +**Does not execute any scripts.** Only lists scripts that passed lint checks. + +```bash +# Human-friendly table (default) +y-script-index + # Last lint: 2026-03-14T12:00:00Z (2 hours ago) + # 187/252 scripts passed, 65 need --help backfill + y-crane Crane binary wrapper for container image operations + y-cluster-provision-k3d Provision a k3d cluster with ystack defaults + ... + +# JSON for agents +y-script-index --output json + +# Dependency graph (dot format, pipe to graphviz) +y-script-index --deps +y-script-index --deps --output dot | dot -Tsvg -o deps.svg + +# Filter by domain +y-script-index --filter cluster + +# Show stale lint warning (e.g. if scripts changed since last lint) +y-script-index + # Last lint: 2026-03-12T09:00:00Z (2 days ago) + # WARNING: 3 scripts modified since last lint run. Run y-script-lint to update. +``` + +The header always shows when lint was last run and how many scripts passed. +If any script's mtime is newer than the dotfile, a warning is printed. +This lets developers and agents judge how trustworthy the index is. + +Because it only reads a dotfile, agents can call `y-script-index` freely +to discover available tools without risk of side effects or slow execution. + +**Multi-repo support:** By default, scans `bin/.y-script-lint.json` in known +repo roots (ystack, checkit, bots). Accepts `--path` to override. + +#### Security checks (extend test.sh or run separately) + +- `shellcheck --severity=warning` (raise the bar from current `error`) +- Check for unquoted variables in file paths +- Check for `eval` usage (flag for review) +- Check that secrets come from env vars or files, never positional args +- Verify `trap cleanup EXIT` when `mktemp` is used +- osv-scanner for Node.js dependency vulnerabilities (already in y-bin.optional.yaml) From d18210fb2dbb14111589198a16dcd2e75fffe8ea Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 06:02:07 +0000 Subject: [PATCH 02/28] implement phase 1 static analysis and phase 2 sandboxed help Two-phase convention checker for y-* scripts: - Phase 1: language detection, header/debug checks, help handler detection, shellcheck integration, npx/eval checks. Never executes scripts. - Phase 2: sandboxed --help execution (env -i, temp HOME, timeout 5s, unshare --net on Linux with fallback). Parses summary and Dependencies. - --check exits 2 on static failures (CI gate), WARN for help issues - --json writes .y-script-lint.json for future y-script-index - TODO: macOS sandbox, eslint integration for Node.js scripts First run on ystack: 73 scripts, 4 static failures (pre-existing eval/shellcheck), 68 help warnings (expected backfill work). Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/.gitignore | 1 + bin/y-script-lint | 514 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 515 insertions(+) create mode 100755 bin/y-script-lint diff --git a/bin/.gitignore b/bin/.gitignore index 496e7e22..a3c66a8b 100644 --- a/bin/.gitignore +++ b/bin/.gitignore @@ -1,5 +1,6 @@ *-bin *-dist +.y-script-lint.json # our executables are symlinked to real names buildctl diff --git a/bin/y-script-lint b/bin/y-script-lint new file mode 100755 index 00000000..28358c22 --- /dev/null +++ b/bin/y-script-lint @@ -0,0 +1,514 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +show_help() { + cat <<'EOF' +y-script-lint - Static analysis and convention checks for y-* scripts + +Usage: y-script-lint [options] [DIR] + +Arguments: + DIR Directory to scan (default: ./bin) + +Options: + --check Exit non-zero if any script fails static checks (for CI) + --phase1-only Skip sandboxed --help execution (phase 2) + --json Write results to DIR/.y-script-lint.json + -h, --help Show this help + +Phase 1 (static, never executes scripts): + - Shebang detection and language classification + - Header checks: set -eo pipefail, DEBUG pattern + - Help handler detection in source + - shellcheck (if y-shellcheck is available) + - No npx usage + - No unguarded eval + +Phase 2 (sandboxed --help execution): + - Only runs for scripts where phase 1 found a help handler + - Sandbox: env -i, temp HOME, temp workdir, timeout 5s + - Linux: network isolation via unshare --net + - TODO: macOS sandboxing (currently skips phase 2 on Darwin) + - Validates help output format: first line, Dependencies section + +Dependencies: + jq Required for --json output + y-shellcheck Optional, shell lint (skipped if not available) + unshare Optional, Linux network sandbox for phase 2 + +Exit codes: + 0 All scripts passed (or --check not set) + 1 Usage error + 2 One or more scripts failed checks (--check mode) +EOF +} + +CHECK_MODE=false +PHASE1_ONLY=false +JSON_OUTPUT=false +TARGET_DIR="" + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) show_help; exit 0 ;; + --check) CHECK_MODE=true; shift ;; + --phase1-only) PHASE1_ONLY=true; shift ;; + --json) JSON_OUTPUT=true; shift ;; + --) shift; break ;; + -*) echo "Unknown flag: $1" >&2; exit 1 ;; + *) TARGET_DIR="$1"; shift ;; + esac +done + +[ -z "$TARGET_DIR" ] && TARGET_DIR="./bin" +[ -d "$TARGET_DIR" ] || { echo "ERROR: directory not found: $TARGET_DIR" >&2; exit 1; } +TARGET_DIR="$(cd "$TARGET_DIR" && pwd)" + +# --- Language detection --- + +detect_language() { + local file="$1" + local shebang + shebang=$(head -1 "$file" 2>/dev/null) || true + + case "$shebang" in + '#!/usr/bin/env bash'|'#!/bin/bash') echo "bash" ;; + '#!/bin/sh') echo "sh" ;; + *node*) echo "node" ;; + *python*) echo "python" ;; + *y-bin-download*) echo "config" ;; + *) + # No recognized shebang - check extension + case "$file" in + *.js|*.ts) echo "library" ;; + *) echo "unknown" ;; + esac + ;; + esac +} + +is_shell() { + local lang="$1" + [ "$lang" = "bash" ] || [ "$lang" = "sh" ] +} + +# --- Static checks (phase 1) --- + +check_header_pipefail() { + local file="$1" + grep -qE '^set -e(o pipefail)?$' "$file" 2>/dev/null +} + +check_header_debug() { + local file="$1" + grep -qE '^\[ -z "\$DEBUG" \] \|\| set -x' "$file" 2>/dev/null +} + +check_help_handler() { + local file="$1" + local lang="$2" + + if is_shell "$lang"; then + # Match: -h|--help, "$1" = "help", "$1" = "--help", =~ help + grep -qE '(\-h\|--help|"\$1" = "help"|"\$1" = "--help"|\$1 =~ help)' "$file" 2>/dev/null + elif [ "$lang" = "node" ]; then + grep -qE "(process\.argv\.includes\(['\"]--help['\"])" "$file" 2>/dev/null + else + return 1 + fi +} + +check_no_npx() { + local file="$1" + # Ignore comments; match only lines where npx is used as a command + ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_npx|"uses npx"|No npx)' | grep -qE '\bnpx\b' +} + +check_no_eval() { + local file="$1" + # Ignore comments; match only lines where eval is used as a command + ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_eval|"uses eval"|No.*eval)' | grep -qE '\beval\b' +} + +# --- Shellcheck --- + +HAS_SHELLCHECK=false +if command -v y-shellcheck >/dev/null 2>&1; then + HAS_SHELLCHECK=true +fi + +run_shellcheck() { + local file="$1" + if [ "$HAS_SHELLCHECK" = "true" ]; then + y-shellcheck --severity=error "$file" >/dev/null 2>&1 + else + return 0 # skip, not a failure + fi +} + +# --- Phase 2: sandboxed --help execution --- + +CAN_SANDBOX=true +HAS_UNSHARE=false +if [ "$(uname -s)" = "Linux" ]; then + # Test if unshare --net actually works (needs user namespace or root) + if command -v unshare >/dev/null 2>&1 && unshare --net true 2>/dev/null; then + HAS_UNSHARE=true + fi +fi +if [ "$(uname -s)" = "Darwin" ]; then + # TODO: macOS sandbox using sandbox-exec or similar + CAN_SANDBOX=false +fi + +run_sandboxed_help() { + local script="$1" + local tmpdir + tmpdir=$(mktemp -d) + + local output="" + local exit_code=0 + + # Build sandbox command + local sandbox_cmd=(env -i PATH="$PATH" HOME="$tmpdir" timeout 5) + if [ "$HAS_UNSHARE" = "true" ]; then + sandbox_cmd=(unshare --net "${sandbox_cmd[@]}") + fi + + # Capture both stdout and stderr — some scripts print help to stderr + output=$("${sandbox_cmd[@]}" /bin/bash -c 'cd "$(mktemp -d)" && exec "$@"' _ "$script" --help 2>&1) || exit_code=$? + + rm -rf "$tmpdir" + + # Timeout (124) or signal kill (137) means the script hung + if [ $exit_code -eq 124 ] || [ $exit_code -eq 137 ]; then + return 1 + fi + + # Some scripts exit non-zero after printing help (e.g. prereq checks before help handler) + # Accept output even on non-zero exit if it looks like help text + if [ -n "$output" ]; then + echo "$output" + return 0 + fi + return 1 +} + +parse_help_summary() { + local output="$1" + local first_line + first_line=$(echo "$output" | head -1) + echo "$first_line" +} + +check_help_format() { + local first_line="$1" + local script_name="$2" + # Expected: y-name - description + echo "$first_line" | grep -qE "^${script_name} - .+" 2>/dev/null +} + +parse_help_dependencies() { + local output="$1" + # Extract lines after "Dependencies:" until the next blank line or section + echo "$output" | awk '/^Dependencies:/{found=1; next} found && /^$/{exit} found && /^[A-Z]/{exit} found{print}' | \ + awk '{print $1}' | grep -v '^$' || true +} + +check_help_has_deps_section() { + local output="$1" + echo "$output" | grep -q '^Dependencies:' 2>/dev/null +} + +# --- Main --- + +TOTAL=0 +PASSED=0 +FAILED_STATIC=0 +FAILED_HELP=0 +SKIPPED=0 +FAIL_LIST="" + +# Collect results as JSON fragments +JSON_SCRIPTS="" + +for file in "$TARGET_DIR"/y-*; do + [ -f "$file" ] || continue + name=$(basename "$file") + + # Skip spec/ directory entries, non-executable, yaml configs + [ -x "$file" ] || continue + case "$name" in + *.yaml|*.yml|*.md) continue ;; + *-bin) continue ;; # compiled binaries, not scripts + *-dist) continue ;; + esac + + TOTAL=$((TOTAL + 1)) + + lang=$(detect_language "$file") + + # Skip non-script files + if [ "$lang" = "config" ] || [ "$lang" = "library" ]; then + SKIPPED=$((SKIPPED + 1)) + if [ "$JSON_OUTPUT" = "true" ]; then + JSON_SCRIPTS="${JSON_SCRIPTS}$(jq -n \ + --arg name "$name" \ + --arg lang "$lang" \ + '{($name): {language: $lang, skipped: true}}' + )," + fi + echo " SKIP $name ($lang)" + continue + fi + + errors=() + checks_shebang=true + checks_header=true + checks_debug=true + checks_help_handler=false + checks_shellcheck=null + checks_no_npx=true + checks_no_eval=true + checks_help_runs=null + checks_help_format=null + checks_deps_declared=null + summary=null + deps_json="[]" + + # Unknown shebang + if [ "$lang" = "unknown" ]; then + checks_shebang=false + errors+=("unrecognized or missing shebang") + fi + + # Header checks (shell scripts only) + if is_shell "$lang"; then + if ! check_header_pipefail "$file"; then + checks_header=false + errors+=("missing set -eo pipefail or set -e") + fi + if ! check_header_debug "$file"; then + checks_debug=false + errors+=("missing DEBUG pattern: [ -z \"\\\$DEBUG\" ] || set -x") + fi + fi + + # Help handler + if check_help_handler "$file" "$lang"; then + checks_help_handler=true + else + errors+=("no help handler found in source") + fi + + # npx check + if ! check_no_npx "$file"; then + checks_no_npx=false + errors+=("uses npx") + fi + + # eval check + if ! check_no_eval "$file"; then + checks_no_eval=false + errors+=("uses eval") + fi + + # Shellcheck (shell scripts only) + if is_shell "$lang"; then + if [ "$HAS_SHELLCHECK" = "true" ]; then + if run_shellcheck "$file"; then + checks_shellcheck=true + else + checks_shellcheck=false + errors+=("shellcheck --severity=error failed") + fi + fi + fi + + # Phase 2: sandboxed --help + if [ "$PHASE1_ONLY" = "false" ] && [ "$checks_help_handler" = "true" ]; then + if [ "$CAN_SANDBOX" = "false" ]; then + # TODO: macOS sandboxing + checks_help_runs=null + checks_help_format=null + checks_deps_declared=null + else + help_output=$(run_sandboxed_help "$file") || true + if [ -n "$help_output" ]; then + checks_help_runs=true + first_line=$(parse_help_summary "$help_output") + summary="$first_line" + + if check_help_format "$first_line" "$name"; then + checks_help_format=true + else + checks_help_format=false + errors+=("help first line does not match format: $name - description") + fi + + if check_help_has_deps_section "$help_output"; then + checks_deps_declared=true + deps_raw=$(parse_help_dependencies "$help_output") + if [ -n "$deps_raw" ]; then + deps_json=$(echo "$deps_raw" | jq -R . | jq -s .) + else + deps_json="[]" + fi + else + checks_deps_declared=false + errors+=("no Dependencies section in help output") + fi + else + checks_help_runs=false + errors+=("--help produced no output or failed") + fi + fi + fi + + # Determine pass/fail + has_static_failure=false + if [ "$checks_shebang" = "false" ] || [ "$checks_header" = "false" ] \ + || [ "$checks_no_npx" = "false" ] || [ "$checks_no_eval" = "false" ] \ + || [ "$checks_shellcheck" = "false" ]; then + has_static_failure=true + fi + + has_help_failure=false + if [ "$checks_help_handler" = "false" ] || [ "$checks_help_runs" = "false" ] \ + || [ "$checks_help_format" = "false" ]; then + has_help_failure=true + fi + + if [ "$has_static_failure" = "true" ]; then + FAILED_STATIC=$((FAILED_STATIC + 1)) + FAIL_LIST="$FAIL_LIST $name" + status="FAIL" + elif [ "$has_help_failure" = "true" ]; then + FAILED_HELP=$((FAILED_HELP + 1)) + status="WARN" + else + PASSED=$((PASSED + 1)) + status="OK " + fi + + # Build error list for display + if [ ${#errors[@]} -gt 0 ]; then + error_detail=$(printf ", %s" "${errors[@]}") + error_detail=${error_detail:2} # strip leading ", " + echo " $status $name ($lang): $error_detail" + else + echo " $status $name ($lang)" + fi + + # Build JSON + if [ "$JSON_OUTPUT" = "true" ]; then + errors_json="[]" + if [ ${#errors[@]} -gt 0 ]; then + errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) + fi + + summary_json="null" + if [ "$summary" != "null" ] && [ -n "$summary" ]; then + summary_json=$(echo "$summary" | jq -R .) + fi + + # Build checks object - use jq to handle null vs bool properly + checks_json=$(jq -n \ + --argjson shebang "$checks_shebang" \ + --argjson header "$checks_header" \ + --argjson debug "$checks_debug" \ + --argjson help_handler "$checks_help_handler" \ + --argjson shellcheck "$checks_shellcheck" \ + --argjson no_npx "$checks_no_npx" \ + --argjson no_eval "$checks_no_eval" \ + --argjson help_runs "$checks_help_runs" \ + --argjson help_format "$checks_help_format" \ + --argjson deps_declared "$checks_deps_declared" \ + '{ + shebang: $shebang, + header: $header, + debug: $debug, + help_handler: $help_handler, + shellcheck: $shellcheck, + no_npx: $no_npx, + no_eval: $no_eval, + help_runs: $help_runs, + help_format: $help_format, + deps_declared: $deps_declared + }') + + script_json=$(jq -n \ + --arg lang "$lang" \ + --argjson summary "$summary_json" \ + --argjson deps "$deps_json" \ + --argjson checks "$checks_json" \ + --argjson errors "$errors_json" \ + '{ + language: $lang, + summary: $summary, + dependencies: $deps, + checks: $checks, + errors: $errors + }') + + JSON_SCRIPTS="${JSON_SCRIPTS}$(jq -n --arg name "$name" --argjson data "$script_json" '{($name): $data}')," + fi +done + +echo "" +echo " Total: $TOTAL Passed: $PASSED Failed(static): $FAILED_STATIC Failed(help): $FAILED_HELP Skipped: $SKIPPED" + +if [ "$HAS_SHELLCHECK" = "false" ]; then + echo " NOTE: y-shellcheck not found, shell lint checks were skipped" +fi +if [ "$CAN_SANDBOX" = "false" ]; then + echo " NOTE: macOS detected, phase 2 sandboxed execution was skipped (TODO)" +fi + +# Write JSON output +if [ "$JSON_OUTPUT" = "true" ]; then + command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required for --json output" >&2; exit 1; } + + # Merge all script JSON fragments + if [ -n "$JSON_SCRIPTS" ]; then + # Remove trailing comma, wrap in array, merge with jq + scripts_merged=$(echo "[${JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)') + else + scripts_merged="{}" + fi + + jq -n \ + --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg tool_version "1" \ + --arg directory "$TARGET_DIR" \ + --argjson total "$TOTAL" \ + --argjson passed "$PASSED" \ + --argjson failed_static "$FAILED_STATIC" \ + --argjson failed_help "$FAILED_HELP" \ + --argjson skipped "$SKIPPED" \ + --argjson scripts "$scripts_merged" \ + '{ + generated: $generated, + tool_version: $tool_version, + directory: $directory, + stats: { + total: $total, + passed: $passed, + failed_static: $failed_static, + failed_help: $failed_help, + skipped: $skipped + }, + scripts: $scripts + }' > "$TARGET_DIR/.y-script-lint.json" + + echo " Wrote $TARGET_DIR/.y-script-lint.json" +fi + +# --check mode: exit non-zero if any static failures +if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then + echo " FAILED: $FAIL_LIST" + exit 2 +fi From f3eeca74748ea18ccf3194b738d3f8443051d356 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 06:05:23 +0000 Subject: [PATCH 03/28] add Node.js and TypeScript shebang support - Detect #!/usr/bin/env node as "node" language - Detect #!/usr/bin/env -S node --experimental-strip-types as "typescript" - Skip header/debug checks for non-shell scripts (null in JSON) - Node.js help handler: process.argv.includes('--help'/'-h') - Language-aware eval check: eval( for JS/TS, bare eval for shell - Language-aware npx check: // comments for JS/TS, # comments for shell - Skip .spec.js/.spec.ts/.test.js/.test.ts files Tested on all three repos: - ystack: 73 scripts (all shell), 9 pass, 4 static failures - checkit: 137 scripts (16 node), 36 pass, 18 static failures - bots: 3 scripts (1 typescript), 2 pass, 1 static failure Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 28358c22..b712a440 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -77,6 +77,7 @@ detect_language() { case "$shebang" in '#!/usr/bin/env bash'|'#!/bin/bash') echo "bash" ;; '#!/bin/sh') echo "sh" ;; + *node*experimental-strip-types*) echo "typescript" ;; *node*) echo "node" ;; *python*) echo "python" ;; *y-bin-download*) echo "config" ;; @@ -95,6 +96,11 @@ is_shell() { [ "$lang" = "bash" ] || [ "$lang" = "sh" ] } +is_node() { + local lang="$1" + [ "$lang" = "node" ] || [ "$lang" = "typescript" ] +} + # --- Static checks (phase 1) --- check_header_pipefail() { @@ -114,8 +120,9 @@ check_help_handler() { if is_shell "$lang"; then # Match: -h|--help, "$1" = "help", "$1" = "--help", =~ help grep -qE '(\-h\|--help|"\$1" = "help"|"\$1" = "--help"|\$1 =~ help)' "$file" 2>/dev/null - elif [ "$lang" = "node" ]; then - grep -qE "(process\.argv\.includes\(['\"]--help['\"])" "$file" 2>/dev/null + elif is_node "$lang"; then + # Match: process.argv.includes('--help') or includes('-h') + grep -qE "process\.argv\.includes\(['\"](-h|--help)['\"]" "$file" 2>/dev/null else return 1 fi @@ -123,14 +130,26 @@ check_help_handler() { check_no_npx() { local file="$1" - # Ignore comments; match only lines where npx is used as a command - ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_npx|"uses npx"|No npx)' | grep -qE '\bnpx\b' + local lang="$2" + if is_node "$lang"; then + # JS/TS: ignore // comments + ! grep -vE '^\s*//' "$file" 2>/dev/null | grep -qE '\bnpx\b' + else + # Shell: ignore # comments and self-references + ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_npx|"uses npx"|No npx)' | grep -qE '\bnpx\b' + fi } check_no_eval() { local file="$1" - # Ignore comments; match only lines where eval is used as a command - ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_eval|"uses eval"|No.*eval)' | grep -qE '\beval\b' + local lang="$2" + if is_node "$lang"; then + # JS/TS: match eval( function call, ignore comments + ! grep -vE '^\s*//' "$file" 2>/dev/null | grep -qE '\beval\s*\(' + else + # Shell: match bare eval command, ignore comments and self-references + ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_eval|"uses eval"|No.*eval)' | grep -qE '\beval\b' + fi } # --- Shellcheck --- @@ -243,6 +262,7 @@ for file in "$TARGET_DIR"/y-*; do [ -x "$file" ] || continue case "$name" in *.yaml|*.yml|*.md) continue ;; + *.spec.js|*.spec.ts|*.test.js|*.test.ts) continue ;; # test files *-bin) continue ;; # compiled binaries, not scripts *-dist) continue ;; esac @@ -251,7 +271,7 @@ for file in "$TARGET_DIR"/y-*; do lang=$(detect_language "$file") - # Skip non-script files + # Skip non-script files (configs, libraries without shebangs) if [ "$lang" = "config" ] || [ "$lang" = "library" ]; then SKIPPED=$((SKIPPED + 1)) if [ "$JSON_OUTPUT" = "true" ]; then @@ -295,6 +315,10 @@ for file in "$TARGET_DIR"/y-*; do checks_debug=false errors+=("missing DEBUG pattern: [ -z \"\\\$DEBUG\" ] || set -x") fi + else + # Not applicable to non-shell scripts + checks_header=null + checks_debug=null fi # Help handler @@ -305,13 +329,13 @@ for file in "$TARGET_DIR"/y-*; do fi # npx check - if ! check_no_npx "$file"; then + if ! check_no_npx "$file" "$lang"; then checks_no_npx=false errors+=("uses npx") fi # eval check - if ! check_no_eval "$file"; then + if ! check_no_eval "$file" "$lang"; then checks_no_eval=false errors+=("uses eval") fi From 4c78f83089eab11f5845e7d118899a7404a3ce24 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 06:18:05 +0000 Subject: [PATCH 04/28] discover y-* scripts from PATH by default Without arguments, scans all PATH directories containing y-* scripts. Groups output by directory with headers. Deduplicates by script name (first PATH entry wins, matching shell resolution). Supports multiple explicit DIR arguments. Writes per-directory .y-script-lint.json. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 220 ++++++++++++++++++++++++++-------------------- 1 file changed, 125 insertions(+), 95 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index b712a440..43746a73 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -8,10 +8,10 @@ show_help() { cat <<'EOF' y-script-lint - Static analysis and convention checks for y-* scripts -Usage: y-script-lint [options] [DIR] +Usage: y-script-lint [options] [DIR...] Arguments: - DIR Directory to scan (default: ./bin) + DIR Directory to scan (default: all PATH dirs containing y-* scripts) Options: --check Exit non-zero if any script fails static checks (for CI) @@ -49,7 +49,7 @@ EOF CHECK_MODE=false PHASE1_ONLY=false JSON_OUTPUT=false -TARGET_DIR="" +TARGET_DIRS=() while [ $# -gt 0 ]; do case "$1" in @@ -59,13 +59,35 @@ while [ $# -gt 0 ]; do --json) JSON_OUTPUT=true; shift ;; --) shift; break ;; -*) echo "Unknown flag: $1" >&2; exit 1 ;; - *) TARGET_DIR="$1"; shift ;; + *) TARGET_DIRS+=("$1"); shift ;; esac done -[ -z "$TARGET_DIR" ] && TARGET_DIR="./bin" -[ -d "$TARGET_DIR" ] || { echo "ERROR: directory not found: $TARGET_DIR" >&2; exit 1; } -TARGET_DIR="$(cd "$TARGET_DIR" && pwd)" +# Default: discover from PATH +if [ ${#TARGET_DIRS[@]} -eq 0 ]; then + IFS=: read -ra path_entries <<< "$PATH" + for dir in "${path_entries[@]}"; do + [ -d "$dir" ] || continue + # Include directories that contain at least one y-* executable + for f in "$dir"/y-*; do + if [ -x "$f" ] && [ -f "$f" ]; then + TARGET_DIRS+=("$dir") + break + fi + done + done + if [ ${#TARGET_DIRS[@]} -eq 0 ]; then + echo "ERROR: no y-* scripts found in PATH" >&2 + exit 1 + fi +fi + +# Resolve to absolute paths +RESOLVED_DIRS=() +for dir in "${TARGET_DIRS[@]}"; do + [ -d "$dir" ] || { echo "ERROR: directory not found: $dir" >&2; exit 1; } + RESOLVED_DIRS+=("$(cd "$dir" && pwd)") +done # --- Language detection --- @@ -242,62 +264,42 @@ check_help_has_deps_section() { echo "$output" | grep -q '^Dependencies:' 2>/dev/null } -# --- Main --- +# --- Per-file lint --- -TOTAL=0 -PASSED=0 -FAILED_STATIC=0 -FAILED_HELP=0 -SKIPPED=0 -FAIL_LIST="" - -# Collect results as JSON fragments -JSON_SCRIPTS="" - -for file in "$TARGET_DIR"/y-*; do - [ -f "$file" ] || continue +lint_file() { + local file="$1" + local name name=$(basename "$file") - # Skip spec/ directory entries, non-executable, yaml configs - [ -x "$file" ] || continue - case "$name" in - *.yaml|*.yml|*.md) continue ;; - *.spec.js|*.spec.ts|*.test.js|*.test.ts) continue ;; # test files - *-bin) continue ;; # compiled binaries, not scripts - *-dist) continue ;; - esac - - TOTAL=$((TOTAL + 1)) - lang=$(detect_language "$file") # Skip non-script files (configs, libraries without shebangs) if [ "$lang" = "config" ] || [ "$lang" = "library" ]; then SKIPPED=$((SKIPPED + 1)) if [ "$JSON_OUTPUT" = "true" ]; then - JSON_SCRIPTS="${JSON_SCRIPTS}$(jq -n \ + DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ --arg name "$name" \ --arg lang "$lang" \ '{($name): {language: $lang, skipped: true}}' )," fi echo " SKIP $name ($lang)" - continue + return fi - errors=() - checks_shebang=true - checks_header=true - checks_debug=true - checks_help_handler=false - checks_shellcheck=null - checks_no_npx=true - checks_no_eval=true - checks_help_runs=null - checks_help_format=null - checks_deps_declared=null - summary=null - deps_json="[]" + local errors=() + local checks_shebang=true + local checks_header=true + local checks_debug=true + local checks_help_handler=false + local checks_shellcheck=null + local checks_no_npx=true + local checks_no_eval=true + local checks_help_runs=null + local checks_help_format=null + local checks_deps_declared=null + local summary=null + local deps_json="[]" # Unknown shebang if [ "$lang" = "unknown" ]; then @@ -316,7 +318,6 @@ for file in "$TARGET_DIR"/y-*; do errors+=("missing DEBUG pattern: [ -z \"\\\$DEBUG\" ] || set -x") fi else - # Not applicable to non-shell scripts checks_header=null checks_debug=null fi @@ -355,14 +356,15 @@ for file in "$TARGET_DIR"/y-*; do # Phase 2: sandboxed --help if [ "$PHASE1_ONLY" = "false" ] && [ "$checks_help_handler" = "true" ]; then if [ "$CAN_SANDBOX" = "false" ]; then - # TODO: macOS sandboxing checks_help_runs=null checks_help_format=null checks_deps_declared=null else + local help_output help_output=$(run_sandboxed_help "$file") || true if [ -n "$help_output" ]; then checks_help_runs=true + local first_line first_line=$(parse_help_summary "$help_output") summary="$first_line" @@ -375,6 +377,7 @@ for file in "$TARGET_DIR"/y-*; do if check_help_has_deps_section "$help_output"; then checks_deps_declared=true + local deps_raw deps_raw=$(parse_help_dependencies "$help_output") if [ -n "$deps_raw" ]; then deps_json=$(echo "$deps_raw" | jq -R . | jq -s .) @@ -393,19 +396,20 @@ for file in "$TARGET_DIR"/y-*; do fi # Determine pass/fail - has_static_failure=false + local has_static_failure=false if [ "$checks_shebang" = "false" ] || [ "$checks_header" = "false" ] \ || [ "$checks_no_npx" = "false" ] || [ "$checks_no_eval" = "false" ] \ || [ "$checks_shellcheck" = "false" ]; then has_static_failure=true fi - has_help_failure=false + local has_help_failure=false if [ "$checks_help_handler" = "false" ] || [ "$checks_help_runs" = "false" ] \ || [ "$checks_help_format" = "false" ]; then has_help_failure=true fi + local status if [ "$has_static_failure" = "true" ]; then FAILED_STATIC=$((FAILED_STATIC + 1)) FAIL_LIST="$FAIL_LIST $name" @@ -418,10 +422,10 @@ for file in "$TARGET_DIR"/y-*; do status="OK " fi - # Build error list for display if [ ${#errors[@]} -gt 0 ]; then + local error_detail error_detail=$(printf ", %s" "${errors[@]}") - error_detail=${error_detail:2} # strip leading ", " + error_detail=${error_detail:2} echo " $status $name ($lang): $error_detail" else echo " $status $name ($lang)" @@ -429,17 +433,17 @@ for file in "$TARGET_DIR"/y-*; do # Build JSON if [ "$JSON_OUTPUT" = "true" ]; then - errors_json="[]" + local errors_json="[]" if [ ${#errors[@]} -gt 0 ]; then errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) fi - summary_json="null" + local summary_json="null" if [ "$summary" != "null" ] && [ -n "$summary" ]; then summary_json=$(echo "$summary" | jq -R .) fi - # Build checks object - use jq to handle null vs bool properly + local checks_json checks_json=$(jq -n \ --argjson shebang "$checks_shebang" \ --argjson header "$checks_header" \ @@ -464,6 +468,7 @@ for file in "$TARGET_DIR"/y-*; do deps_declared: $deps_declared }') + local script_json script_json=$(jq -n \ --arg lang "$lang" \ --argjson summary "$summary_json" \ @@ -478,7 +483,71 @@ for file in "$TARGET_DIR"/y-*; do errors: $errors }') - JSON_SCRIPTS="${JSON_SCRIPTS}$(jq -n --arg name "$name" --argjson data "$script_json" '{($name): $data}')," + DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n --arg name "$name" --argjson data "$script_json" '{($name): $data}')," + fi +} + +# --- Main --- + +TOTAL=0 +PASSED=0 +FAILED_STATIC=0 +FAILED_HELP=0 +SKIPPED=0 +FAIL_LIST="" + +# Track seen script names to skip PATH duplicates +declare -A SEEN_SCRIPTS + +for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do + # Collect y-* scripts in this directory + dir_scripts=() + for file in "$TARGET_DIR"/y-*; do + [ -f "$file" ] && [ -x "$file" ] || continue + name=$(basename "$file") + case "$name" in + *.yaml|*.yml|*.md) continue ;; + *.spec.js|*.spec.ts|*.test.js|*.test.ts) continue ;; + *-bin) continue ;; + *-dist) continue ;; + esac + # Skip duplicates (first PATH entry wins) + [ -n "${SEEN_SCRIPTS[$name]:-}" ] && continue + SEEN_SCRIPTS[$name]=1 + dir_scripts+=("$file") + done + + [ ${#dir_scripts[@]} -eq 0 ] && continue + + echo "" + echo " $TARGET_DIR (${#dir_scripts[@]} scripts)" + + DIR_JSON_SCRIPTS="" + + for file in "${dir_scripts[@]}"; do + TOTAL=$((TOTAL + 1)) + lint_file "$file" + done + + # Write per-directory JSON + if [ "$JSON_OUTPUT" = "true" ] && [ -n "$DIR_JSON_SCRIPTS" ]; then + command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required for --json output" >&2; exit 1; } + + local_scripts_merged=$(echo "[${DIR_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)') + + jq -n \ + --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg tool_version "1" \ + --arg directory "$TARGET_DIR" \ + --argjson scripts "$local_scripts_merged" \ + '{ + generated: $generated, + tool_version: $tool_version, + directory: $directory, + scripts: $scripts + }' > "$TARGET_DIR/.y-script-lint.json" + + echo " Wrote $TARGET_DIR/.y-script-lint.json" fi done @@ -492,45 +561,6 @@ if [ "$CAN_SANDBOX" = "false" ]; then echo " NOTE: macOS detected, phase 2 sandboxed execution was skipped (TODO)" fi -# Write JSON output -if [ "$JSON_OUTPUT" = "true" ]; then - command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required for --json output" >&2; exit 1; } - - # Merge all script JSON fragments - if [ -n "$JSON_SCRIPTS" ]; then - # Remove trailing comma, wrap in array, merge with jq - scripts_merged=$(echo "[${JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)') - else - scripts_merged="{}" - fi - - jq -n \ - --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --arg tool_version "1" \ - --arg directory "$TARGET_DIR" \ - --argjson total "$TOTAL" \ - --argjson passed "$PASSED" \ - --argjson failed_static "$FAILED_STATIC" \ - --argjson failed_help "$FAILED_HELP" \ - --argjson skipped "$SKIPPED" \ - --argjson scripts "$scripts_merged" \ - '{ - generated: $generated, - tool_version: $tool_version, - directory: $directory, - stats: { - total: $total, - passed: $passed, - failed_static: $failed_static, - failed_help: $failed_help, - skipped: $skipped - }, - scripts: $scripts - }' > "$TARGET_DIR/.y-script-lint.json" - - echo " Wrote $TARGET_DIR/.y-script-lint.json" -fi - # --check mode: exit non-zero if any static failures if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then echo " FAILED: $FAIL_LIST" From 05d0bd46eac9b9e7f7e83dfb451845c178414e85 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 06:24:59 +0000 Subject: [PATCH 05/28] configurable sandbox timeout via Y_SCRIPT_LINT_TIMEOUT_S Default 5 seconds. Passed directly to timeout(1). Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 43746a73..f21fe084 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -34,6 +34,9 @@ Phase 2 (sandboxed --help execution): - TODO: macOS sandboxing (currently skips phase 2 on Darwin) - Validates help output format: first line, Dependencies section +Environment: + Y_SCRIPT_LINT_TIMEOUT_S Phase 2 sandbox timeout in seconds (default: 5) + Dependencies: jq Required for --json output y-shellcheck Optional, shell lint (skipped if not available) @@ -192,6 +195,8 @@ run_shellcheck() { # --- Phase 2: sandboxed --help execution --- +SANDBOX_TIMEOUT_S="${Y_SCRIPT_LINT_TIMEOUT_S:-5}" + CAN_SANDBOX=true HAS_UNSHARE=false if [ "$(uname -s)" = "Linux" ]; then @@ -214,7 +219,7 @@ run_sandboxed_help() { local exit_code=0 # Build sandbox command - local sandbox_cmd=(env -i PATH="$PATH" HOME="$tmpdir" timeout 5) + local sandbox_cmd=(env -i PATH="$PATH" HOME="$tmpdir" timeout "$SANDBOX_TIMEOUT_S") if [ "$HAS_UNSHARE" = "true" ]; then sandbox_cmd=(unshare --net "${sandbox_cmd[@]}") fi From cc5accc489a4d1cf1f224a0657e866dbecbf4355 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 07:07:09 +0000 Subject: [PATCH 06/28] tree output format with LOC and per-error lines Output format: [y-script-lint] /path/to/bin (N scripts) y-example bash 42L OK y-broken bash 15L FAIL uses eval shellcheck --severity=error failed Total: N Passed: N Failed: N Warnings: N Skipped: N Wrote /path/to/bin/.y-script-lint.json Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index f21fe084..2274907c 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -288,10 +288,12 @@ lint_file() { '{($name): {language: $lang, skipped: true}}' )," fi - echo " SKIP $name ($lang)" return fi + local loc + loc=$(wc -l < "$file") + local errors=() local checks_shebang=true local checks_header=true @@ -424,17 +426,13 @@ lint_file() { status="WARN" else PASSED=$((PASSED + 1)) - status="OK " + status="OK" fi - if [ ${#errors[@]} -gt 0 ]; then - local error_detail - error_detail=$(printf ", %s" "${errors[@]}") - error_detail=${error_detail:2} - echo " $status $name ($lang): $error_detail" - else - echo " $status $name ($lang)" - fi + echo " $name $lang ${loc}L $status" + for err in "${errors[@]}"; do + echo " $err" + done # Build JSON if [ "$JSON_OUTPUT" = "true" ]; then @@ -503,6 +501,7 @@ FAIL_LIST="" # Track seen script names to skip PATH duplicates declare -A SEEN_SCRIPTS +JSON_FILES=() for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do # Collect y-* scripts in this directory @@ -524,8 +523,7 @@ for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do [ ${#dir_scripts[@]} -eq 0 ] && continue - echo "" - echo " $TARGET_DIR (${#dir_scripts[@]} scripts)" + echo "[y-script-lint] $TARGET_DIR (${#dir_scripts[@]} scripts)" DIR_JSON_SCRIPTS="" @@ -540,6 +538,7 @@ for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do local_scripts_merged=$(echo "[${DIR_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)') + JSON_FILE="$TARGET_DIR/.y-script-lint.json" jq -n \ --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --arg tool_version "1" \ @@ -550,24 +549,27 @@ for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do tool_version: $tool_version, directory: $directory, scripts: $scripts - }' > "$TARGET_DIR/.y-script-lint.json" + }' > "$JSON_FILE" - echo " Wrote $TARGET_DIR/.y-script-lint.json" + JSON_FILES+=("$JSON_FILE") fi done -echo "" -echo " Total: $TOTAL Passed: $PASSED Failed(static): $FAILED_STATIC Failed(help): $FAILED_HELP Skipped: $SKIPPED" - +NOTES="" if [ "$HAS_SHELLCHECK" = "false" ]; then - echo " NOTE: y-shellcheck not found, shell lint checks were skipped" + NOTES="${NOTES}, no shellcheck" fi if [ "$CAN_SANDBOX" = "false" ]; then - echo " NOTE: macOS detected, phase 2 sandboxed execution was skipped (TODO)" + NOTES="${NOTES}, no macOS sandbox" fi +echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" +for jf in "${JSON_FILES[@]}"; do + echo "Wrote $jf" +done + # --check mode: exit non-zero if any static failures if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then - echo " FAILED: $FAIL_LIST" + echo "FAILED:$FAIL_LIST" exit 2 fi From a26ae14f60c8b34bde9fb0bf1184ed9cf5d7a264 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 07:17:34 +0000 Subject: [PATCH 07/28] move WARN/FAIL labels to individual error lines Each issue line is now prefixed with its severity: y-bin-download bash 189L WARN missing DEBUG pattern FAIL uses eval Script name line no longer carries a status label. Clean scripts have no children in the tree. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 2274907c..49645d59 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -416,22 +416,23 @@ lint_file() { has_help_failure=true fi - local status if [ "$has_static_failure" = "true" ]; then FAILED_STATIC=$((FAILED_STATIC + 1)) FAIL_LIST="$FAIL_LIST $name" - status="FAIL" elif [ "$has_help_failure" = "true" ]; then FAILED_HELP=$((FAILED_HELP + 1)) - status="WARN" else PASSED=$((PASSED + 1)) - status="OK" fi - echo " $name $lang ${loc}L $status" + echo " $name $lang ${loc}L" for err in "${errors[@]}"; do - echo " $err" + # Static failures are FAIL, help-related are WARN + local level="WARN" + case "$err" in + "unrecognized or missing shebang"|"missing set -eo pipefail"*|"uses npx"|"uses eval"|"shellcheck "*) level="FAIL" ;; + esac + echo " $level $err" done # Build JSON From d5e048fecea3f54e028d52fa89c6adb5ba3a0e54 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 07:11:32 +0000 Subject: [PATCH 08/28] fix Ctrl+C and add timeout to shellcheck - Trap SIGINT to exit 130 immediately (Ctrl+C was blocked by set -e and subshell pipe chains) - Wrap y-shellcheck in timeout using same Y_SCRIPT_LINT_TIMEOUT_S (default 5s per file) to prevent hangs on phase 1 Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 49645d59..282d862a 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -2,6 +2,9 @@ [ -z "$DEBUG" ] || set -x set -eo pipefail +# Ensure Ctrl+C kills the whole process group +trap 'exit 130' INT + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" show_help() { @@ -187,7 +190,7 @@ fi run_shellcheck() { local file="$1" if [ "$HAS_SHELLCHECK" = "true" ]; then - y-shellcheck --severity=error "$file" >/dev/null 2>&1 + timeout "$SANDBOX_TIMEOUT_S" y-shellcheck --severity=error "$file" >/dev/null 2>&1 else return 0 # skip, not a failure fi From 5909e2449e2cb18ec428c671fb3876bc81f032b6 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 07:22:12 +0000 Subject: [PATCH 09/28] fix Ctrl+C by killing entire process group Previous trap waited for hung children to finish before exiting. New trap resets itself then sends SIGINT to the process group (-$$), killing all children (shellcheck, timeout, grep) immediately. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 282d862a..4425d6d0 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -2,8 +2,8 @@ [ -z "$DEBUG" ] || set -x set -eo pipefail -# Ensure Ctrl+C kills the whole process group -trap 'exit 130' INT +# Ensure Ctrl+C kills the whole process group, including hung children +trap 'trap - INT; kill -INT -$$' INT SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" From a66ed4624b9d8d14a5172b5d8d1dfcf8110560d3 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 07:42:45 +0000 Subject: [PATCH 10/28] fix timeout not killing hung children timeout sends SIGTERM which can be ignored by child process trees (e.g. y-vault calling y-bin-dependency-download making network requests). Add --kill-after=2 to both sandbox and shellcheck timeouts so they SIGKILL after 2s grace period if SIGTERM is insufficient. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 4425d6d0..2faadf42 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -190,7 +190,7 @@ fi run_shellcheck() { local file="$1" if [ "$HAS_SHELLCHECK" = "true" ]; then - timeout "$SANDBOX_TIMEOUT_S" y-shellcheck --severity=error "$file" >/dev/null 2>&1 + timeout --kill-after=2 "$SANDBOX_TIMEOUT_S" y-shellcheck --severity=error "$file" >/dev/null 2>&1 else return 0 # skip, not a failure fi @@ -222,7 +222,7 @@ run_sandboxed_help() { local exit_code=0 # Build sandbox command - local sandbox_cmd=(env -i PATH="$PATH" HOME="$tmpdir" timeout "$SANDBOX_TIMEOUT_S") + local sandbox_cmd=(env -i PATH="$PATH" HOME="$tmpdir" timeout --kill-after=2 "$SANDBOX_TIMEOUT_S") if [ "$HAS_UNSHARE" = "true" ]; then sandbox_cmd=(unshare --net "${sandbox_cmd[@]}") fi From a98a338dc3b8962124f658744b33f59cfc476de4 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 09:42:19 +0000 Subject: [PATCH 11/28] document YHELP variable and help subcommand pattern Rewrite Y_SCRIPT_AUTHORING.md: - Lead with quick reference showing compliant bash/node/typescript templates - Encourage help as subcommand (first arg) over --help flag - Introduce YHELP variable convention for static help text extraction - Document y-script-lint checks table with FAIL vs WARN levels - Remove execution-dependent indexing, describe static-only strategy - Drop -h from y-script-lint's own interface, use help subcommand Co-Authored-By: Claude Opus 4.6 (1M context) --- Y_SCRIPT_AUTHORING.md | 495 +++++++++++++++++------------------------- bin/y-script-lint | 20 +- 2 files changed, 206 insertions(+), 309 deletions(-) diff --git a/Y_SCRIPT_AUTHORING.md b/Y_SCRIPT_AUTHORING.md index 04961565..54752592 100644 --- a/Y_SCRIPT_AUTHORING.md +++ b/Y_SCRIPT_AUTHORING.md @@ -4,36 +4,21 @@ Scripts in `bin/` follow the `y-` prefix convention for PATH discoverability via This guide covers conventions for writing new scripts and improving existing ones, with emphasis on making scripts useful to both humans and AI agents. -The y-* convention spans multiple repos (ystack ~90 scripts, checkit ~152 scripts, bots ~10 scripts). +The y-* convention spans multiple repositories. Each repo's `bin/` is added to PATH, and scripts can call each other across repos. -## General Conventions +## Quick Reference: Making a Compliant Script -### Header +`y-script-lint` validates scripts statically (no execution). To pass all checks: -Every script must start with a shebang and standard preamble: +### Bash ```bash #!/usr/bin/env bash [ -z "$DEBUG" ] || set -x set -eo pipefail -``` - -Use `#!/bin/sh` only for POSIX-portable scripts with no bashisms. - -### Help Text (Required) - -Every script MUST support `--help` / `-h` and print a structured help block. -The **first line** of help output is used by index scripts for discovery, -so it must be a concise one-line summary of what the script does. -**Migration note**: Existing scripts use inconsistent help triggers: -`[ "$1" = "help" ]`, `--help`, or regex `[[ "$1" =~ help$ ]]`. -New scripts MUST use `--help` / `-h`. When touching existing scripts, migrate to `--help`/`-h` -while keeping `help` (no dashes) as a fallback for backwards compatibility. - -``` -y-example - Bump image tags in kustomization files +YHELP='y-example - Bump image tags in kustomization files Usage: y-example IMAGE TAG PATH [--dry-run] @@ -44,7 +29,6 @@ Arguments: Options: --dry-run Show what would change without writing - --help Show this help Environment: REGISTRY Override default registry (default: docker.io) @@ -57,17 +41,166 @@ Exit codes: 0 Success 1 Missing or invalid arguments 2 Image not found in registry +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +# Validate +[ -z "$1" ] && echo "First arg must be an image like yolean/node-kafka" >&2 && exit 1 + +# Main logic here +``` + +### Node.js + +```javascript +#!/usr/bin/env node + +const YHELP = `y-example - One line description + +Usage: y-example [options] + +Options: + --output json Output as JSON (default: human-readable) + +Environment: + MY_VAR Description (default: value) + +Dependencies: + y-crane Used to resolve digests (via shell) +`; + +if (process.argv[2] === 'help' || process.argv[2] === '--help') { + console.log(YHELP.trim()); + process.exit(0); +} + +// Main logic here +``` + +### TypeScript + +```typescript +#!/usr/bin/env -S node --experimental-strip-types + +const YHELP = `y-example - One line description +...same structure as Node.js... +`; + +if (process.argv[2] === 'help' || process.argv[2] === '--help') { + console.log(YHELP.trim()); + process.exit(0); +} +``` + +### What y-script-lint checks (all static, never executes your script) + +| Check | FAIL or WARN | Shell | Node.js | How to pass | +|-------|-------------|-------|---------|-------------| +| Shebang | FAIL | `#!/usr/bin/env bash` or `#!/bin/sh` | `#!/usr/bin/env node` or `*-strip-types` | First line | +| `set -eo pipefail` | FAIL | Required | n/a | Second or third line | +| DEBUG pattern | WARN | `[ -z "$DEBUG" ] \|\| set -x` | n/a | Second line | +| Help handler | WARN | `"$1" = "help"` in case/if | `process.argv` includes `help` | See templates above | +| No `npx` | FAIL | Not in non-comment lines | Not in non-comment lines | Use project deps | +| No `eval` | FAIL | Not in non-comment lines | No `eval(` calls | Avoid eval | +| shellcheck | FAIL | `--severity=error` | n/a | Fix shellcheck errors | + +### The help text format + +``` +y-name - One sentence summary + +Usage: y-name [options] ARGS + +Arguments: + ... + +Options: + ... + +Environment: + ... + +Dependencies: + y-other Why it's needed + jq JSON processing + +Exit codes: + 0 Success + 1 Usage error +``` + +**Rules:** +- First line: `y-name - description` (the "index line", used by tooling for discovery) +- `Dependencies:` section lists every y-* command and external tool the script calls +- Print help to stdout, exit 0 +- Keep it factual and compact — no ASCII art, no long examples + +### The `help` subcommand pattern + +New scripts use `help` as a positional subcommand (first argument): + +```bash +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; # backwards compat +esac +``` + +```javascript +if (process.argv[2] === 'help' || process.argv[2] === '--help') { +``` + +This is preferred over `--help`-only because: +- It reads naturally: `y-crane help`, `y-cluster-provision-k3d help` +- It follows the subcommand pattern used by git, docker, kubectl +- The help check happens **before** argument parsing, before prerequisite checks, + before any side effects — so it always works, even in a sandboxed env +- `--help` is accepted for backwards compatibility but not documented in new scripts + +**Existing scripts** use various patterns (`--help`, `-h|--help`, `[[ "$1" =~ help$ ]]`). +`y-script-lint` detects all of these. When touching an existing script, migrate to the +`help` subcommand pattern. + +### The YHELP variable pattern + +Store help text in a variable named `YHELP` (bash) or `YHELP` (JS/TS const): + +```bash +YHELP='y-name - description +... +' +``` + +```javascript +const YHELP = `y-name - description +... +`; +``` + +This enables `y-script-lint` to extract the summary and dependencies by reading +the source file — no execution required. The variable name `YHELP` is the convention +that tooling looks for. + +## General Conventions + +### Header + +Every shell script must start with a shebang and standard preamble: + +```bash +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail ``` -Key rules: -- First line: `y-name - One sentence description` (this is the "index line") -- List all positional arguments with descriptions -- List all flags/options -- List environment variables that affect behavior -- List y-* dependencies so tooling can build a dependency graph -- List non-obvious exit codes -- Print help to stdout (not stderr) so it can be piped/parsed -- Exit 0 after printing help +Use `#!/bin/sh` only for POSIX-portable scripts with no bashisms. + +Node.js scripts use `#!/usr/bin/env node`. +TypeScript scripts use `#!/usr/bin/env -S node --experimental-strip-types` (Node 22.6+). ### Structured Output for Agents @@ -82,13 +215,10 @@ else fi ``` -Guidelines from [Rewrite your CLI for AI Agents](https://justin.poehnelt.com/posts/rewrite-your-cli-for-ai-agents/): - -- **Token efficiency**: Keep output minimal. Agents pay per token of context consumed. Avoid banners, decorative output, progress bars, or repeated information. -- **Field selection**: For commands that return rich data, consider `--fields` to let callers request only what they need. -- **Dry run**: Mutating commands should support `--dry-run` to let agents validate before acting. -- **Deterministic output**: Same inputs should produce same output format. Don't mix human commentary into structured output. -- **Error output to stderr**: Human-readable messages go to stderr. Machine-parseable results go to stdout. This lets agents reliably capture output while still showing errors. +- **Token efficiency**: Keep output minimal. Avoid banners, progress bars, repeated info. +- **Dry run**: Mutating commands should support `--dry-run`. +- **Deterministic output**: Same inputs produce same output format. +- **Error output to stderr**: `echo "ERROR: message" >&2`. Structured results go to stdout. ### Naming @@ -96,16 +226,19 @@ Guidelines from [Rewrite your CLI for AI Agents](https://justin.poehnelt.com/pos - Hierarchical naming: `y-{domain}-{action}` (e.g. `y-cluster-provision-k3d`, `y-image-bump`) - Wrapper scripts for binaries: `y-{toolname}` (e.g. `y-crane`, `y-kubectl`) - Keep names descriptive enough that tab-completing `y-cluster-` reveals related commands -- Consistent suffixes for CLI wrappers that wrap versioned binaries: `y-{tool}-v{version}-bin` ### Argument Parsing -Use `--flag=value` style for named parameters. Parse with a while loop: +Use `--flag=value` style for named parameters. Handle help first, then flags: ```bash +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + while [ $# -gt 0 ]; do case "$1" in - -h|--help) show_help; exit 0 ;; --context=*) CTX="${1#*=}"; shift ;; --dry-run) DRY_RUN=true; shift ;; --) shift; break ;; @@ -123,8 +256,6 @@ Validate required arguments early with clear error messages: ### Exit Codes -Reserve ranges for consistent meaning across all y-* scripts: - | Range | Meaning | Example | |-------|---------|---------| | 0 | Success | | @@ -133,8 +264,6 @@ Reserve ranges for consistent meaning across all y-* scripts: | 10-19 | Precondition failures | Wrong tool version, missing prerequisite | | 90-99 | Convention/policy violations | Used by checkit for monorepo policy checks | -Document non-zero exit codes in the `Exit codes:` section of `--help`. - ### Error Handling - Use `set -eo pipefail` always @@ -145,76 +274,12 @@ Document non-zero exit codes in the `Exit codes:` section of `--help`. ### Environment Variables -- Document all env vars in `--help` output +- Document all env vars in help text - Provide sensible defaults: `[ -z "$REGISTRY" ] && REGISTRY="docker.io"` - Use `DEBUG` for `set -x` tracing (convention: `[ -z "$DEBUG" ] || set -x`) -- Use `DEBUGDEBUG` for extra verbose output in complex scripts - Never require secrets as positional args; use env vars or files -### Dependencies Section in Help - -To enable automated dependency discovery, include a `Dependencies:` section in help output listing every `y-*` command and external tool the script calls: - -``` -Dependencies: - y-crane Resolve image digests - y-buildctl Run buildkit builds - yq YAML processing - jq JSON processing - git Version control queries -``` - -This lets an indexer script parse `--help` output to build a dependency graph across all `y-*` scripts. - -## Bash-Specific Conventions - -### Script Template - -```bash -#!/usr/bin/env bash -[ -z "$DEBUG" ] || set -x -set -eo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -show_help() { - cat <<'EOF' -y-example - One line description - -Usage: y-example [options] ARG - -Arguments: - ARG Description of argument - -Options: - --dry-run Preview changes without applying - -h,--help Show this help - -Environment: - MY_VAR Description (default: value) - -Dependencies: - y-crane Used to resolve digests -EOF -} - -while [ $# -gt 0 ]; do - case "$1" in - -h|--help) show_help; exit 0 ;; - --dry-run) DRY_RUN=true; shift ;; - --) shift; break ;; - -*) echo "Unknown flag: $1" >&2; exit 1 ;; - *) break ;; - esac -done - -# Validate -[ -z "$1" ] && echo "ARG is required. See --help" >&2 && exit 1 - -# Main logic here -``` - -### Shell Practices +## Shell Practices - Quote variables: `"$VAR"` not `$VAR` (shellcheck will catch this) - Use `$(command)` not backticks @@ -225,50 +290,17 @@ done ### Shellcheck -All bash scripts in `bin/` are linted by `test.sh` using `y-shellcheck`. -Current minimum severity: `error`. Fix all shellcheck warnings at the `warning` level or above. +All shell scripts in `bin/` are linted by `y-script-lint` using `y-shellcheck`. +Current minimum severity: `error`. Use inline directives sparingly and with justification: ```bash # shellcheck disable=SC2086 # intentional word splitting for BUILDCTL_OPTS ``` -## Node.js-Specific Conventions - -### Script Template - -Use TypeScript with `--experimental-strip-types` (Node 22.6+): - -```typescript -#!/usr/bin/env -S node --experimental-strip-types - -const HELP = `y-example - One line description - -Usage: y-example [options] - -Options: - --output json Output as JSON (default: human-readable) - -h, --help Show this help - -Environment: - MY_VAR Description (default: value) - -Dependencies: - y-crane Used to resolve digests (via shell) -`; - -if (process.argv.includes('--help') || process.argv.includes('-h')) { - console.log(HELP.trim()); - process.exit(0); -} - -// Main logic here -``` - -### Node.js Practices +## Node.js Practices - Use project dependencies only (never `npx`) -- Import types from typed packages (e.g. `@octokit/graphql-schema`) - Exit with appropriate codes: `process.exit(1)` for errors - Write errors to stderr: `console.error("ERROR: ...")` - For JSON output, use `JSON.stringify(result, null, 2)` for human mode, compact for `--output json` @@ -287,182 +319,47 @@ Ystack and bots scripts call y-* commands via PATH, allowing cross-repo invocati This is more flexible but makes dependency tracing harder — the indexer must scan all repos' bin/ directories to resolve a dependency. -### Recommendation - Prefer PATH-based calls for cross-repo dependencies (e.g. checkit calling `y-crane` from ystack). Use `$YBIN/` only when you specifically need to call a sibling script and want to avoid PATH ambiguity. -### Agent Output Modes - -Checkit's `y-test-fast` already supports `Y_BUILD_OUTPUT_MODE=agents` for reduced output. -Standardize this pattern: scripts with verbose human output should check for an env var -that switches to minimal, machine-parseable output. - ## Discoverability and Indexing -### How index.sh Works +### How y-script-lint discovers help text -The `index.sh` pattern (piloted in bots) iterates over `bin/*`, calls `--help` on each script, and displays the first line. This relies on: +`y-script-lint` reads script source files and extracts information statically: -1. Every script supporting `--help` -2. The first line of help being a meaningful summary -3. Help printing to stdout +1. **Help handler detection**: Greps for known patterns (`"$1" = "help"`, `--help` in case, + `process.argv.includes`). Detects existing scripts regardless of which pattern they use. +2. **Summary and dependencies** (planned): Parse the `YHELP` variable from source to extract + the first line (index line) and `Dependencies:` section without executing the script. + +Scripts are never executed during lint. This means: +- Lint is fast (~0.2s per script) +- No side effects, no network requests, no prerequisite checks +- Works on any OS without sandbox concerns +- Works on scripts that require specific env (KUBECONFIG, docker, etc.) ### Dependency Graph Discovery Two complementary approaches: -**1. Declared dependencies (preferred for new scripts):** -Parse the `Dependencies:` section from each script's `--help` output. +**1. Declared dependencies (preferred):** +Parse the `Dependencies:` section from the `YHELP` variable in source. **2. Static analysis (works on existing scripts without changes):** Grep script source for `y-*` invocations. For checkit-style `$YBIN/y-*` calls this is straightforward. For PATH-based calls, look for `y-[a-z]` patterns that aren't inside comments or strings. -Both approaches enable: - -- Visualizing which scripts depend on which -- Detecting circular dependencies -- Understanding the blast radius of changes to foundational scripts like `y-crane` -- Helping agents understand which scripts to use together -- Identifying cross-repo dependency chains (e.g. checkit → ystack → binary) - ### Agent-Oriented Conventions When agents use y-* scripts, token cost and parse reliability matter: -1. **Concise help**: Keep `--help` output factual and compact. No ASCII art, no examples longer than one line each. +1. **Concise help**: Keep help text factual and compact. 2. **Structured errors**: Prefix errors with `ERROR:` so agents can regex-match failures. -3. **JSON mode**: For commands that list or query, support `--output json` to avoid parsing human-formatted tables. +3. **JSON mode**: For commands that list or query, support `--output json`. 4. **Idempotency**: Where possible, make scripts safe to re-run. Document when a script is NOT idempotent. -5. **Predictable exit codes**: 0 = success, 1 = usage error, 2+ = domain-specific. Document in help. +5. **Predictable exit codes**: 0 = success, 1 = usage error, 2+ = domain-specific. 6. **No interactive prompts**: Never prompt for input. Use flags, env vars, or fail with a clear message. -7. **Minimal side effects**: Scripts should do one thing. If a script has modes that do very different things, consider splitting it. - -## Tooling for Style and Security Checks - -### Current: shellcheck via test.sh - -`test.sh` runs `y-shellcheck --severity=error` on all shell scripts in `bin/`. - -### Recommended Additions - -Place these in ystack as scripts or CI steps: - -#### y-script-lint (slow, runs in CI or on demand) - -Scans a repo's `bin/` directory, executes `--help` on each y-* script, -and validates convention alignment. Writes results to a dotfile -that `y-script-index` can read without re-executing anything. - -**Two-phase approach — static checks never execute the script:** - -Phase 1 (static, safe): Read the script source and check: -- Shebang present and recognized (`bash`, `sh`, `node`) -- Standard header (`set -eo pipefail`, DEBUG pattern) -- Contains a help handler (recognizable `--help` / `-h` in a case/if pattern) -- No use of `npx` -- No `eval` on user input - -Scripts that fail phase 1 are recorded with `"help": false` and **never executed**. - -Phase 2 (sandboxed execution): For scripts that pass static screening, -run `--help` in a sandbox to extract the summary and declared dependencies: - -```bash -env -i \ - PATH="$PATH" \ - HOME="$(mktemp -d)" \ - timeout 5 \ - /bin/bash -c 'cd "$(mktemp -d)" && exec "$@"' _ "$script" --help -``` - -Sandbox properties: -- **Stripped env**: `env -i` clears credentials, cloud auth, KUBECONFIG etc. - Only PATH is preserved (needed to resolve y-* dependencies in help code). -- **Temp HOME**: Prevents reading credentials, ssh keys, cloud config files. -- **Temp working dir**: Prevents writes to the repo. -- **Timeout 5s**: Kills scripts that hang or do real work on `--help`. -- **No network** (Linux bots): Wrap with `unshare --net` where available. - -**Checks from the sandboxed run:** -- Exits 0 and produces output to stdout -- First help line matches `y-name - description` format -- `Dependencies:` section exists in help output - -**Output:** Writes `.y-script-lint.json` in the repo's `bin/` directory: - -```json -{ - "generated": "2026-03-14T12:00:00Z", - "scripts": { - "y-crane": { - "summary": "Crane binary wrapper for container image operations", - "dependencies": ["y-bin-download"], - "checks": {"help": true, "header": true, "deps_declared": true} - }, - "y-build": { - "summary": null, - "dependencies": [], - "checks": {"help": false, "header": true, "deps_declared": false}, - "errors": ["no --help support", "no Dependencies section"] - } - } -} -``` - -Scripts that fail the `help` check are included with `"summary": null` -so the lint result doubles as a backfill TODO list. - -The dotfile should be gitignored — it's a local cache, not source of truth. - -#### y-script-index (fast, safe for agents) - -Reads `.y-script-lint.json` from one or more repo `bin/` directories. -**Does not execute any scripts.** Only lists scripts that passed lint checks. - -```bash -# Human-friendly table (default) -y-script-index - # Last lint: 2026-03-14T12:00:00Z (2 hours ago) - # 187/252 scripts passed, 65 need --help backfill - y-crane Crane binary wrapper for container image operations - y-cluster-provision-k3d Provision a k3d cluster with ystack defaults - ... - -# JSON for agents -y-script-index --output json - -# Dependency graph (dot format, pipe to graphviz) -y-script-index --deps -y-script-index --deps --output dot | dot -Tsvg -o deps.svg - -# Filter by domain -y-script-index --filter cluster - -# Show stale lint warning (e.g. if scripts changed since last lint) -y-script-index - # Last lint: 2026-03-12T09:00:00Z (2 days ago) - # WARNING: 3 scripts modified since last lint run. Run y-script-lint to update. -``` - -The header always shows when lint was last run and how many scripts passed. -If any script's mtime is newer than the dotfile, a warning is printed. -This lets developers and agents judge how trustworthy the index is. - -Because it only reads a dotfile, agents can call `y-script-index` freely -to discover available tools without risk of side effects or slow execution. - -**Multi-repo support:** By default, scans `bin/.y-script-lint.json` in known -repo roots (ystack, checkit, bots). Accepts `--path` to override. - -#### Security checks (extend test.sh or run separately) - -- `shellcheck --severity=warning` (raise the bar from current `error`) -- Check for unquoted variables in file paths -- Check for `eval` usage (flag for review) -- Check that secrets come from env vars or files, never positional args -- Verify `trap cleanup EXIT` when `mktemp` is used -- osv-scanner for Node.js dependency vulnerabilities (already in y-bin.optional.yaml) +7. **Minimal side effects**: Scripts should do one thing. diff --git a/bin/y-script-lint b/bin/y-script-lint index 2faadf42..88127c63 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -7,11 +7,9 @@ trap 'trap - INT; kill -INT -$$' INT SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -show_help() { - cat <<'EOF' -y-script-lint - Static analysis and convention checks for y-* scripts +YHELP='y-script-lint - Static analysis and convention checks for y-* scripts -Usage: y-script-lint [options] [DIR...] +Usage: y-script-lint [help] [options] [DIR...] Arguments: DIR Directory to scan (default: all PATH dirs containing y-* scripts) @@ -20,7 +18,6 @@ Options: --check Exit non-zero if any script fails static checks (for CI) --phase1-only Skip sandboxed --help execution (phase 2) --json Write results to DIR/.y-script-lint.json - -h, --help Show this help Phase 1 (static, never executes scripts): - Shebang detection and language classification @@ -32,13 +29,13 @@ Phase 1 (static, never executes scripts): Phase 2 (sandboxed --help execution): - Only runs for scripts where phase 1 found a help handler - - Sandbox: env -i, temp HOME, temp workdir, timeout 5s + - Sandbox: env -i, temp HOME, temp workdir, timeout - Linux: network isolation via unshare --net - TODO: macOS sandboxing (currently skips phase 2 on Darwin) - Validates help output format: first line, Dependencies section Environment: - Y_SCRIPT_LINT_TIMEOUT_S Phase 2 sandbox timeout in seconds (default: 5) + Y_SCRIPT_LINT_TIMEOUT_S Sandbox timeout in seconds (default: 5) Dependencies: jq Required for --json output @@ -49,8 +46,12 @@ Exit codes: 0 All scripts passed (or --check not set) 1 Usage error 2 One or more scripts failed checks (--check mode) -EOF -} +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac CHECK_MODE=false PHASE1_ONLY=false @@ -59,7 +60,6 @@ TARGET_DIRS=() while [ $# -gt 0 ]; do case "$1" in - -h|--help) show_help; exit 0 ;; --check) CHECK_MODE=true; shift ;; --phase1-only) PHASE1_ONLY=true; shift ;; --json) JSON_OUTPUT=true; shift ;; From 46803277a5bfb7fd13e7239e12f527b5ecdd7058 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 10:07:50 +0000 Subject: [PATCH 12/28] add OS compatibility section to authoring guide macOS 14.8+, Debian Trixie+, Ubuntu 24.04+. Documents BSD vs GNU userland incompatibilities (sed -i, stat, etc.) Co-Authored-By: Claude Opus 4.6 (1M context) --- Y_SCRIPT_AUTHORING.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Y_SCRIPT_AUTHORING.md b/Y_SCRIPT_AUTHORING.md index 54752592..dc7ff295 100644 --- a/Y_SCRIPT_AUTHORING.md +++ b/Y_SCRIPT_AUTHORING.md @@ -7,6 +7,29 @@ with emphasis on making scripts useful to both humans and AI agents. The y-* convention spans multiple repositories. Each repo's `bin/` is added to PATH, and scripts can call each other across repos. +## OS Compatibility + +Scripts must work on: +- **macOS 14.8+** (Sonoma) — BSD userland, no GNU coreutils +- **Debian Trixie+** (13) +- **Ubuntu 24.04+** (Noble) + +macOS ships BSD versions of core utilities. Key incompatibilities: +- `sed -i ''` on macOS vs `sed -i` on Linux — avoid `sed -i` entirely, use a temp file or `ed` +- `stat -f %m` on macOS vs `stat -c %Y` on Linux +- `date` flag differences (`-d` vs `-v`) +- `readlink -f` unavailable on macOS without coreutils +- `grep -P` (PCRE) unavailable on macOS, use `grep -E` (extended regex) + +When a portable solution is impractical, use platform detection: + +```bash +case "$(uname -s)" in + Darwin) ;; # macOS-specific + Linux) ;; # Linux-specific +esac +``` + ## Quick Reference: Making a Compliant Script `y-script-lint` validates scripts statically (no execution). To pass all checks: From 98b16980e3a3ae76c48bc8ccf0de25232e649781 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 10:10:48 +0000 Subject: [PATCH 13/28] authoring guide: date compat, empty Dependencies, remove bold - date -Iseconds unavailable on macOS, document portable alternative - Dependencies: section left empty in examples (maintained by tooling) - Remove bold markdown styling throughout Co-Authored-By: Claude Opus 4.6 (1M context) --- Y_SCRIPT_AUTHORING.md | 52 ++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/Y_SCRIPT_AUTHORING.md b/Y_SCRIPT_AUTHORING.md index dc7ff295..dfba6093 100644 --- a/Y_SCRIPT_AUTHORING.md +++ b/Y_SCRIPT_AUTHORING.md @@ -10,14 +10,15 @@ Each repo's `bin/` is added to PATH, and scripts can call each other across repo ## OS Compatibility Scripts must work on: -- **macOS 14.8+** (Sonoma) — BSD userland, no GNU coreutils -- **Debian Trixie+** (13) -- **Ubuntu 24.04+** (Noble) +- macOS 14.8+ (Sonoma) — BSD userland, no GNU coreutils +- Debian Trixie+ (13) +- Ubuntu 24.04+ (Noble) macOS ships BSD versions of core utilities. Key incompatibilities: - `sed -i ''` on macOS vs `sed -i` on Linux — avoid `sed -i` entirely, use a temp file or `ed` - `stat -f %m` on macOS vs `stat -c %Y` on Linux -- `date` flag differences (`-d` vs `-v`) +- `date -Iseconds` (GNU ISO 8601) unavailable on macOS, use `date -u +%Y-%m-%dT%H:%M:%SZ` +- `date -d` (GNU parse) vs `date -v` (BSD adjust) - `readlink -f` unavailable on macOS without coreutils - `grep -P` (PCRE) unavailable on macOS, use `grep -E` (extended regex) @@ -57,8 +58,6 @@ Environment: REGISTRY Override default registry (default: docker.io) Dependencies: - y-crane Used to resolve image digests - yq Used for kustomization.yaml updates Exit codes: 0 Success @@ -93,7 +92,6 @@ Environment: MY_VAR Description (default: value) Dependencies: - y-crane Used to resolve digests (via shell) `; if (process.argv[2] === 'help' || process.argv[2] === '--help') { @@ -148,17 +146,15 @@ Environment: ... Dependencies: - y-other Why it's needed - jq JSON processing Exit codes: 0 Success 1 Usage error ``` -**Rules:** +Rules: - First line: `y-name - description` (the "index line", used by tooling for discovery) -- `Dependencies:` section lists every y-* command and external tool the script calls +- `Dependencies:` section is maintained by tooling — leave it empty in new scripts - Print help to stdout, exit 0 - Keep it factual and compact — no ASCII art, no long examples @@ -180,11 +176,11 @@ if (process.argv[2] === 'help' || process.argv[2] === '--help') { This is preferred over `--help`-only because: - It reads naturally: `y-crane help`, `y-cluster-provision-k3d help` - It follows the subcommand pattern used by git, docker, kubectl -- The help check happens **before** argument parsing, before prerequisite checks, +- The help check happens before argument parsing, before prerequisite checks, before any side effects — so it always works, even in a sandboxed env - `--help` is accepted for backwards compatibility but not documented in new scripts -**Existing scripts** use various patterns (`--help`, `-h|--help`, `[[ "$1" =~ help$ ]]`). +Existing scripts use various patterns (`--help`, `-h|--help`, `[[ "$1" =~ help$ ]]`). `y-script-lint` detects all of these. When touching an existing script, migrate to the `help` subcommand pattern. @@ -238,10 +234,10 @@ else fi ``` -- **Token efficiency**: Keep output minimal. Avoid banners, progress bars, repeated info. -- **Dry run**: Mutating commands should support `--dry-run`. -- **Deterministic output**: Same inputs produce same output format. -- **Error output to stderr**: `echo "ERROR: message" >&2`. Structured results go to stdout. +- Token efficiency: Keep output minimal. Avoid banners, progress bars, repeated info. +- Dry run: Mutating commands should support `--dry-run`. +- Deterministic output: Same inputs produce same output format. +- Error output to stderr: `echo "ERROR: message" >&2`. Structured results go to stdout. ### Naming @@ -352,9 +348,9 @@ and want to avoid PATH ambiguity. `y-script-lint` reads script source files and extracts information statically: -1. **Help handler detection**: Greps for known patterns (`"$1" = "help"`, `--help` in case, +1. Help handler detection: Greps for known patterns (`"$1" = "help"`, `--help` in case, `process.argv.includes`). Detects existing scripts regardless of which pattern they use. -2. **Summary and dependencies** (planned): Parse the `YHELP` variable from source to extract +2. Summary and dependencies (planned): Parse the `YHELP` variable from source to extract the first line (index line) and `Dependencies:` section without executing the script. Scripts are never executed during lint. This means: @@ -367,10 +363,10 @@ Scripts are never executed during lint. This means: Two complementary approaches: -**1. Declared dependencies (preferred):** +1. Declared dependencies (preferred): Parse the `Dependencies:` section from the `YHELP` variable in source. -**2. Static analysis (works on existing scripts without changes):** +2. Static analysis (works on existing scripts without changes): Grep script source for `y-*` invocations. For checkit-style `$YBIN/y-*` calls this is straightforward. For PATH-based calls, look for `y-[a-z]` patterns that aren't inside comments or strings. @@ -379,10 +375,10 @@ that aren't inside comments or strings. When agents use y-* scripts, token cost and parse reliability matter: -1. **Concise help**: Keep help text factual and compact. -2. **Structured errors**: Prefix errors with `ERROR:` so agents can regex-match failures. -3. **JSON mode**: For commands that list or query, support `--output json`. -4. **Idempotency**: Where possible, make scripts safe to re-run. Document when a script is NOT idempotent. -5. **Predictable exit codes**: 0 = success, 1 = usage error, 2+ = domain-specific. -6. **No interactive prompts**: Never prompt for input. Use flags, env vars, or fail with a clear message. -7. **Minimal side effects**: Scripts should do one thing. +1. Concise help: Keep help text factual and compact. +2. Structured errors: Prefix errors with `ERROR:` so agents can regex-match failures. +3. JSON mode: For commands that list or query, support `--output json`. +4. Idempotency: Where possible, make scripts safe to re-run. Document when a script is NOT idempotent. +5. Predictable exit codes: 0 = success, 1 = usage error, 2+ = domain-specific. +6. No interactive prompts: Never prompt for input. Use flags, env vars, or fail with a clear message. +7. Minimal side effects: Scripts should do one thing. From b98396a41f20b5e8d22fc4ff9e31efd70c2a27bd Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 10:23:57 +0000 Subject: [PATCH 14/28] remove phase 2 execution, add static YHELP parsing and --dependencies-add No script execution at all. All checks are source parsing: - YHELP variable parsed from bash (single-quoted) and node (template literal) - Summary extracted from first non-empty line of YHELP - Dependencies: section parsed from YHELP - y-* invocations detected by grepping source (comments excluded) --dependencies-add writes detected y-* deps into the YHELP Dependencies: section of scripts that already have a compliant section. Uses line-number insertion to preserve surrounding whitespace. Idempotent. JSON output adds detected_dependencies alongside declared dependencies. tool_version bumped to 2. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 554 +++++++++++++++------------------------------- 1 file changed, 181 insertions(+), 373 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 88127c63..7f3b6f1c 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -2,50 +2,33 @@ [ -z "$DEBUG" ] || set -x set -eo pipefail -# Ensure Ctrl+C kills the whole process group, including hung children trap 'trap - INT; kill -INT -$$' INT -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - YHELP='y-script-lint - Static analysis and convention checks for y-* scripts Usage: y-script-lint [help] [options] [DIR...] Arguments: - DIR Directory to scan (default: all PATH dirs containing y-* scripts) + DIR Directory to scan (default: all PATH dirs containing y-* scripts) Options: - --check Exit non-zero if any script fails static checks (for CI) - --phase1-only Skip sandboxed --help execution (phase 2) - --json Write results to DIR/.y-script-lint.json - -Phase 1 (static, never executes scripts): - - Shebang detection and language classification - - Header checks: set -eo pipefail, DEBUG pattern - - Help handler detection in source - - shellcheck (if y-shellcheck is available) - - No npx usage - - No unguarded eval - -Phase 2 (sandboxed --help execution): - - Only runs for scripts where phase 1 found a help handler - - Sandbox: env -i, temp HOME, temp workdir, timeout - - Linux: network isolation via unshare --net - - TODO: macOS sandboxing (currently skips phase 2 on Darwin) - - Validates help output format: first line, Dependencies section + --check Exit non-zero if any script fails static checks (for CI) + --json Write results to DIR/.y-script-lint.json + --dependencies-add Detect y-* invocations and add to Dependencies section in YHELP + +All checks are static (source parsing only, no script execution). Environment: - Y_SCRIPT_LINT_TIMEOUT_S Sandbox timeout in seconds (default: 5) + Y_SCRIPT_LINT_TIMEOUT_S Shellcheck timeout in seconds (default: 5) Dependencies: - jq Required for --json output - y-shellcheck Optional, shell lint (skipped if not available) - unshare Optional, Linux network sandbox for phase 2 + y-bin-download + y-shellcheck Exit codes: - 0 All scripts passed (or --check not set) - 1 Usage error - 2 One or more scripts failed checks (--check mode) + 0 All scripts passed (or --check not set) + 1 Usage error + 2 One or more scripts failed checks (--check mode) ' case "${1:-}" in @@ -54,27 +37,25 @@ case "${1:-}" in esac CHECK_MODE=false -PHASE1_ONLY=false JSON_OUTPUT=false +DEPS_ADD=false TARGET_DIRS=() while [ $# -gt 0 ]; do case "$1" in --check) CHECK_MODE=true; shift ;; - --phase1-only) PHASE1_ONLY=true; shift ;; --json) JSON_OUTPUT=true; shift ;; + --dependencies-add) DEPS_ADD=true; shift ;; --) shift; break ;; -*) echo "Unknown flag: $1" >&2; exit 1 ;; *) TARGET_DIRS+=("$1"); shift ;; esac done -# Default: discover from PATH if [ ${#TARGET_DIRS[@]} -eq 0 ]; then IFS=: read -ra path_entries <<< "$PATH" for dir in "${path_entries[@]}"; do [ -d "$dir" ] || continue - # Include directories that contain at least one y-* executable for f in "$dir"/y-*; do if [ -x "$f" ] && [ -f "$f" ]; then TARGET_DIRS+=("$dir") @@ -82,13 +63,9 @@ if [ ${#TARGET_DIRS[@]} -eq 0 ]; then fi done done - if [ ${#TARGET_DIRS[@]} -eq 0 ]; then - echo "ERROR: no y-* scripts found in PATH" >&2 - exit 1 - fi + [ ${#TARGET_DIRS[@]} -eq 0 ] && { echo "ERROR: no y-* scripts found in PATH" >&2; exit 1; } fi -# Resolve to absolute paths RESOLVED_DIRS=() for dir in "${TARGET_DIRS[@]}"; do [ -d "$dir" ] || { echo "ERROR: directory not found: $dir" >&2; exit 1; } @@ -98,10 +75,8 @@ done # --- Language detection --- detect_language() { - local file="$1" local shebang - shebang=$(head -1 "$file" 2>/dev/null) || true - + shebang=$(head -1 "$1" 2>/dev/null) || true case "$shebang" in '#!/usr/bin/env bash'|'#!/bin/bash') echo "bash" ;; '#!/bin/sh') echo "sh" ;; @@ -109,319 +84,226 @@ detect_language() { *node*) echo "node" ;; *python*) echo "python" ;; *y-bin-download*) echo "config" ;; - *) - # No recognized shebang - check extension - case "$file" in - *.js|*.ts) echo "library" ;; - *) echo "unknown" ;; - esac - ;; + *) case "$1" in *.js|*.ts) echo "library" ;; *) echo "unknown" ;; esac ;; esac } -is_shell() { - local lang="$1" - [ "$lang" = "bash" ] || [ "$lang" = "sh" ] -} +is_shell() { [ "$1" = "bash" ] || [ "$1" = "sh" ]; } +is_node() { [ "$1" = "node" ] || [ "$1" = "typescript" ]; } -is_node() { - local lang="$1" - [ "$lang" = "node" ] || [ "$lang" = "typescript" ] -} - -# --- Static checks (phase 1) --- - -check_header_pipefail() { - local file="$1" - grep -qE '^set -e(o pipefail)?$' "$file" 2>/dev/null -} +# --- Static checks --- -check_header_debug() { - local file="$1" - grep -qE '^\[ -z "\$DEBUG" \] \|\| set -x' "$file" 2>/dev/null -} +check_header_pipefail() { grep -qE '^set -e(o pipefail)?$' "$1" 2>/dev/null; } +check_header_debug() { grep -qE '^\[ -z "\$DEBUG" \] \|\| set -x' "$1" 2>/dev/null; } check_help_handler() { - local file="$1" - local lang="$2" - + local file="$1" lang="$2" if is_shell "$lang"; then - # Match: -h|--help, "$1" = "help", "$1" = "--help", =~ help grep -qE '(\-h\|--help|"\$1" = "help"|"\$1" = "--help"|\$1 =~ help)' "$file" 2>/dev/null elif is_node "$lang"; then - # Match: process.argv.includes('--help') or includes('-h') - grep -qE "process\.argv\.includes\(['\"](-h|--help)['\"]" "$file" 2>/dev/null + grep -qE "process\.argv\.includes\(['\"](-h|--help)['\"]" "$file" 2>/dev/null || + grep -qE "process\.argv\[2\] === ['\"]help['\"]" "$file" 2>/dev/null else return 1 fi } check_no_npx() { - local file="$1" - local lang="$2" + local file="$1" lang="$2" if is_node "$lang"; then - # JS/TS: ignore // comments ! grep -vE '^\s*//' "$file" 2>/dev/null | grep -qE '\bnpx\b' else - # Shell: ignore # comments and self-references ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_npx|"uses npx"|No npx)' | grep -qE '\bnpx\b' fi } check_no_eval() { - local file="$1" - local lang="$2" + local file="$1" lang="$2" if is_node "$lang"; then - # JS/TS: match eval( function call, ignore comments ! grep -vE '^\s*//' "$file" 2>/dev/null | grep -qE '\beval\s*\(' else - # Shell: match bare eval command, ignore comments and self-references ! grep -vE '^\s*#' "$file" 2>/dev/null | grep -vE '(no_eval|"uses eval"|No.*eval)' | grep -qE '\beval\b' fi } -# --- Shellcheck --- - +SHELLCHECK_TIMEOUT_S="${Y_SCRIPT_LINT_TIMEOUT_S:-5}" HAS_SHELLCHECK=false -if command -v y-shellcheck >/dev/null 2>&1; then - HAS_SHELLCHECK=true -fi +command -v y-shellcheck >/dev/null 2>&1 && HAS_SHELLCHECK=true run_shellcheck() { - local file="$1" - if [ "$HAS_SHELLCHECK" = "true" ]; then - timeout --kill-after=2 "$SANDBOX_TIMEOUT_S" y-shellcheck --severity=error "$file" >/dev/null 2>&1 - else - return 0 # skip, not a failure - fi + [ "$HAS_SHELLCHECK" = "true" ] && timeout --kill-after=2 "$SHELLCHECK_TIMEOUT_S" y-shellcheck --severity=error "$1" >/dev/null 2>&1 } -# --- Phase 2: sandboxed --help execution --- - -SANDBOX_TIMEOUT_S="${Y_SCRIPT_LINT_TIMEOUT_S:-5}" +# --- YHELP source parsing --- -CAN_SANDBOX=true -HAS_UNSHARE=false -if [ "$(uname -s)" = "Linux" ]; then - # Test if unshare --net actually works (needs user namespace or root) - if command -v unshare >/dev/null 2>&1 && unshare --net true 2>/dev/null; then - HAS_UNSHARE=true +parse_yhelp_from_source() { + local file="$1" lang="$2" + if is_shell "$lang"; then + awk " + /^YHELP='/ { + sub(/^YHELP='/, \"\") + if (/'$/) { sub(/'$/, \"\"); print; next } + print + while (getline > 0) { + if (/'$/) { sub(/'$/, \"\"); print; exit } + print + } + } + " "$file" + elif is_node "$lang"; then + awk ' + /const YHELP = `/ { + sub(/.*const YHELP = `/, "") + if (/`;/) { sub(/`;.*/, ""); print; next } + if (/`/) { sub(/`.*/, ""); print; exit } + print + while (getline > 0) { + if (/`;/) { sub(/`;.*/, ""); print; exit } + if (/`/) { sub(/`.*/, ""); print; exit } + print + } + } + ' "$file" fi -fi -if [ "$(uname -s)" = "Darwin" ]; then - # TODO: macOS sandbox using sandbox-exec or similar - CAN_SANDBOX=false -fi +} -run_sandboxed_help() { - local script="$1" - local tmpdir - tmpdir=$(mktemp -d) +parse_summary_from_yhelp() { echo "$1" | grep -m1 -vE '^\s*$'; } +parse_deps_from_yhelp() { echo "$1" | awk '/^Dependencies:/{f=1;next} f&&/^$/{exit} f&&/^[A-Z]/{exit} f{print}' | awk 'NF{print $1}' || true; } +yhelp_has_deps_section() { echo "$1" | grep -q '^Dependencies:' 2>/dev/null; } - local output="" - local exit_code=0 +# --- Static dependency detection --- - # Build sandbox command - local sandbox_cmd=(env -i PATH="$PATH" HOME="$tmpdir" timeout --kill-after=2 "$SANDBOX_TIMEOUT_S") - if [ "$HAS_UNSHARE" = "true" ]; then - sandbox_cmd=(unshare --net "${sandbox_cmd[@]}") +detect_y_invocations() { + local file="$1" lang="$2" self + self=$(basename "$file") + local content + if is_shell "$lang"; then + content=$(grep -vE '^\s*#' "$file" 2>/dev/null || true) + elif is_node "$lang"; then + content=$(grep -vE '^\s*//' "$file" 2>/dev/null || true) + else + content=$(cat "$file" 2>/dev/null || true) fi + echo "$content" | grep -oE '\by-[a-z][a-z0-9_-]*' | grep -v "^${self}$" | sort -u || true +} - # Capture both stdout and stderr — some scripts print help to stderr - output=$("${sandbox_cmd[@]}" /bin/bash -c 'cd "$(mktemp -d)" && exec "$@"' _ "$script" --help 2>&1) || exit_code=$? +# --- --dependencies-add --- - rm -rf "$tmpdir" +add_deps_to_file() { + local file="$1" lang="$2" detected_deps="$3" + [ -z "$detected_deps" ] && return - # Timeout (124) or signal kill (137) means the script hung - if [ $exit_code -eq 124 ] || [ $exit_code -eq 137 ]; then - return 1 - fi + local yhelp + yhelp=$(parse_yhelp_from_source "$file" "$lang") + [ -z "$yhelp" ] && return + yhelp_has_deps_section "$yhelp" || return - # Some scripts exit non-zero after printing help (e.g. prereq checks before help handler) - # Accept output even on non-zero exit if it looks like help text - if [ -n "$output" ]; then - echo "$output" - return 0 - fi - return 1 -} + local existing_deps + existing_deps=$(parse_deps_from_yhelp "$yhelp") -parse_help_summary() { - local output="$1" - local first_line - first_line=$(echo "$output" | head -1) - echo "$first_line" -} + local new_deps=() + while IFS= read -r dep; do + [ -z "$dep" ] && continue + echo "$existing_deps" | grep -qxF "$dep" || new_deps+=("$dep") + done <<< "$detected_deps" + [ ${#new_deps[@]} -eq 0 ] && return -check_help_format() { - local first_line="$1" - local script_name="$2" - # Expected: y-name - description - echo "$first_line" | grep -qE "^${script_name} - .+" 2>/dev/null -} + # Find the line number of Dependencies: in the file, insert after it + local deps_line + deps_line=$(grep -n '^Dependencies:' "$file" | head -1 | cut -d: -f1) + [ -z "$deps_line" ] && return -parse_help_dependencies() { - local output="$1" - # Extract lines after "Dependencies:" until the next blank line or section - echo "$output" | awk '/^Dependencies:/{found=1; next} found && /^$/{exit} found && /^[A-Z]/{exit} found{print}' | \ - awk '{print $1}' | grep -v '^$' || true -} + local tmpfile insert_file + tmpfile=$(mktemp) + insert_file=$(mktemp) + for dep in "${new_deps[@]}"; do echo " ${dep}" >> "$insert_file"; done + + head -n "$deps_line" "$file" > "$tmpfile" + cat "$insert_file" >> "$tmpfile" + tail -n +"$((deps_line + 1))" "$file" >> "$tmpfile" -check_help_has_deps_section() { - local output="$1" - echo "$output" | grep -q '^Dependencies:' 2>/dev/null + if ! cmp -s "$file" "$tmpfile"; then + cp "$tmpfile" "$file" + for dep in "${new_deps[@]}"; do echo " + $dep"; done + fi + rm -f "$tmpfile" "$insert_file" } # --- Per-file lint --- lint_file() { - local file="$1" - local name + local file="$1" name name=$(basename "$file") - lang=$(detect_language "$file") - # Skip non-script files (configs, libraries without shebangs) if [ "$lang" = "config" ] || [ "$lang" = "library" ]; then SKIPPED=$((SKIPPED + 1)) - if [ "$JSON_OUTPUT" = "true" ]; then - DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ - --arg name "$name" \ - --arg lang "$lang" \ - '{($name): {language: $lang, skipped: true}}' - )," - fi + [ "$JSON_OUTPUT" = "true" ] && DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ + --arg name "$name" --arg lang "$lang" '{($name): {language: $lang, skipped: true}}')," return fi - local loc + local loc errors=() loc=$(wc -l < "$file") + local checks_shebang=true checks_header=true checks_debug=true + local checks_help_handler=false checks_shellcheck=null + local checks_no_npx=true checks_no_eval=true + local checks_help_format=null checks_deps_declared=null + local summary=null deps_json="[]" detected_deps="" - local errors=() - local checks_shebang=true - local checks_header=true - local checks_debug=true - local checks_help_handler=false - local checks_shellcheck=null - local checks_no_npx=true - local checks_no_eval=true - local checks_help_runs=null - local checks_help_format=null - local checks_deps_declared=null - local summary=null - local deps_json="[]" - - # Unknown shebang - if [ "$lang" = "unknown" ]; then - checks_shebang=false - errors+=("unrecognized or missing shebang") - fi + [ "$lang" = "unknown" ] && { checks_shebang=false; errors+=("unrecognized or missing shebang"); } - # Header checks (shell scripts only) if is_shell "$lang"; then - if ! check_header_pipefail "$file"; then - checks_header=false - errors+=("missing set -eo pipefail or set -e") - fi - if ! check_header_debug "$file"; then - checks_debug=false - errors+=("missing DEBUG pattern: [ -z \"\\\$DEBUG\" ] || set -x") - fi + check_header_pipefail "$file" || { checks_header=false; errors+=("missing set -eo pipefail or set -e"); } + check_header_debug "$file" || { checks_debug=false; errors+=("missing DEBUG pattern: [ -z \"\\\$DEBUG\" ] || set -x"); } else - checks_header=null - checks_debug=null + checks_header=null; checks_debug=null fi - # Help handler - if check_help_handler "$file" "$lang"; then - checks_help_handler=true - else - errors+=("no help handler found in source") - fi - - # npx check - if ! check_no_npx "$file" "$lang"; then - checks_no_npx=false - errors+=("uses npx") - fi + check_help_handler "$file" "$lang" && checks_help_handler=true || errors+=("no help handler found in source") + check_no_npx "$file" "$lang" || { checks_no_npx=false; errors+=("uses npx"); } + check_no_eval "$file" "$lang" || { checks_no_eval=false; errors+=("uses eval"); } - # eval check - if ! check_no_eval "$file" "$lang"; then - checks_no_eval=false - errors+=("uses eval") + if is_shell "$lang" && [ "$HAS_SHELLCHECK" = "true" ]; then + run_shellcheck "$file" && checks_shellcheck=true || { checks_shellcheck=false; errors+=("shellcheck --severity=error failed"); } fi - # Shellcheck (shell scripts only) - if is_shell "$lang"; then - if [ "$HAS_SHELLCHECK" = "true" ]; then - if run_shellcheck "$file"; then - checks_shellcheck=true - else - checks_shellcheck=false - errors+=("shellcheck --severity=error failed") - fi + # Parse YHELP from source + local yhelp + yhelp=$(parse_yhelp_from_source "$file" "$lang") + if [ -n "$yhelp" ]; then + local first_line + first_line=$(parse_summary_from_yhelp "$yhelp") + if echo "$first_line" | grep -qE "^${name} - .+"; then + checks_help_format=true; summary="$first_line" + else + checks_help_format=false; errors+=("YHELP first line does not match: $name - description") fi - fi - - # Phase 2: sandboxed --help - if [ "$PHASE1_ONLY" = "false" ] && [ "$checks_help_handler" = "true" ]; then - if [ "$CAN_SANDBOX" = "false" ]; then - checks_help_runs=null - checks_help_format=null - checks_deps_declared=null + if yhelp_has_deps_section "$yhelp"; then + checks_deps_declared=true + local declared_deps + declared_deps=$(parse_deps_from_yhelp "$yhelp") + [ -n "$declared_deps" ] && deps_json=$(echo "$declared_deps" | jq -R . | jq -s .) else - local help_output - help_output=$(run_sandboxed_help "$file") || true - if [ -n "$help_output" ]; then - checks_help_runs=true - local first_line - first_line=$(parse_help_summary "$help_output") - summary="$first_line" - - if check_help_format "$first_line" "$name"; then - checks_help_format=true - else - checks_help_format=false - errors+=("help first line does not match format: $name - description") - fi - - if check_help_has_deps_section "$help_output"; then - checks_deps_declared=true - local deps_raw - deps_raw=$(parse_help_dependencies "$help_output") - if [ -n "$deps_raw" ]; then - deps_json=$(echo "$deps_raw" | jq -R . | jq -s .) - else - deps_json="[]" - fi - else - checks_deps_declared=false - errors+=("no Dependencies section in help output") - fi - else - checks_help_runs=false - errors+=("--help produced no output or failed") - fi + checks_deps_declared=false; errors+=("YHELP missing Dependencies: section") fi fi - # Determine pass/fail - local has_static_failure=false - if [ "$checks_shebang" = "false" ] || [ "$checks_header" = "false" ] \ - || [ "$checks_no_npx" = "false" ] || [ "$checks_no_eval" = "false" ] \ - || [ "$checks_shellcheck" = "false" ]; then - has_static_failure=true - fi + detected_deps=$(detect_y_invocations "$file" "$lang") + [ "$DEPS_ADD" = "true" ] && [ -n "$detected_deps" ] && add_deps_to_file "$file" "$lang" "$detected_deps" - local has_help_failure=false - if [ "$checks_help_handler" = "false" ] || [ "$checks_help_runs" = "false" ] \ - || [ "$checks_help_format" = "false" ]; then - has_help_failure=true - fi + local detected_deps_json="[]" + [ -n "$detected_deps" ] && detected_deps_json=$(echo "$detected_deps" | jq -R . | jq -s .) + + # Pass/fail + local has_static_failure=false has_help_failure=false + { [ "$checks_shebang" = "false" ] || [ "$checks_header" = "false" ] \ + || [ "$checks_no_npx" = "false" ] || [ "$checks_no_eval" = "false" ] \ + || [ "$checks_shellcheck" = "false" ]; } && has_static_failure=true + { [ "$checks_help_handler" = "false" ] || [ "$checks_help_format" = "false" ]; } && has_help_failure=true if [ "$has_static_failure" = "true" ]; then - FAILED_STATIC=$((FAILED_STATIC + 1)) - FAIL_LIST="$FAIL_LIST $name" + FAILED_STATIC=$((FAILED_STATIC + 1)); FAIL_LIST="$FAIL_LIST $name" elif [ "$has_help_failure" = "true" ]; then FAILED_HELP=$((FAILED_HELP + 1)) else @@ -430,7 +312,6 @@ lint_file() { echo " $name $lang ${loc}L" for err in "${errors[@]}"; do - # Static failures are FAIL, help-related are WARN local level="WARN" case "$err" in "unrecognized or missing shebang"|"missing set -eo pipefail"*|"uses npx"|"uses eval"|"shellcheck "*) level="FAIL" ;; @@ -438,142 +319,69 @@ lint_file() { echo " $level $err" done - # Build JSON if [ "$JSON_OUTPUT" = "true" ]; then - local errors_json="[]" - if [ ${#errors[@]} -gt 0 ]; then - errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) - fi - - local summary_json="null" - if [ "$summary" != "null" ] && [ -n "$summary" ]; then - summary_json=$(echo "$summary" | jq -R .) - fi - + local errors_json="[]" summary_json="null" + [ ${#errors[@]} -gt 0 ] && errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) + [ "$summary" != "null" ] && [ -n "$summary" ] && summary_json=$(echo "$summary" | jq -R .) local checks_json checks_json=$(jq -n \ - --argjson shebang "$checks_shebang" \ - --argjson header "$checks_header" \ - --argjson debug "$checks_debug" \ - --argjson help_handler "$checks_help_handler" \ - --argjson shellcheck "$checks_shellcheck" \ - --argjson no_npx "$checks_no_npx" \ - --argjson no_eval "$checks_no_eval" \ - --argjson help_runs "$checks_help_runs" \ - --argjson help_format "$checks_help_format" \ + --argjson shebang "$checks_shebang" --argjson header "$checks_header" \ + --argjson debug "$checks_debug" --argjson help_handler "$checks_help_handler" \ + --argjson shellcheck "$checks_shellcheck" --argjson no_npx "$checks_no_npx" \ + --argjson no_eval "$checks_no_eval" --argjson help_format "$checks_help_format" \ --argjson deps_declared "$checks_deps_declared" \ - '{ - shebang: $shebang, - header: $header, - debug: $debug, - help_handler: $help_handler, - shellcheck: $shellcheck, - no_npx: $no_npx, - no_eval: $no_eval, - help_runs: $help_runs, - help_format: $help_format, - deps_declared: $deps_declared - }') - - local script_json - script_json=$(jq -n \ - --arg lang "$lang" \ - --argjson summary "$summary_json" \ - --argjson deps "$deps_json" \ - --argjson checks "$checks_json" \ - --argjson errors "$errors_json" \ - '{ - language: $lang, - summary: $summary, - dependencies: $deps, - checks: $checks, - errors: $errors - }') - - DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n --arg name "$name" --argjson data "$script_json" '{($name): $data}')," + '{shebang:$shebang,header:$header,debug:$debug,help_handler:$help_handler, + shellcheck:$shellcheck,no_npx:$no_npx,no_eval:$no_eval, + help_format:$help_format,deps_declared:$deps_declared}') + DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ + --arg name "$name" --arg lang "$lang" --argjson summary "$summary_json" \ + --argjson deps "$deps_json" --argjson detected "$detected_deps_json" \ + --argjson checks "$checks_json" --argjson errors "$errors_json" \ + '{($name):{language:$lang,summary:$summary,dependencies:$deps, + detected_dependencies:$detected,checks:$checks,errors:$errors}}')," fi } # --- Main --- -TOTAL=0 -PASSED=0 -FAILED_STATIC=0 -FAILED_HELP=0 -SKIPPED=0 +TOTAL=0 PASSED=0 FAILED_STATIC=0 FAILED_HELP=0 SKIPPED=0 FAIL_LIST="" - -# Track seen script names to skip PATH duplicates declare -A SEEN_SCRIPTS JSON_FILES=() for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do - # Collect y-* scripts in this directory dir_scripts=() for file in "$TARGET_DIR"/y-*; do [ -f "$file" ] && [ -x "$file" ] || continue name=$(basename "$file") - case "$name" in - *.yaml|*.yml|*.md) continue ;; - *.spec.js|*.spec.ts|*.test.js|*.test.ts) continue ;; - *-bin) continue ;; - *-dist) continue ;; - esac - # Skip duplicates (first PATH entry wins) + case "$name" in *.yaml|*.yml|*.md|*.spec.js|*.spec.ts|*.test.js|*.test.ts|*-bin|*-dist) continue ;; esac [ -n "${SEEN_SCRIPTS[$name]:-}" ] && continue SEEN_SCRIPTS[$name]=1 dir_scripts+=("$file") done - [ ${#dir_scripts[@]} -eq 0 ] && continue echo "[y-script-lint] $TARGET_DIR (${#dir_scripts[@]} scripts)" - DIR_JSON_SCRIPTS="" - for file in "${dir_scripts[@]}"; do TOTAL=$((TOTAL + 1)) lint_file "$file" done - # Write per-directory JSON if [ "$JSON_OUTPUT" = "true" ] && [ -n "$DIR_JSON_SCRIPTS" ]; then - command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required for --json output" >&2; exit 1; } - - local_scripts_merged=$(echo "[${DIR_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)') - + command -v jq >/dev/null 2>&1 || { echo "ERROR: jq required for --json" >&2; exit 1; } JSON_FILE="$TARGET_DIR/.y-script-lint.json" - jq -n \ - --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --arg tool_version "1" \ - --arg directory "$TARGET_DIR" \ - --argjson scripts "$local_scripts_merged" \ - '{ - generated: $generated, - tool_version: $tool_version, - directory: $directory, - scripts: $scripts - }' > "$JSON_FILE" - + echo "[${DIR_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ + jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "2" \ + --arg directory "$TARGET_DIR" --argjson scripts "$(cat)" \ + '{generated:$generated,tool_version:$tool_version,directory:$directory,scripts:$scripts}' > "$JSON_FILE" JSON_FILES+=("$JSON_FILE") fi done NOTES="" -if [ "$HAS_SHELLCHECK" = "false" ]; then - NOTES="${NOTES}, no shellcheck" -fi -if [ "$CAN_SANDBOX" = "false" ]; then - NOTES="${NOTES}, no macOS sandbox" -fi - +[ "$HAS_SHELLCHECK" = "false" ] && NOTES="${NOTES}, no shellcheck" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" -for jf in "${JSON_FILES[@]}"; do - echo "Wrote $jf" -done +for jf in "${JSON_FILES[@]}"; do echo "Wrote $jf"; done -# --check mode: exit non-zero if any static failures -if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then - echo "FAILED:$FAIL_LIST" - exit 2 -fi +[ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ] && { echo "FAILED:$FAIL_LIST"; exit 2; } From 5efef0d895b5d3d1f51d7b3c431a1b09c1870fd3 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 11:43:15 +0000 Subject: [PATCH 15/28] always write index, fix exit code with set -e Index (.y-script-lint.json) is now always written when jq is available, not gated on --json flag. The --json option is removed. Fix: last line used && chain which caused exit 1 under set -e when --check was not set. Replaced with if/then. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 7f3b6f1c..6e642740 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -13,7 +13,6 @@ Arguments: Options: --check Exit non-zero if any script fails static checks (for CI) - --json Write results to DIR/.y-script-lint.json --dependencies-add Detect y-* invocations and add to Dependencies section in YHELP All checks are static (source parsing only, no script execution). @@ -37,14 +36,13 @@ case "${1:-}" in esac CHECK_MODE=false -JSON_OUTPUT=false DEPS_ADD=false +JSON_OUTPUT=true TARGET_DIRS=() while [ $# -gt 0 ]; do case "$1" in --check) CHECK_MODE=true; shift ;; - --json) JSON_OUTPUT=true; shift ;; --dependencies-add) DEPS_ADD=true; shift ;; --) shift; break ;; -*) echo "Unknown flag: $1" >&2; exit 1 ;; @@ -368,8 +366,7 @@ for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do lint_file "$file" done - if [ "$JSON_OUTPUT" = "true" ] && [ -n "$DIR_JSON_SCRIPTS" ]; then - command -v jq >/dev/null 2>&1 || { echo "ERROR: jq required for --json" >&2; exit 1; } + if [ -n "$DIR_JSON_SCRIPTS" ] && command -v jq >/dev/null 2>&1; then JSON_FILE="$TARGET_DIR/.y-script-lint.json" echo "[${DIR_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "2" \ @@ -384,4 +381,7 @@ NOTES="" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" for jf in "${JSON_FILES[@]}"; do echo "Wrote $jf"; done -[ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ] && { echo "FAILED:$FAIL_LIST"; exit 2; } +if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then + echo "FAILED:$FAIL_LIST" + exit 2 +fi From f0c8e087019ef925bc2d0e26ef0a76462c8e307b Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 11:44:42 +0000 Subject: [PATCH 16/28] remove sandbox/execution design from docs TODO_Y_SCRIPTS_LINT.md rewritten to reflect static-only implementation. Removed: phase 2 sandbox, unshare, env -i, macOS sandbox TODO, timeout design, help execution, prereq-before-help workarounds, dependency tree diagram, open questions about sandbox. Co-Authored-By: Claude Opus 4.6 (1M context) --- Y_SCRIPT_AUTHORING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Y_SCRIPT_AUTHORING.md b/Y_SCRIPT_AUTHORING.md index dfba6093..d6a3612a 100644 --- a/Y_SCRIPT_AUTHORING.md +++ b/Y_SCRIPT_AUTHORING.md @@ -177,7 +177,7 @@ This is preferred over `--help`-only because: - It reads naturally: `y-crane help`, `y-cluster-provision-k3d help` - It follows the subcommand pattern used by git, docker, kubectl - The help check happens before argument parsing, before prerequisite checks, - before any side effects — so it always works, even in a sandboxed env + before any side effects - `--help` is accepted for backwards compatibility but not documented in new scripts Existing scripts use various patterns (`--help`, `-h|--help`, `[[ "$1" =~ help$ ]]`). From 74b0de800ee084e76e4ef8f1aad65181e485163f Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 11:48:56 +0000 Subject: [PATCH 17/28] skip help requirement for trivial binary wrappers Detect the y-bin-download wrapper pattern: <=8 lines, y-bin-download call, "$@" passthrough, no conditionals/loops/functions. These scripts delegate to the wrapped binary's own help. 21 wrappers detected in ystack, all 8 lines. y-turbo (26L, has extra logic) correctly excluded. Passed: 10 -> 31. Also: prefix index write path with [y-script-lint]. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 6e642740..f300065e 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -124,6 +124,17 @@ check_no_eval() { fi } +# Detect trivial binary wrapper: header + YBIN + y-bin-download + versioned exec, nothing else +is_bin_wrapper() { + local file="$1" + # Must be sh, exactly 8 lines, contain y-bin-download and "$@" + [ "$(wc -l < "$file")" -le 8 ] || return 1 + grep -qE '^version=\$\(y-bin-download ' "$file" 2>/dev/null || return 1 + grep -qE '"\$@"' "$file" 2>/dev/null || return 1 + # Must not have anything beyond the wrapper pattern (no conditionals, loops, functions) + ! grep -qE '^(if |for |while |case |function |\w+\(\))' "$file" 2>/dev/null +} + SHELLCHECK_TIMEOUT_S="${Y_SCRIPT_LINT_TIMEOUT_S:-5}" HAS_SHELLCHECK=false command -v y-shellcheck >/dev/null 2>&1 && HAS_SHELLCHECK=true @@ -258,7 +269,16 @@ lint_file() { checks_header=null; checks_debug=null fi - check_help_handler "$file" "$lang" && checks_help_handler=true || errors+=("no help handler found in source") + local bin_wrapper=false + is_shell "$lang" && is_bin_wrapper "$file" && bin_wrapper=true + + if [ "$bin_wrapper" = "true" ]; then + checks_help_handler=true # not required for trivial wrappers + elif check_help_handler "$file" "$lang"; then + checks_help_handler=true + else + errors+=("no help handler found in source") + fi check_no_npx "$file" "$lang" || { checks_no_npx=false; errors+=("uses npx"); } check_no_eval "$file" "$lang" || { checks_no_eval=false; errors+=("uses eval"); } @@ -308,7 +328,9 @@ lint_file() { PASSED=$((PASSED + 1)) fi - echo " $name $lang ${loc}L" + local type_label="$lang" + [ "$bin_wrapper" = "true" ] && type_label="$lang wrapper" + echo " $name $type_label ${loc}L" for err in "${errors[@]}"; do local level="WARN" case "$err" in @@ -379,7 +401,7 @@ done NOTES="" [ "$HAS_SHELLCHECK" = "false" ] && NOTES="${NOTES}, no shellcheck" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" -for jf in "${JSON_FILES[@]}"; do echo "Wrote $jf"; done +for jf in "${JSON_FILES[@]}"; do echo "[y-script-lint] Wrote $jf"; done if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then echo "FAILED:$FAIL_LIST" From e52dd50c53d68fa83e52e50b7178456250c5f634 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Wed, 18 Mar 2026 12:00:30 +0000 Subject: [PATCH 18/28] add y-help to list scripts from index, add help_line to index y-help reads .y-script-lint.json from all PATH bin/ directories and prints one line per script: name, help summary, NOLINT if applicable. Supports optional substring filter argument. y-script-lint now writes help_line and lint_ok to the index: - YHELP scripts: description extracted from "y-name - description" - Binary wrappers: "toolname binary wrapper" - Others: null (y-help shows "no help section") Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-help | 56 +++++++++++++++++++++++++++++++++++++++++++++++ bin/y-script-lint | 18 ++++++++++----- 2 files changed, 69 insertions(+), 5 deletions(-) create mode 100755 bin/y-help diff --git a/bin/y-help b/bin/y-help new file mode 100755 index 00000000..c0940674 --- /dev/null +++ b/bin/y-help @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +YHELP='y-help - List y-* scripts with their help summaries + +Usage: y-help [help] [FILTER] + +Arguments: + FILTER Only show scripts matching this substring + +Reads .y-script-lint.json index files from bin/ directories in PATH. +Run y-script-lint first to generate or update the index. + +Dependencies: +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +FILTER="${1:-}" + +command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required" >&2; exit 1; } + +IFS=: read -ra path_entries <<< "$PATH" +found_index=false + +for dir in "${path_entries[@]}"; do + index="$dir/.y-script-lint.json" + [ -f "$index" ] || continue + found_index=true + + jq -r --arg filter "$FILTER" ' + .scripts | to_entries[] + | select(.key | contains($filter)) + | select(.value.skipped != true) + | .key as $name + | .value.help_line as $help + | .value.lint_ok as $ok + | [$name, ($help // "no help section"), (if $ok then "" else "NOLINT" end)] + | @tsv + ' "$index" | while IFS=$'\t' read -r name help nolint; do + if [ -n "$nolint" ]; then + printf " %-40s %s %s\n" "$name" "$help" "$nolint" + else + printf " %-40s %s\n" "$name" "$help" + fi + done +done + +if [ "$found_index" = "false" ]; then + echo "No .y-script-lint.json found in PATH. Run y-script-lint first." >&2 + exit 1 +fi diff --git a/bin/y-script-lint b/bin/y-script-lint index f300065e..928cb891 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -258,7 +258,7 @@ lint_file() { local checks_help_handler=false checks_shellcheck=null local checks_no_npx=true checks_no_eval=true local checks_help_format=null checks_deps_declared=null - local summary=null deps_json="[]" detected_deps="" + local summary=null help_line=null deps_json="[]" detected_deps="" [ "$lang" = "unknown" ] && { checks_shebang=false; errors+=("unrecognized or missing shebang"); } @@ -273,7 +273,11 @@ lint_file() { is_shell "$lang" && is_bin_wrapper "$file" && bin_wrapper=true if [ "$bin_wrapper" = "true" ]; then - checks_help_handler=true # not required for trivial wrappers + checks_help_handler=true + # Extract wrapped binary name for help_line + local wrapped_bin + wrapped_bin=$(grep 'y-bin-download' "$file" 2>/dev/null | sed 's/.*y-bin-download [^ ]* //;s/[)].*//') + [ -n "$wrapped_bin" ] && help_line="$wrapped_bin binary wrapper" elif check_help_handler "$file" "$lang"; then checks_help_handler=true else @@ -294,6 +298,7 @@ lint_file() { first_line=$(parse_summary_from_yhelp "$yhelp") if echo "$first_line" | grep -qE "^${name} - .+"; then checks_help_format=true; summary="$first_line" + help_line="${first_line#"${name} - "}" else checks_help_format=false; errors+=("YHELP first line does not match: $name - description") fi @@ -340,9 +345,10 @@ lint_file() { done if [ "$JSON_OUTPUT" = "true" ]; then - local errors_json="[]" summary_json="null" + local errors_json="[]" summary_json="null" help_line_json="null" [ ${#errors[@]} -gt 0 ] && errors_json=$(printf '%s\n' "${errors[@]}" | jq -R . | jq -s .) [ "$summary" != "null" ] && [ -n "$summary" ] && summary_json=$(echo "$summary" | jq -R .) + [ "$help_line" != "null" ] && [ -n "$help_line" ] && help_line_json=$(echo "$help_line" | jq -R .) local checks_json checks_json=$(jq -n \ --argjson shebang "$checks_shebang" --argjson header "$checks_header" \ @@ -355,10 +361,12 @@ lint_file() { help_format:$help_format,deps_declared:$deps_declared}') DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ --arg name "$name" --arg lang "$lang" --argjson summary "$summary_json" \ + --argjson help_line "$help_line_json" \ --argjson deps "$deps_json" --argjson detected "$detected_deps_json" \ --argjson checks "$checks_json" --argjson errors "$errors_json" \ - '{($name):{language:$lang,summary:$summary,dependencies:$deps, - detected_dependencies:$detected,checks:$checks,errors:$errors}}')," + --argjson lint_ok "$([ "$has_static_failure" = "false" ] && [ "$has_help_failure" = "false" ] && echo true || echo false)" \ + '{($name):{language:$lang,summary:$summary,help_line:$help_line,lint_ok:$lint_ok, + dependencies:$deps,detected_dependencies:$detected,checks:$checks,errors:$errors}}')," fi } From 1d6c546cd4429d61929cc88822375292cc7ab0b0 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Thu, 19 Mar 2026 07:36:19 +0000 Subject: [PATCH 19/28] single-script mode and detect help) case pattern y-script-lint y-turbo # lookup by name in PATH y-script-lint ./bin/y-turbo # explicit path y-script-lint /abs/path/y-x # absolute path Single-script mode: no index written, exit 0 if lint passes, exit 1 for warnings, exit 2 for static failures. Also detect help) case pattern (the YHELP subcommand convention) in addition to existing --help patterns. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 75 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 928cb891..592093b8 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -38,6 +38,7 @@ esac CHECK_MODE=false DEPS_ADD=false JSON_OUTPUT=true +SINGLE_FILE="" TARGET_DIRS=() while [ $# -gt 0 ]; do @@ -50,25 +51,56 @@ while [ $# -gt 0 ]; do esac done -if [ ${#TARGET_DIRS[@]} -eq 0 ]; then - IFS=: read -ra path_entries <<< "$PATH" - for dir in "${path_entries[@]}"; do - [ -d "$dir" ] || continue - for f in "$dir"/y-*; do - if [ -x "$f" ] && [ -f "$f" ]; then - TARGET_DIRS+=("$dir") +# Detect single-script mode: one arg that is a file or a bare script name +if [ ${#TARGET_DIRS[@]} -eq 1 ]; then + arg="${TARGET_DIRS[0]}" + if [ -f "$arg" ]; then + # Explicit path to a file + SINGLE_FILE="$(cd "$(dirname "$arg")" && pwd)/$(basename "$arg")" + elif [ -d "$arg" ]; then + : # directory, handled below + elif [[ "$arg" != */* ]]; then + # Bare name like "y-turbo" — search PATH + found="" + IFS=: read -ra path_entries <<< "$PATH" + for dir in "${path_entries[@]}"; do + if [ -x "$dir/$arg" ] && [ -f "$dir/$arg" ]; then + found="$dir/$arg" break fi done - done - [ ${#TARGET_DIRS[@]} -eq 0 ] && { echo "ERROR: no y-* scripts found in PATH" >&2; exit 1; } + [ -z "$found" ] && { echo "ERROR: $arg not found in PATH" >&2; exit 1; } + SINGLE_FILE="$found" + else + # Path with / but not a file + echo "ERROR: file not found: $arg" >&2; exit 1 + fi fi -RESOLVED_DIRS=() -for dir in "${TARGET_DIRS[@]}"; do - [ -d "$dir" ] || { echo "ERROR: directory not found: $dir" >&2; exit 1; } - RESOLVED_DIRS+=("$(cd "$dir" && pwd)") -done +if [ -n "$SINGLE_FILE" ]; then + JSON_OUTPUT=false + RESOLVED_DIRS=() +else + if [ ${#TARGET_DIRS[@]} -eq 0 ]; then + IFS=: read -ra path_entries <<< "$PATH" + for dir in "${path_entries[@]}"; do + [ -d "$dir" ] || continue + for f in "$dir"/y-*; do + if [ -x "$f" ] && [ -f "$f" ]; then + TARGET_DIRS+=("$dir") + break + fi + done + done + [ ${#TARGET_DIRS[@]} -eq 0 ] && { echo "ERROR: no y-* scripts found in PATH" >&2; exit 1; } + fi + + RESOLVED_DIRS=() + for dir in "${TARGET_DIRS[@]}"; do + [ -d "$dir" ] || { echo "ERROR: directory not found: $dir" >&2; exit 1; } + RESOLVED_DIRS+=("$(cd "$dir" && pwd)") + done +fi # --- Language detection --- @@ -97,7 +129,7 @@ check_header_debug() { grep -qE '^\[ -z "\$DEBUG" \] \|\| set -x' "$1" 2>/dev/nu check_help_handler() { local file="$1" lang="$2" if is_shell "$lang"; then - grep -qE '(\-h\|--help|"\$1" = "help"|"\$1" = "--help"|\$1 =~ help)' "$file" 2>/dev/null + grep -qE '(\-h\|--help|"\$1" = "help"|"\$1" = "--help"|\$1 =~ help|^\s*help\))' "$file" 2>/dev/null elif is_node "$lang"; then grep -qE "process\.argv\.includes\(['\"](-h|--help)['\"]" "$file" 2>/dev/null || grep -qE "process\.argv\[2\] === ['\"]help['\"]" "$file" 2>/dev/null @@ -374,6 +406,19 @@ lint_file() { TOTAL=0 PASSED=0 FAILED_STATIC=0 FAILED_HELP=0 SKIPPED=0 FAIL_LIST="" + +# Single-file mode: lint one script, exit with its result +if [ -n "$SINGLE_FILE" ]; then + TOTAL=1 + lint_file "$SINGLE_FILE" + if [ "$FAILED_STATIC" -gt 0 ]; then + exit 2 + elif [ "$FAILED_HELP" -gt 0 ]; then + exit 1 + fi + exit 0 +fi + declare -A SEEN_SCRIPTS JSON_FILES=() From 9c3f97adbf55914ca125d61736f0e5fedd3c45e3 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Fri, 20 Mar 2026 04:38:39 +0000 Subject: [PATCH 20/28] print authoring guide hint on lint failures Shows ~/path/to/Y_SCRIPT_AUTHORING.md when any script has FAIL or WARN, in both single-file and batch modes. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 592093b8..ccf6bc1b 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -4,6 +4,10 @@ set -eo pipefail trap 'trap - INT; kill -INT -$$' INT +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +AUTHORING_GUIDE="$(cd "$SCRIPT_DIR/.." && pwd)/Y_SCRIPT_AUTHORING.md" +AUTHORING_GUIDE_DISPLAY="~/${AUTHORING_GUIDE#"$HOME/"}" + YHELP='y-script-lint - Static analysis and convention checks for y-* scripts Usage: y-script-lint [help] [options] [DIR...] @@ -411,9 +415,9 @@ FAIL_LIST="" if [ -n "$SINGLE_FILE" ]; then TOTAL=1 lint_file "$SINGLE_FILE" - if [ "$FAILED_STATIC" -gt 0 ]; then - exit 2 - elif [ "$FAILED_HELP" -gt 0 ]; then + if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then + echo "See $AUTHORING_GUIDE_DISPLAY" + [ "$FAILED_STATIC" -gt 0 ] && exit 2 exit 1 fi exit 0 @@ -456,6 +460,10 @@ NOTES="" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" for jf in "${JSON_FILES[@]}"; do echo "[y-script-lint] Wrote $jf"; done +if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then + echo "See $AUTHORING_GUIDE_DISPLAY" +fi + if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then echo "FAILED:$FAIL_LIST" exit 2 From 7f5e4b2233ee998b8205a36ddea87e236c63bd00 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Fri, 20 Mar 2026 04:44:47 +0000 Subject: [PATCH 21/28] move index to single ~/.cache/ystack-script-lint.json All scripts from all directories merged into one index file. Each script entry includes parent (repo root) as a property. Removes per-directory .y-script-lint.json files from bin/. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/.gitignore | 1 - bin/y-help | 49 ++++++++++++++++++++--------------------------- bin/y-script-lint | 32 +++++++++++++++++-------------- 3 files changed, 39 insertions(+), 43 deletions(-) diff --git a/bin/.gitignore b/bin/.gitignore index a3c66a8b..496e7e22 100644 --- a/bin/.gitignore +++ b/bin/.gitignore @@ -1,6 +1,5 @@ *-bin *-dist -.y-script-lint.json # our executables are symlinked to real names buildctl diff --git a/bin/y-help b/bin/y-help index c0940674..e38f40ab 100755 --- a/bin/y-help +++ b/bin/y-help @@ -9,7 +9,7 @@ Usage: y-help [help] [FILTER] Arguments: FILTER Only show scripts matching this substring -Reads .y-script-lint.json index files from bin/ directories in PATH. +Reads ~/.cache/ystack-script-lint.json index. Run y-script-lint first to generate or update the index. Dependencies: @@ -24,33 +24,26 @@ FILTER="${1:-}" command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required" >&2; exit 1; } -IFS=: read -ra path_entries <<< "$PATH" -found_index=false - -for dir in "${path_entries[@]}"; do - index="$dir/.y-script-lint.json" - [ -f "$index" ] || continue - found_index=true - - jq -r --arg filter "$FILTER" ' - .scripts | to_entries[] - | select(.key | contains($filter)) - | select(.value.skipped != true) - | .key as $name - | .value.help_line as $help - | .value.lint_ok as $ok - | [$name, ($help // "no help section"), (if $ok then "" else "NOLINT" end)] - | @tsv - ' "$index" | while IFS=$'\t' read -r name help nolint; do - if [ -n "$nolint" ]; then - printf " %-40s %s %s\n" "$name" "$help" "$nolint" - else - printf " %-40s %s\n" "$name" "$help" - fi - done -done +INDEX="${HOME}/.cache/ystack-script-lint.json" -if [ "$found_index" = "false" ]; then - echo "No .y-script-lint.json found in PATH. Run y-script-lint first." >&2 +if [ ! -f "$INDEX" ]; then + echo "No index found at $INDEX. Run y-script-lint first." >&2 exit 1 fi + +jq -r --arg filter "$FILTER" ' + .scripts | to_entries[] + | select(.key | contains($filter)) + | select(.value.skipped != true) + | .key as $name + | .value.help_line as $help + | .value.lint_ok as $ok + | [$name, ($help // "no help section"), (if $ok then "" else "NOLINT" end)] + | @tsv +' "$INDEX" | while IFS=$'\t' read -r name help nolint; do + if [ -n "$nolint" ]; then + printf " %-40s %s %s\n" "$name" "$help" "$nolint" + else + printf " %-40s %s\n" "$name" "$help" + fi +done diff --git a/bin/y-script-lint b/bin/y-script-lint index ccf6bc1b..71f2ff21 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -284,7 +284,8 @@ lint_file() { if [ "$lang" = "config" ] || [ "$lang" = "library" ]; then SKIPPED=$((SKIPPED + 1)) [ "$JSON_OUTPUT" = "true" ] && DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ - --arg name "$name" --arg lang "$lang" '{($name): {language: $lang, skipped: true}}')," + --arg name "$name" --arg lang "$lang" --arg parent "$CURRENT_PARENT" \ + '{($name): {parent: $parent, language: $lang, skipped: true}}')," return fi @@ -396,12 +397,12 @@ lint_file() { shellcheck:$shellcheck,no_npx:$no_npx,no_eval:$no_eval, help_format:$help_format,deps_declared:$deps_declared}') DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ - --arg name "$name" --arg lang "$lang" --argjson summary "$summary_json" \ - --argjson help_line "$help_line_json" \ + --arg name "$name" --arg lang "$lang" --arg parent "$CURRENT_PARENT" \ + --argjson summary "$summary_json" --argjson help_line "$help_line_json" \ --argjson deps "$deps_json" --argjson detected "$detected_deps_json" \ --argjson checks "$checks_json" --argjson errors "$errors_json" \ --argjson lint_ok "$([ "$has_static_failure" = "false" ] && [ "$has_help_failure" = "false" ] && echo true || echo false)" \ - '{($name):{language:$lang,summary:$summary,help_line:$help_line,lint_ok:$lint_ok, + '{($name):{parent:$parent,language:$lang,summary:$summary,help_line:$help_line,lint_ok:$lint_ok, dependencies:$deps,detected_dependencies:$detected,checks:$checks,errors:$errors}}')," fi } @@ -424,7 +425,7 @@ if [ -n "$SINGLE_FILE" ]; then fi declare -A SEEN_SCRIPTS -JSON_FILES=() +ALL_JSON_SCRIPTS="" for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do dir_scripts=() @@ -440,25 +441,28 @@ for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do echo "[y-script-lint] $TARGET_DIR (${#dir_scripts[@]} scripts)" DIR_JSON_SCRIPTS="" + CURRENT_PARENT="$(dirname "$TARGET_DIR")" for file in "${dir_scripts[@]}"; do TOTAL=$((TOTAL + 1)) lint_file "$file" done - if [ -n "$DIR_JSON_SCRIPTS" ] && command -v jq >/dev/null 2>&1; then - JSON_FILE="$TARGET_DIR/.y-script-lint.json" - echo "[${DIR_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ - jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "2" \ - --arg directory "$TARGET_DIR" --argjson scripts "$(cat)" \ - '{generated:$generated,tool_version:$tool_version,directory:$directory,scripts:$scripts}' > "$JSON_FILE" - JSON_FILES+=("$JSON_FILE") - fi + ALL_JSON_SCRIPTS="${ALL_JSON_SCRIPTS}${DIR_JSON_SCRIPTS}" done NOTES="" [ "$HAS_SHELLCHECK" = "false" ] && NOTES="${NOTES}, no shellcheck" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" -for jf in "${JSON_FILES[@]}"; do echo "[y-script-lint] Wrote $jf"; done + +# Write single index file +INDEX_FILE="${HOME}/.cache/ystack-script-lint.json" +if [ -n "$ALL_JSON_SCRIPTS" ] && command -v jq >/dev/null 2>&1; then + echo "[${ALL_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ + jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "3" \ + --argjson scripts "$(cat)" \ + '{generated:$generated,tool_version:$tool_version,scripts:$scripts}' > "$INDEX_FILE" + echo "[y-script-lint] Wrote $INDEX_FILE" +fi if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then echo "See $AUTHORING_GUIDE_DISPLAY" From 9524222e84e3613273620191d0ad2fd8204f4102 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Fri, 20 Mar 2026 04:50:25 +0000 Subject: [PATCH 22/28] rename parent to reporoot, add actual parent (bin dir) parent: the directory containing the script (bin/) reporoot: git repo root if the directory is in a git repo, null otherwise Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 71f2ff21..41791c2c 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -285,7 +285,8 @@ lint_file() { SKIPPED=$((SKIPPED + 1)) [ "$JSON_OUTPUT" = "true" ] && DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ --arg name "$name" --arg lang "$lang" --arg parent "$CURRENT_PARENT" \ - '{($name): {parent: $parent, language: $lang, skipped: true}}')," + --arg reporoot "$CURRENT_REPOROOT" \ + '{($name): {parent: $parent, reporoot: (if $reporoot == "" then null else $reporoot end), language: $lang, skipped: true}}')," return fi @@ -398,11 +399,13 @@ lint_file() { help_format:$help_format,deps_declared:$deps_declared}') DIR_JSON_SCRIPTS="${DIR_JSON_SCRIPTS}$(jq -n \ --arg name "$name" --arg lang "$lang" --arg parent "$CURRENT_PARENT" \ + --arg reporoot "$CURRENT_REPOROOT" \ --argjson summary "$summary_json" --argjson help_line "$help_line_json" \ --argjson deps "$deps_json" --argjson detected "$detected_deps_json" \ --argjson checks "$checks_json" --argjson errors "$errors_json" \ --argjson lint_ok "$([ "$has_static_failure" = "false" ] && [ "$has_help_failure" = "false" ] && echo true || echo false)" \ - '{($name):{parent:$parent,language:$lang,summary:$summary,help_line:$help_line,lint_ok:$lint_ok, + '{($name):{parent:$parent,reporoot:(if $reporoot == "" then null else $reporoot end), + language:$lang,summary:$summary,help_line:$help_line,lint_ok:$lint_ok, dependencies:$deps,detected_dependencies:$detected,checks:$checks,errors:$errors}}')," fi } @@ -441,7 +444,8 @@ for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do echo "[y-script-lint] $TARGET_DIR (${#dir_scripts[@]} scripts)" DIR_JSON_SCRIPTS="" - CURRENT_PARENT="$(dirname "$TARGET_DIR")" + CURRENT_PARENT="$TARGET_DIR" + CURRENT_REPOROOT=$(git -C "$TARGET_DIR" rev-parse --show-toplevel 2>/dev/null || echo "") for file in "${dir_scripts[@]}"; do TOTAL=$((TOTAL + 1)) lint_file "$file" From 940681c3904d730ed67882bfa975ad9e2a5f1d71 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Fri, 20 Mar 2026 05:02:24 +0000 Subject: [PATCH 23/28] add --fail=degrade mode for CI-safe lint gating --fail=all: overwrites index, exits 2 if any static failures (same as --check) --fail=degrade: writes new index alongside old, calls y-script-lint-compare y-script-lint-compare takes two index files and exits 2 if: - Any check that passed (true) in old index now fails (false) in new - Any new script (not in old index) has a failing check Checks that were already false or null are ignored. On success in degrade mode, new index promotes to replace old. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 53 +++++++++++++++++++++------- bin/y-script-lint-compare | 72 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 12 deletions(-) create mode 100755 bin/y-script-lint-compare diff --git a/bin/y-script-lint b/bin/y-script-lint index 41791c2c..10393419 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -16,7 +16,9 @@ Arguments: DIR Directory to scan (default: all PATH dirs containing y-* scripts) Options: - --check Exit non-zero if any script fails static checks (for CI) + --check Alias for --fail=all + --fail=all Exit non-zero if any script fails + --fail=degrade Exit non-zero only if a script degraded or a new script fails --dependencies-add Detect y-* invocations and add to Dependencies section in YHELP All checks are static (source parsing only, no script execution). @@ -29,9 +31,9 @@ Dependencies: y-shellcheck Exit codes: - 0 All scripts passed (or --check not set) + 0 All scripts passed (or no degradation in degrade mode) 1 Usage error - 2 One or more scripts failed checks (--check mode) + 2 Lint failures detected ' case "${1:-}" in @@ -39,7 +41,7 @@ case "${1:-}" in --help) echo "$YHELP"; exit 0 ;; esac -CHECK_MODE=false +FAIL_MODE="" DEPS_ADD=false JSON_OUTPUT=true SINGLE_FILE="" @@ -47,7 +49,10 @@ TARGET_DIRS=() while [ $# -gt 0 ]; do case "$1" in - --check) CHECK_MODE=true; shift ;; + --check) FAIL_MODE=all; shift ;; + --fail=all) FAIL_MODE=all; shift ;; + --fail=degrade) FAIL_MODE=degrade; shift ;; + --fail=*) echo "ERROR: unknown fail mode: ${1#*=} (use all or degrade)" >&2; exit 1 ;; --dependencies-add) DEPS_ADD=true; shift ;; --) shift; break ;; -*) echo "Unknown flag: $1" >&2; exit 1 ;; @@ -458,21 +463,45 @@ NOTES="" [ "$HAS_SHELLCHECK" = "false" ] && NOTES="${NOTES}, no shellcheck" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" -# Write single index file +# Write index INDEX_FILE="${HOME}/.cache/ystack-script-lint.json" if [ -n "$ALL_JSON_SCRIPTS" ] && command -v jq >/dev/null 2>&1; then - echo "[${ALL_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ - jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "3" \ - --argjson scripts "$(cat)" \ - '{generated:$generated,tool_version:$tool_version,scripts:$scripts}' > "$INDEX_FILE" - echo "[y-script-lint] Wrote $INDEX_FILE" + build_index() { + echo "[${ALL_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ + jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "3" \ + --argjson scripts "$(cat)" \ + '{generated:$generated,tool_version:$tool_version,scripts:$scripts}' + } + + if [ "$FAIL_MODE" = "degrade" ] && [ -f "$INDEX_FILE" ]; then + NEW_INDEX="${INDEX_FILE%.json}.new.json" + build_index > "$NEW_INDEX" + echo "[y-script-lint] Wrote $NEW_INDEX" + + if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then + echo "See $AUTHORING_GUIDE_DISPLAY" + fi + + "$SCRIPT_DIR/y-script-lint-compare" "$INDEX_FILE" "$NEW_INDEX" + compare_exit=$? + # On success, promote new index + if [ $compare_exit -eq 0 ]; then + mv "$NEW_INDEX" "$INDEX_FILE" + else + rm -f "$NEW_INDEX" + fi + exit $compare_exit + else + build_index > "$INDEX_FILE" + echo "[y-script-lint] Wrote $INDEX_FILE" + fi fi if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then echo "See $AUTHORING_GUIDE_DISPLAY" fi -if [ "$CHECK_MODE" = "true" ] && [ "$FAILED_STATIC" -gt 0 ]; then +if [ "$FAIL_MODE" = "all" ] && [ "$FAILED_STATIC" -gt 0 ]; then echo "FAILED:$FAIL_LIST" exit 2 fi diff --git a/bin/y-script-lint-compare b/bin/y-script-lint-compare new file mode 100755 index 00000000..b7d30938 --- /dev/null +++ b/bin/y-script-lint-compare @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +YHELP='y-script-lint-compare - Compare two lint index files for degradation + +Usage: y-script-lint-compare [help] OLD_INDEX NEW_INDEX + +A check is degraded if it passed (true) in OLD and fails (false) in NEW. +A new script (present in NEW but not OLD) must pass all its checks. +Checks that were null (not applicable) or already false are ignored. + +Exit codes: + 0 No degradation + 2 Degradation or new script failure detected + +Dependencies: +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +[ -z "$1" ] || [ -z "$2" ] && { echo "Usage: y-script-lint-compare OLD_INDEX NEW_INDEX" >&2; exit 1; } +[ -f "$1" ] || { echo "ERROR: old index not found: $1" >&2; exit 1; } +[ -f "$2" ] || { echo "ERROR: new index not found: $2" >&2; exit 1; } + +OLD="$1" +NEW="$2" + +# jq does all the work: compare checks between old and new +RESULT=$(jq -n \ + --slurpfile old "$OLD" \ + --slurpfile new "$NEW" ' + + def check_names: ["shebang","header","debug","help_handler","shellcheck","no_npx","no_eval","help_format","deps_declared"]; + + ($old[0].scripts // {}) as $old_scripts | + ($new[0].scripts // {}) as $new_scripts | + + # Find degraded checks in existing scripts + [ + $new_scripts | to_entries[] | + .key as $name | .value as $new_val | + select($new_val.skipped != true) | + if ($old_scripts[$name] // null) != null and ($old_scripts[$name].skipped != true) then + ($old_scripts[$name].checks // {}) as $old_checks | + ($new_val.checks // {}) as $new_checks | + check_names[] | + select($old_checks[.] == true and $new_checks[.] == false) | + {script: $name, check: ., type: "degraded"} + else + # New script: any false check is a failure + ($new_val.checks // {}) as $new_checks | + check_names[] | + select($new_checks[.] == false) | + {script: $name, check: ., type: "new_failure"} + end + ] +') + +COUNT=$(echo "$RESULT" | jq 'length') + +if [ "$COUNT" -eq 0 ]; then + echo "[y-script-lint-compare] No degradation detected" + exit 0 +fi + +echo "[y-script-lint-compare] $COUNT issue(s) found:" +echo "$RESULT" | jq -r '.[] | " \(.type): \(.script) check \(.check)"' +exit 2 From 3b0d6ef06eb082b78827755bc26b4c54f52973a9 Mon Sep 17 00:00:00 2001 From: Yolean k8s-qa Date: Fri, 20 Mar 2026 06:11:52 +0000 Subject: [PATCH 24/28] --fail=degrade with cached baselines for CI Manages two baselines in ~/.cache/: ystack-script-lint.main.json - saved on main/master builds ystack-script-lint.branch.json - saved on branch builds (on success) Baseline selection: branch > main > none (skip compare). Y_SCRIPT_LINT_BRANCH env overrides git branch detection. CI needs only: cache the two baseline files + run --fail=degrade. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/images.yaml | 3 ++ .github/workflows/lint.yaml | 32 +++++++++++++ bin/y-script-lint | 87 ++++++++++++++++++++++------------- bin/y-script-lint-compare | 19 +++++--- 4 files changed, 102 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/lint.yaml diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 78df6c1a..1aa7b227 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -6,7 +6,10 @@ on: - main jobs: + lint: + uses: ./.github/workflows/lint.yaml docker: + needs: lint runs-on: ubuntu-latest permissions: packages: write diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..01403380 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,32 @@ +name: lint + +on: + push: + branches: + - main + pull_request: + workflow_call: + +jobs: + script-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/cache/restore@v4 + with: + key: script-lint-${{ github.ref_name }}- + restore-keys: | + script-lint-main- + path: | + ~/.cache/ystack-script-lint.main.json + ~/.cache/ystack-script-lint.branch.json + - name: Script lint + run: bin/y-script-lint --fail=degrade bin/ + env: + Y_SCRIPT_LINT_BRANCH: ${{ github.ref_name }} + - uses: actions/cache/save@v4 + with: + key: script-lint-${{ github.ref_name }}-${{ github.run_id }} + path: | + ~/.cache/ystack-script-lint.main.json + ~/.cache/ystack-script-lint.branch.json diff --git a/bin/y-script-lint b/bin/y-script-lint index 10393419..a303c9a8 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -18,13 +18,22 @@ Arguments: Options: --check Alias for --fail=all --fail=all Exit non-zero if any script fails - --fail=degrade Exit non-zero only if a script degraded or a new script fails + --fail=degrade Exit non-zero only if a script degraded vs baseline --dependencies-add Detect y-* invocations and add to Dependencies section in YHELP +Degradation baselines (in ~/.cache/): + ystack-script-lint.main.json Saved on main branch builds + ystack-script-lint.branch.json Saved on branch builds + Branch baseline preferred, falls back to main. + First build with no baseline skips comparison. + On main: result always saves as main baseline. + On branch: result saves as branch baseline on success. + All checks are static (source parsing only, no script execution). Environment: Y_SCRIPT_LINT_TIMEOUT_S Shellcheck timeout in seconds (default: 5) + Y_SCRIPT_LINT_BRANCH Branch name (default: git current branch, "main" = main baseline) Dependencies: y-bin-download @@ -52,7 +61,7 @@ while [ $# -gt 0 ]; do --check) FAIL_MODE=all; shift ;; --fail=all) FAIL_MODE=all; shift ;; --fail=degrade) FAIL_MODE=degrade; shift ;; - --fail=*) echo "ERROR: unknown fail mode: ${1#*=} (use all or degrade)" >&2; exit 1 ;; + --fail=*) echo "ERROR: unknown fail mode: ${1#*=} (use: all, degrade)" >&2; exit 1 ;; --dependencies-add) DEPS_ADD=true; shift ;; --) shift; break ;; -*) echo "Unknown flag: $1" >&2; exit 1 ;; @@ -464,37 +473,20 @@ NOTES="" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" # Write index -INDEX_FILE="${HOME}/.cache/ystack-script-lint.json" -if [ -n "$ALL_JSON_SCRIPTS" ] && command -v jq >/dev/null 2>&1; then - build_index() { - echo "[${ALL_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ - jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "3" \ - --argjson scripts "$(cat)" \ - '{generated:$generated,tool_version:$tool_version,scripts:$scripts}' - } - - if [ "$FAIL_MODE" = "degrade" ] && [ -f "$INDEX_FILE" ]; then - NEW_INDEX="${INDEX_FILE%.json}.new.json" - build_index > "$NEW_INDEX" - echo "[y-script-lint] Wrote $NEW_INDEX" - - if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then - echo "See $AUTHORING_GUIDE_DISPLAY" - fi +CACHE_DIR="${HOME}/.cache" +mkdir -p "$CACHE_DIR" +INDEX_FILE="${CACHE_DIR}/ystack-script-lint.json" + +build_index() { + echo "[${ALL_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ + jq -n --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg tool_version "3" \ + --argjson scripts "$(cat)" \ + '{generated:$generated,tool_version:$tool_version,scripts:$scripts}' +} - "$SCRIPT_DIR/y-script-lint-compare" "$INDEX_FILE" "$NEW_INDEX" - compare_exit=$? - # On success, promote new index - if [ $compare_exit -eq 0 ]; then - mv "$NEW_INDEX" "$INDEX_FILE" - else - rm -f "$NEW_INDEX" - fi - exit $compare_exit - else - build_index > "$INDEX_FILE" - echo "[y-script-lint] Wrote $INDEX_FILE" - fi +if [ -n "$ALL_JSON_SCRIPTS" ] && command -v jq >/dev/null 2>&1; then + build_index > "$INDEX_FILE" + echo "[y-script-lint] Wrote $INDEX_FILE" fi if [ "$FAILED_STATIC" -gt 0 ] || [ "$FAILED_HELP" -gt 0 ]; then @@ -505,3 +497,34 @@ if [ "$FAIL_MODE" = "all" ] && [ "$FAILED_STATIC" -gt 0 ]; then echo "FAILED:$FAIL_LIST" exit 2 fi + +if [ "$FAIL_MODE" = "degrade" ] && [ -f "$INDEX_FILE" ]; then + BRANCH="${Y_SCRIPT_LINT_BRANCH:-$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "")}" + MAIN_BASELINE="${CACHE_DIR}/ystack-script-lint.main.json" + BRANCH_BASELINE="${CACHE_DIR}/ystack-script-lint.branch.json" + + # Select best baseline: branch cache > main cache > none + BASELINE="" + [ -f "$BRANCH_BASELINE" ] && BASELINE="$BRANCH_BASELINE" + [ -z "$BASELINE" ] && [ -f "$MAIN_BASELINE" ] && BASELINE="$MAIN_BASELINE" + + if [ -z "$BASELINE" ]; then + echo "[y-script-lint] No baseline found, skipping degradation check" + else + echo "[y-script-lint] Comparing against $(basename "$BASELINE")" + "$SCRIPT_DIR/y-script-lint-compare" "$BASELINE" "$INDEX_FILE" + compare_exit=$? + if [ $compare_exit -ne 0 ]; then + exit $compare_exit + fi + fi + + # Save baseline for future runs + if [ "$BRANCH" = "main" ] || [ "$BRANCH" = "master" ]; then + cp "$INDEX_FILE" "$MAIN_BASELINE" + echo "[y-script-lint] Saved main baseline" + else + cp "$INDEX_FILE" "$BRANCH_BASELINE" + echo "[y-script-lint] Saved branch baseline" + fi +fi diff --git a/bin/y-script-lint-compare b/bin/y-script-lint-compare index b7d30938..58ed680b 100755 --- a/bin/y-script-lint-compare +++ b/bin/y-script-lint-compare @@ -7,8 +7,10 @@ YHELP='y-script-lint-compare - Compare two lint index files for degradation Usage: y-script-lint-compare [help] OLD_INDEX NEW_INDEX A check is degraded if it passed (true) in OLD and fails (false) in NEW. -A new script (present in NEW but not OLD) must pass all its checks. -Checks that were null (not applicable) or already false are ignored. +A new script (present in NEW but not OLD) must pass static checks +(shebang, header, shellcheck, no_npx, no_eval). Warn-level checks +(help_handler, help_format, etc.) are not enforced for new scripts. +Scripts deleted between OLD and NEW are ignored. Exit codes: 0 No degradation @@ -34,26 +36,29 @@ RESULT=$(jq -n \ --slurpfile old "$OLD" \ --slurpfile new "$NEW" ' - def check_names: ["shebang","header","debug","help_handler","shellcheck","no_npx","no_eval","help_format","deps_declared"]; + # Checks where false is a static failure (FAIL level) + def fail_checks: ["shebang","header","shellcheck","no_npx","no_eval"]; + # All checks including warn-level + def all_checks: ["shebang","header","debug","help_handler","shellcheck","no_npx","no_eval","help_format","deps_declared"]; ($old[0].scripts // {}) as $old_scripts | ($new[0].scripts // {}) as $new_scripts | - # Find degraded checks in existing scripts [ $new_scripts | to_entries[] | .key as $name | .value as $new_val | select($new_val.skipped != true) | if ($old_scripts[$name] // null) != null and ($old_scripts[$name].skipped != true) then + # Existing script: any check that passed before must still pass ($old_scripts[$name].checks // {}) as $old_checks | ($new_val.checks // {}) as $new_checks | - check_names[] | + all_checks[] | select($old_checks[.] == true and $new_checks[.] == false) | {script: $name, check: ., type: "degraded"} else - # New script: any false check is a failure + # New script: only static failures count (not warn-level checks) ($new_val.checks // {}) as $new_checks | - check_names[] | + fail_checks[] | select($new_checks[.] == false) | {script: $name, check: ., type: "new_failure"} end From d317b974a5d35522ad96944b0f294b80bf5f604c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 20 Mar 2026 08:01:52 +0100 Subject: [PATCH 25/28] fix y-script-lint for macOS bash 3.2 compatibility Replace declare -A (associative array, bash 4+) with string-based dedup using case pattern matching. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index a303c9a8..1b30fa42 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -441,7 +441,7 @@ if [ -n "$SINGLE_FILE" ]; then exit 0 fi -declare -A SEEN_SCRIPTS +SEEN_SCRIPTS="" ALL_JSON_SCRIPTS="" for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do @@ -450,8 +450,8 @@ for TARGET_DIR in "${RESOLVED_DIRS[@]}"; do [ -f "$file" ] && [ -x "$file" ] || continue name=$(basename "$file") case "$name" in *.yaml|*.yml|*.md|*.spec.js|*.spec.ts|*.test.js|*.test.ts|*-bin|*-dist) continue ;; esac - [ -n "${SEEN_SCRIPTS[$name]:-}" ] && continue - SEEN_SCRIPTS[$name]=1 + case "$SEEN_SCRIPTS" in *"|$name|"*) continue ;; esac + SEEN_SCRIPTS="${SEEN_SCRIPTS}|${name}|" dir_scripts+=("$file") done [ ${#dir_scripts[@]} -eq 0 ] && continue From ca38c48692f4ff4bdf0468ac4c3966c65d6b1805 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 20 Mar 2026 08:04:14 +0100 Subject: [PATCH 26/28] fix shellcheck on macOS where timeout command is unavailable GNU timeout is not available on macOS by default. The missing command caused run_shellcheck to always exit non-zero, reporting all shell scripts as shellcheck failures. Run shellcheck without timeout when the command is not found. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index 1b30fa42..fcac6639 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -189,8 +189,16 @@ SHELLCHECK_TIMEOUT_S="${Y_SCRIPT_LINT_TIMEOUT_S:-5}" HAS_SHELLCHECK=false command -v y-shellcheck >/dev/null 2>&1 && HAS_SHELLCHECK=true +HAS_TIMEOUT=false +command -v timeout >/dev/null 2>&1 && HAS_TIMEOUT=true + run_shellcheck() { - [ "$HAS_SHELLCHECK" = "true" ] && timeout --kill-after=2 "$SHELLCHECK_TIMEOUT_S" y-shellcheck --severity=error "$1" >/dev/null 2>&1 + [ "$HAS_SHELLCHECK" = "true" ] || return 0 + if [ "$HAS_TIMEOUT" = "true" ]; then + timeout --kill-after=2 "$SHELLCHECK_TIMEOUT_S" y-shellcheck --severity=error "$1" >/dev/null 2>&1 + else + y-shellcheck --severity=error "$1" >/dev/null 2>&1 + fi } # --- YHELP source parsing --- From bdaaf6baf8ecf3405cb3896136d3ff720efdba4b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 20 Mar 2026 08:12:26 +0100 Subject: [PATCH 27/28] move lint cache to ~/.cache/ystack/ subdirectory Follow XDG convention of using a subdirectory under ~/.cache/. Rename files from ystack-script-lint.* to script-lint.* since the ystack prefix is now in the directory name. Update workflow cache paths accordingly. First CI run will skip degradation check (no baseline at new path) then resume normally. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/lint.yaml | 8 ++------ bin/y-help | 4 ++-- bin/y-script-lint | 14 +++++++------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 01403380..144ef918 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -17,9 +17,7 @@ jobs: key: script-lint-${{ github.ref_name }}- restore-keys: | script-lint-main- - path: | - ~/.cache/ystack-script-lint.main.json - ~/.cache/ystack-script-lint.branch.json + path: ~/.cache/ystack - name: Script lint run: bin/y-script-lint --fail=degrade bin/ env: @@ -27,6 +25,4 @@ jobs: - uses: actions/cache/save@v4 with: key: script-lint-${{ github.ref_name }}-${{ github.run_id }} - path: | - ~/.cache/ystack-script-lint.main.json - ~/.cache/ystack-script-lint.branch.json + path: ~/.cache/ystack diff --git a/bin/y-help b/bin/y-help index e38f40ab..9b864502 100755 --- a/bin/y-help +++ b/bin/y-help @@ -9,7 +9,7 @@ Usage: y-help [help] [FILTER] Arguments: FILTER Only show scripts matching this substring -Reads ~/.cache/ystack-script-lint.json index. +Reads ~/.cache/ystack/y-script-lint.json index. Run y-script-lint first to generate or update the index. Dependencies: @@ -24,7 +24,7 @@ FILTER="${1:-}" command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required" >&2; exit 1; } -INDEX="${HOME}/.cache/ystack-script-lint.json" +INDEX="${HOME}/.cache/ystack/y-script-lint.json" if [ ! -f "$INDEX" ]; then echo "No index found at $INDEX. Run y-script-lint first." >&2 diff --git a/bin/y-script-lint b/bin/y-script-lint index fcac6639..f3b81413 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -21,9 +21,9 @@ Options: --fail=degrade Exit non-zero only if a script degraded vs baseline --dependencies-add Detect y-* invocations and add to Dependencies section in YHELP -Degradation baselines (in ~/.cache/): - ystack-script-lint.main.json Saved on main branch builds - ystack-script-lint.branch.json Saved on branch builds +Degradation baselines (in ~/.cache/ystack/): + y-script-lint.main.json Saved on main branch builds + y-script-lint.branch.json Saved on branch builds Branch baseline preferred, falls back to main. First build with no baseline skips comparison. On main: result always saves as main baseline. @@ -481,9 +481,9 @@ NOTES="" echo "Total: $TOTAL Passed: $PASSED Failed: $FAILED_STATIC Warnings: $FAILED_HELP Skipped: $SKIPPED${NOTES}" # Write index -CACHE_DIR="${HOME}/.cache" +CACHE_DIR="${HOME}/.cache/ystack" mkdir -p "$CACHE_DIR" -INDEX_FILE="${CACHE_DIR}/ystack-script-lint.json" +INDEX_FILE="${CACHE_DIR}/y-script-lint.json" build_index() { echo "[${ALL_JSON_SCRIPTS%,}]" | jq 'reduce .[] as $item ({}; . + $item)' | \ @@ -508,8 +508,8 @@ fi if [ "$FAIL_MODE" = "degrade" ] && [ -f "$INDEX_FILE" ]; then BRANCH="${Y_SCRIPT_LINT_BRANCH:-$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "")}" - MAIN_BASELINE="${CACHE_DIR}/ystack-script-lint.main.json" - BRANCH_BASELINE="${CACHE_DIR}/ystack-script-lint.branch.json" + MAIN_BASELINE="${CACHE_DIR}/y-script-lint.main.json" + BRANCH_BASELINE="${CACHE_DIR}/y-script-lint.branch.json" # Select best baseline: branch cache > main cache > none BASELINE="" From 162a8cf52ed281946a5bb1e4c79657e1956895dd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 20 Mar 2026 09:48:37 +0100 Subject: [PATCH 28/28] removes dependence on timeout tool not available by default on osx Timeout was introduced for the original execution-based design where scripts were run to capture help output. Shellcheck is static analysis and does not need a timeout. Co-Authored-By: Claude Opus 4.6 (1M context) --- bin/y-script-lint | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/bin/y-script-lint b/bin/y-script-lint index f3b81413..67b58644 100755 --- a/bin/y-script-lint +++ b/bin/y-script-lint @@ -32,7 +32,6 @@ Degradation baselines (in ~/.cache/ystack/): All checks are static (source parsing only, no script execution). Environment: - Y_SCRIPT_LINT_TIMEOUT_S Shellcheck timeout in seconds (default: 5) Y_SCRIPT_LINT_BRANCH Branch name (default: git current branch, "main" = main baseline) Dependencies: @@ -185,20 +184,11 @@ is_bin_wrapper() { ! grep -qE '^(if |for |while |case |function |\w+\(\))' "$file" 2>/dev/null } -SHELLCHECK_TIMEOUT_S="${Y_SCRIPT_LINT_TIMEOUT_S:-5}" HAS_SHELLCHECK=false command -v y-shellcheck >/dev/null 2>&1 && HAS_SHELLCHECK=true -HAS_TIMEOUT=false -command -v timeout >/dev/null 2>&1 && HAS_TIMEOUT=true - run_shellcheck() { - [ "$HAS_SHELLCHECK" = "true" ] || return 0 - if [ "$HAS_TIMEOUT" = "true" ]; then - timeout --kill-after=2 "$SHELLCHECK_TIMEOUT_S" y-shellcheck --severity=error "$1" >/dev/null 2>&1 - else - y-shellcheck --severity=error "$1" >/dev/null 2>&1 - fi + [ "$HAS_SHELLCHECK" = "true" ] && y-shellcheck --severity=error "$1" >/dev/null 2>&1 } # --- YHELP source parsing ---