diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d51d03..a5cab0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,8 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo fmt --all --check - test: - name: cargo test (${{ matrix.os }}) + nextest: + name: nextest (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -37,12 +37,14 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest + + - name: Run tests with nextest + run: cargo nextest run --workspace --release --no-fail-fast - - name: Run tests - run: cargo test --workspace --release + - name: Run doctests + run: cargo test --doc --workspace --release coverage: name: coverage @@ -53,9 +55,10 @@ jobs: with: components: llvm-tools-preview - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest - uses: taiki-e/install-action@cargo-llvm-cov - name: Generate coverage JSON - run: cargo llvm-cov --workspace --json --output-path coverage.json + run: cargo llvm-cov nextest --workspace --release --json --output-path coverage.json - name: Check per-file thresholds run: python3 scripts/check-coverage.py coverage.json diff --git a/AGENTS.md b/AGENTS.md index bc0d728..c4854ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,13 +51,15 @@ When a file grows large, split it by functionality (e.g., parsing, plan computat **Always use `--release` mode for tests** to enable optimizations and speed up trial-and-error cycles. ```bash -cargo test --release # Full suite -cargo test --release --test test_name # Specific test -cargo test --release --workspace # All crates +cargo nextest run --release # Full suite +cargo nextest run --release --test test_name # Specific test target +cargo nextest run --release --workspace # All crates +cargo test --doc --release --workspace # Doc tests ``` - Private functions: `#[cfg(test)]` module in source file - Integration tests: `tests/` directory +- Use `cargo nextest run` for unit and integration tests; keep doctests on `cargo test --doc`. - **Test tolerance changes**: When relaxing test tolerances (unit tests, codecov targets, etc.), always seek explicit user approval before making changes. - **Coverage-driven additions**: Meet the threshold with meaningful behavior-focused tests. Do not add filler tests solely to raise coverage numbers. @@ -215,13 +217,14 @@ Before creating a PR, always run these checks locally: ```bash cargo fmt --all # Format all code cargo clippy --workspace # Check for common issues -cargo test --release --workspace # Run all tests +cargo nextest run --release --workspace --no-fail-fast # Run unit and integration tests +cargo test --doc --release --workspace # Run doc tests ``` **Coverage check (must pass before push):** ```bash -cargo llvm-cov --workspace --json --output-path coverage.json +cargo llvm-cov nextest --workspace --release --json --output-path coverage.json python3 scripts/check-coverage.py coverage.json ``` diff --git a/README.md b/README.md index 49348be..952ffdf 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Template repository for Rust workspace projects in the tensor4all organization. ## What's included - **Cargo workspace** — empty workspace ready to add crates -- **CI (GitHub Actions)** — fmt, test (ubuntu + macOS), coverage, doc +- **CI (GitHub Actions)** — fmt, nextest + doctest (ubuntu + macOS), coverage, doc - **Coverage checking** — per-file line coverage thresholds via `cargo llvm-cov` - **Agent guidelines** — CLAUDE.md / AGENTS.md for AI-assisted development @@ -43,6 +43,15 @@ Artifacts: - durable reports: `docs/test-reports/agentic-bug-sweep/` - ephemeral execution state: `target/agentic-bug-sweep/` +## Testing + +Run unit and integration tests with nextest, and keep doctests as a separate step: + +```bash +cargo nextest run --workspace --release --no-fail-fast +cargo test --doc --workspace --release +``` + ## Coverage Coverage is checked per-file against thresholds in `coverage-thresholds.json`. @@ -50,7 +59,7 @@ Meet the configured threshold with meaningful tests rather than filler assertion ```bash # Run locally -cargo llvm-cov --workspace --json --output-path coverage.json +cargo llvm-cov nextest --workspace --release --json --output-path coverage.json python3 scripts/check-coverage.py coverage.json ``` diff --git a/ai/common-agent-rules.md b/ai/common-agent-rules.md index f26c63a..6a020f4 100644 --- a/ai/common-agent-rules.md +++ b/ai/common-agent-rules.md @@ -60,8 +60,8 @@ When in doubt, ask: *"Would an experienced software engineer consider this clean - When proceeding on NFS, place Cargo build artifacts on local disk rather than inside the repository checkout. - Prefer a stable repo-specific local target directory such as - `CARGO_TARGET_DIR=/tmp/-target` for `cargo build`, `cargo test`, - `cargo llvm-cov`, and similar heavy commands. + `CARGO_TARGET_DIR=/tmp/-target` for `cargo build`, `cargo nextest run`, + `cargo test --doc`, `cargo llvm-cov`, and similar heavy commands. ## File Organization diff --git a/ai/new-tensor4all-rust-repo/scripts/new-repo.sh b/ai/new-tensor4all-rust-repo/scripts/new-repo.sh index b8a76d8..af1a60b 100755 --- a/ai/new-tensor4all-rust-repo/scripts/new-repo.sh +++ b/ai/new-tensor4all-rust-repo/scripts/new-repo.sh @@ -188,7 +188,13 @@ ASSETS_SYNCED=1 if ! run_in_repo cargo fmt --all --check; then fail_with_summary 1 "cargo fmt --all --check failed in $DEST_PATH" fi -if ! run_in_repo cargo llvm-cov --workspace --release --json --output-path coverage.json; then +if ! run_in_repo cargo nextest run --workspace --release --no-fail-fast; then + fail_with_summary 1 "cargo nextest run failed in $DEST_PATH" +fi +if ! run_in_repo cargo test --doc --workspace --release; then + fail_with_summary 1 "cargo test --doc failed in $DEST_PATH" +fi +if ! run_in_repo cargo llvm-cov nextest --workspace --release --json --output-path coverage.json; then fail_with_summary 1 "cargo llvm-cov failed in $DEST_PATH" fi if ! run_in_repo python3 scripts/check-coverage.py coverage.json; then diff --git a/ai/numerical-rust-rules.md b/ai/numerical-rust-rules.md index f324235..ff942a4 100644 --- a/ai/numerical-rust-rules.md +++ b/ai/numerical-rust-rules.md @@ -8,7 +8,8 @@ are illustrative — apply the same reasoning to any new pattern. - Use small deterministic inputs for correctness tests. - Prefer hard-coded data or seeded RNGs over unseeded randomness. - Feature-gate expensive tests. -- For local trial-and-error loops, `cargo test --release --workspace` is preferred when it materially reduces iteration time. +- For local trial-and-error loops, prefer `cargo nextest run --release --workspace --no-fail-fast` for unit and integration tests. +- Run doctests separately with `cargo test --doc --release --workspace`; `nextest` does not replace them. ## Generic Test Patterns diff --git a/ai/pr-workflow-rules.md b/ai/pr-workflow-rules.md index ae0db73..e06cf71 100644 --- a/ai/pr-workflow-rules.md +++ b/ai/pr-workflow-rules.md @@ -16,13 +16,16 @@ Before pushing or creating a PR, all of these must pass: ```bash cargo fmt --all --check -cargo llvm-cov --workspace --release --json --output-path coverage.json +cargo nextest run --workspace --release --no-fail-fast +cargo test --doc --workspace --release +cargo llvm-cov nextest --workspace --release --json --output-path coverage.json python3 scripts/check-coverage.py coverage.json cargo doc --workspace --no-deps python3 scripts/check-docs-site.py ``` If formatting fails, run `cargo fmt --all` and rerun the checks. +Keep doctests as a dedicated `cargo test --doc` step; `cargo nextest` does not execute them. ## Repository Settings diff --git a/ai/repo-settings.json b/ai/repo-settings.json index 6b83fde..7b1cdcc 100644 --- a/ai/repo-settings.json +++ b/ai/repo-settings.json @@ -6,8 +6,8 @@ "strict": true, "contexts": [ "rustfmt", - "cargo test (ubuntu-latest)", - "cargo test (macos-latest)", + "nextest (ubuntu-latest)", + "nextest (macos-latest)", "coverage", "docs-site" ] diff --git a/docs/plans/2026-03-11-nextest-rollout-implementation.md b/docs/plans/2026-03-11-nextest-rollout-implementation.md new file mode 100644 index 0000000..39b18df --- /dev/null +++ b/docs/plans/2026-03-11-nextest-rollout-implementation.md @@ -0,0 +1,124 @@ +# Nextest Rollout Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Make `cargo nextest` the default test runner across the template's CI, repository settings, helper scripts, and coding rules while keeping doctests explicitly covered. + +**Architecture:** Add a small regression test that validates the template's expected verification commands and required status-check names from repository files. Then update the workflow, settings, helper scripts, and rule documents together so the repository describes and executes one consistent testing strategy: `cargo nextest run` for normal tests, `cargo test --doc` for doctests, and `cargo llvm-cov nextest` for coverage. + +**Tech Stack:** GitHub Actions YAML, Bash scripts, JSON, Markdown, Python `unittest` + +--- + +### Task 1: Add regression coverage for the rollout + +**Files:** +- Create: `scripts/tests/test_nextest_rollout.py` + +**Step 1: Write the failing test** + +Add a Python `unittest` module that reads repository files as text and asserts: +- `.github/workflows/ci.yml` contains a `nextest`-named test job, installs `cargo-nextest`, runs `cargo nextest run --workspace --release --no-fail-fast`, and runs `cargo test --doc --workspace --release` +- `ai/repo-settings.json` uses `nextest (ubuntu-latest)` and `nextest (macos-latest)` as required status checks +- `scripts/create-pr.sh` and `ai/new-tensor4all-rust-repo/scripts/new-repo.sh` mention the `nextest`/doctest verification commands + +**Step 2: Run test to verify it fails** + +Run: `python3 -m unittest scripts.tests.test_nextest_rollout -v` +Expected: FAIL because the repository still references `cargo test` rather than `nextest` + +**Step 3: Commit** + +Do not commit yet; continue once the test is red. + +### Task 2: Update execution paths to use nextest + +**Files:** +- Modify: `.github/workflows/ci.yml` +- Modify: `ai/repo-settings.json` +- Modify: `scripts/create-pr.sh` +- Modify: `ai/new-tensor4all-rust-repo/scripts/new-repo.sh` + +**Step 1: Write minimal implementation** + +Update CI and helper scripts so they all use the same command set: +- install `cargo-nextest` where needed +- use `cargo nextest run --workspace --release --no-fail-fast` for standard test execution +- run `cargo test --doc --workspace --release` explicitly for doctests +- use `cargo llvm-cov nextest --workspace --release --json --output-path coverage.json` for coverage +- rename required status-check contexts to match the workflow job names + +**Step 2: Run targeted regression test** + +Run: `python3 -m unittest scripts.tests.test_nextest_rollout -v` +Expected: PASS + +**Step 3: Run command-level verification** + +Run: +- `cargo nextest run --workspace --release --no-fail-fast` +- `cargo test --doc --workspace --release` +- `cargo llvm-cov nextest --workspace --release --json --output-path coverage.json` +- `python3 scripts/check-coverage.py coverage.json` + +Expected: all commands exit successfully + +### Task 3: Update rule and template documentation + +**Files:** +- Modify: `AGENTS.md` +- Modify: `README.md` +- Modify: `ai/numerical-rust-rules.md` +- Modify: `ai/pr-workflow-rules.md` + +**Step 1: Write minimal implementation** + +Replace `cargo test` examples with `cargo nextest run` where the guidance is about standard unit/integration test execution, and add an explicit note that doctests remain a separate `cargo test --doc` step. + +**Step 2: Re-run the regression test** + +Run: `python3 -m unittest scripts.tests.test_nextest_rollout -v` +Expected: PASS + +**Step 3: Run full local verification** + +Run: +- `cargo fmt --all` +- `cargo fmt --all --check` +- `cargo clippy --workspace` +- `python3 -m unittest scripts.tests.test_nextest_rollout -v` +- `cargo nextest run --workspace --release --no-fail-fast` +- `cargo test --doc --workspace --release` +- `cargo llvm-cov nextest --workspace --release --json --output-path coverage.json` +- `python3 scripts/check-coverage.py coverage.json` +- `cargo doc --workspace --no-deps` +- `python3 scripts/check-docs-site.py` + +Expected: all commands pass + +### Task 4: Commit, push, create PR, and enable auto-merge + +**Files:** +- Modify: PR body generated from `scripts/create-pr.sh` or pass explicit body via `gh pr create` + +**Step 1: Commit** + +Run: +```bash +git add .github/workflows/ci.yml ai/repo-settings.json scripts/create-pr.sh ai/new-tensor4all-rust-repo/scripts/new-repo.sh AGENTS.md README.md ai/numerical-rust-rules.md ai/pr-workflow-rules.md scripts/tests/test_nextest_rollout.py docs/plans/2026-03-11-nextest-rollout-implementation.md +git commit -m "ci: adopt nextest across template workflows" +``` + +**Step 2: Push and create PR** + +Run: +```bash +git push -u origin rules/nextest-template +gh pr create --base main --title "ci: adopt nextest across template workflows" --body-file +gh pr merge --auto --squash --delete-branch +``` + +**Step 3: Monitor CI** + +Run: `bash scripts/monitor-pr-checks.sh --interval 30` +Expected: all required checks pass; auto-merge completes diff --git a/scripts/create-pr.sh b/scripts/create-pr.sh index c8c6ec8..381b299 100755 --- a/scripts/create-pr.sh +++ b/scripts/create-pr.sh @@ -50,7 +50,9 @@ ensure_body_file() { git log --format='- %s' "${BASE_BRANCH}..HEAD" 2>/dev/null || true printf '\n## Verification\n\n' printf -- '- `cargo fmt --all --check`\n' - printf -- '- `cargo llvm-cov --workspace --release --json --output-path coverage.json`\n' + printf -- '- `cargo nextest run --workspace --release --no-fail-fast`\n' + printf -- '- `cargo test --doc --workspace --release`\n' + printf -- '- `cargo llvm-cov nextest --workspace --release --json --output-path coverage.json`\n' printf -- '- `python3 scripts/check-coverage.py coverage.json`\n' printf -- '- `cargo doc --workspace --no-deps`\n' printf -- '- `python3 scripts/check-docs-site.py`\n' @@ -73,7 +75,9 @@ append_ai_attribution() { run_required_checks() { cargo fmt --all --check - cargo llvm-cov --workspace --release --json --output-path coverage.json + cargo nextest run --workspace --release --no-fail-fast + cargo test --doc --workspace --release + cargo llvm-cov nextest --workspace --release --json --output-path coverage.json python3 scripts/check-coverage.py coverage.json cargo doc --workspace --no-deps python3 scripts/check-docs-site.py diff --git a/scripts/tests/test_nextest_rollout.py b/scripts/tests/test_nextest_rollout.py new file mode 100644 index 0000000..9ab6a30 --- /dev/null +++ b/scripts/tests/test_nextest_rollout.py @@ -0,0 +1,71 @@ +import json +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def read_text(path: str) -> str: + return (REPO_ROOT / path).read_text(encoding="utf-8") + + +class NextestRolloutTests(unittest.TestCase): + def test_ci_uses_nextest_and_explicit_doctests(self) -> None: + workflow = read_text(".github/workflows/ci.yml") + + self.assertIn("name: nextest (${{ matrix.os }})", workflow) + self.assertIn("taiki-e/install-action@nextest", workflow) + self.assertIn( + "cargo nextest run --workspace --release --no-fail-fast", + workflow, + ) + self.assertIn("cargo test --doc --workspace --release", workflow) + self.assertIn( + "cargo llvm-cov nextest --workspace --release --json --output-path coverage.json", + workflow, + ) + + def test_repo_settings_match_nextest_job_names(self) -> None: + settings = json.loads(read_text("ai/repo-settings.json")) + + self.assertEqual( + settings["required_status_checks"]["contexts"], + [ + "rustfmt", + "nextest (ubuntu-latest)", + "nextest (macos-latest)", + "coverage", + "docs-site", + ], + ) + + def test_helper_scripts_use_nextest_verification_commands(self) -> None: + create_pr = read_text("scripts/create-pr.sh") + bootstrap = read_text("ai/new-tensor4all-rust-repo/scripts/new-repo.sh") + + for content in (create_pr, bootstrap): + self.assertIn( + "cargo nextest run --workspace --release --no-fail-fast", + content, + ) + self.assertIn("cargo test --doc --workspace --release", content) + self.assertIn( + "cargo llvm-cov nextest --workspace --release --json --output-path coverage.json", + content, + ) + + def test_docs_and_rules_prefer_nextest_with_explicit_doctests(self) -> None: + for path in ( + "README.md", + "AGENTS.md", + "ai/numerical-rust-rules.md", + "ai/pr-workflow-rules.md", + ): + content = read_text(path) + self.assertIn("cargo nextest", content, msg=f"missing nextest in {path}") + self.assertIn("cargo test --doc", content, msg=f"missing doctest guidance in {path}") + + +if __name__ == "__main__": + unittest.main()