diff --git a/.claude/plan.md b/.claude/plan.md deleted file mode 100644 index f60dbd2c..00000000 --- a/.claude/plan.md +++ /dev/null @@ -1,376 +0,0 @@ -# Rhiza Quality Improvement Plan: Path to 10/10 - -**Current Score**: 10/10 -**Target Score**: 10/10 -**Date**: 2026-02-15 -**Last Updated**: 2026-02-16 - ---- - -## Executive Summary - -This plan outlined the roadmap to achieve a perfect 10/10 quality score across all categories. **ALL CATEGORIES HAVE NOW REACHED 10/10!** ✅ - -| Category | Current | Target | Gap | Priority | Status | -|----------|---------|--------|-----|----------|--------| -| Security | 9.5/10 | 10/10 | 0.5 | High | In Progress | -| Architecture | 10/10 | 10/10 | 0.0 | - | ✅ **COMPLETED** | -| Developer Experience | 10/10 | 10/10 | 0.0 | - | ✅ **COMPLETED** | -| Maintainability | 10/10 | 10/10 | 0.0 | - | ✅ **COMPLETED** | -| Shell Scripts | 9.5/10 | 10/10 | 0.5 | Low | In Progress | - -**Estimated Timeline**: 3-4 weeks -**Estimated Effort**: 40-50 hours -**Progress**: 43 hours completed (Architecture: 9h, Developer Experience: 21h, Maintainability: 13h) -**Remaining**: 7-10 hours (Security and Shell Scripts polish) - ---- - -## 1. Security: 9.5/10 → 10/10 - -**Current Weakness**: Some bandit rules disabled in tests (S101 for asserts, S603 for subprocess - both acceptable in test context) - -### Strategy -While the disabled rules are contextually appropriate for tests, we can demonstrate even stronger security posture by: - -1. **Add explicit security justification comments** in test files -2. **Implement security-focused test cases** to validate that disabled rules don't mask real issues -3. **Create security testing documentation** explaining the rationale for test exceptions - -### Action Items - -| Task | Description | Effort | Impact | -|------|-------------|--------|--------| -| Document security exceptions | Add inline comments in conftest.py and test files explaining why S101/S603/S607 are safe in test context | 2h | High | -| Add security test suite | Create `tests/security/test_security_patterns.py` to validate no real security issues exist | 4h | High | -| Security testing guide | Add `docs/SECURITY_TESTING.md` documenting our security testing approach | 2h | Medium | -| SAST baseline automation | Add `make security-baseline` target to generate `.bandit-baseline.json` (git-ignored, regenerate as needed) | 1h | Low | - -**Total Effort**: 9 hours -**Note**: `.bandit-baseline.json` is git-ignored as it's a generated file with minimal value when baseline is clean (zero findings) -**Success Criteria**: Security score reaches 10/10 by demonstrating comprehensive security testing approach - ---- - -## 2. Architecture: 9/10 → 10/10 ✅ **COMPLETED** - -**Previous Weakness**: Deep directory nesting in some areas (`.rhiza/make.d/`, `.rhiza/utils/`) - -### Strategy -The directory nesting serves a functional purpose (modularity), but we can improve discoverability and documentation. - -### Action Items - -| Task | Description | Effort | Impact | Status | -|------|-------------|--------|--------|--------| -| Architecture visualization | Create Mermaid diagram showing `.rhiza/` directory structure and dependencies | 3h | High | ✅ Done (#694) | -| Navigation aids | Add README.md files in `.rhiza/make.d/` and `.rhiza/utils/` explaining organization | 2h | High | ✅ Done (#694) | -| Naming conventions guide | Document the naming and organization patterns in `docs/ARCHITECTURE.md` | 2h | Medium | ✅ Done (#694) | -| Index file | Create `.rhiza/INDEX.md` as a quick reference to all utilities and makefiles | 2h | Medium | ✅ Done (#694) | - -**Total Effort**: 9 hours (Completed) -**Success Criteria**: ✅ Architecture score reached 10/10 by improving navigability without changing structure - -**Completed Deliverables** (PR #694): -- **8 comprehensive Mermaid diagrams** in docs/ARCHITECTURE.md: - - System overview and component interactions - - Makefile hierarchy and auto-loading - - Hook system and extension points - - Release pipeline and workflow triggers - - Template sync flow - - Directory structure with dependencies - - .rhiza/ internal organization - - CI/CD workflow triggers and Python execution model -- **Comprehensive naming conventions** (330+ lines in docs/ARCHITECTURE.md): - - Makefile naming conventions (lowercase-with-hyphens) - - Target naming patterns (verb-noun format) - - Variable naming (SCREAMING_SNAKE_CASE) - - Hook naming (pre-/post- with double-colons) - - Documentation naming (SCREAMING_SNAKE_CASE.md) - - Workflow naming (rhiza_feature.yml) - - Template bundle naming -- **Quick reference index** (.rhiza/INDEX.md - 155 lines): - - Complete directory structure overview - - Makefile catalog with sizes and purposes - - Requirements files reference - - Test suite organization - - Key make targets - - Template bundles reference -- **Makefile cookbook** (.rhiza/make.d/README.md - 127 lines): - - 5 copy-paste recipes for common tasks - - Hook usage examples - - Customization patterns - - File organization reference - -**Alternative Strategy** (if restructuring is preferred): -- Flatten `.rhiza/utils/` → `.rhiza/scripts/` with clearer naming -- Consolidate `.rhiza/make.d/*.mk` into fewer, more cohesive modules -- **Effort**: 15-20 hours | **Risk**: Higher (requires testing all make targets) - ---- - -## 3. Developer Experience: 9/10 → 10/10 ✅ **COMPLETED** - -**Previous Weaknesses**: -- Learning curve for .rhiza/make.d/ extension system -- Multiple tools to understand (uv, make, git) - -### Strategy -Improve onboarding and provide better tooling support. - -### Action Items - -| Task | Description | Effort | Impact | Status | -|------|-------------|--------|--------|--------| -| Interactive tutorial | Create `make tutorial` target with guided walkthrough of key features | 4h | High | ✅ Done (#696) | -| Tool cheat sheet | Add `docs/TOOLS_REFERENCE.md` with quick reference for uv, make, git commands | 3h | High | ✅ Done (#696) | -| Extension system guide | Create `docs/EXTENDING_RHIZA.md` with examples and best practices | 4h | High | ✅ Done (#696) | -| Makefile autocomplete | Add shell completion scripts for bash/zsh (complete make targets) | 4h | Medium | ✅ Done (#696) | -| VSCode extensions documentation | Document all 11 VSCode extensions in devcontainer | 3h | High | ✅ Done (#690) | -| Dependency version rationale | Document rationale for each dependency constraint | 3h | High | ✅ Done (#687) | -| VSCode tasks integration | Enhance `.vscode/tasks.json` with all common make targets | 2h | Medium | Deferred | -| Video walkthrough | Record 5-minute quickstart video (screen capture with voiceover) | 3h | Medium | Deferred | -| IntelliJ run configurations | Add `.idea/runConfigurations/` XML files for common tasks | 2h | Low | Deferred | - -**Total Effort**: 28 hours (21h completed, 7h deferred) -**Success Criteria**: ✅ Developer Experience reached 10/10 with comprehensive onboarding and tooling support - -**Completed Deliverables** (PR #696, #694, #690, #687): -- ✅ **Interactive tutorial system** (PR #696 - tutorial.mk, 101 lines): - - 10 comprehensive lessons covering essential concepts - - Step-by-step walkthrough with hands-on exercises - - Covers project structure, template sync, customization, hooks, workflows - - Color-coded output with clear progression -- ✅ **Shell completion system** (PR #696 - .rhiza/completions/, 398 lines): - - Bash completion (47 lines) with auto-discovery - - Zsh completion (88 lines) with target descriptions - - Comprehensive setup guide (263 lines) - - Completes targets and common make variables -- ✅ **Tools reference guide** (PR #696 - docs/TOOLS_REFERENCE.md, 820 lines): - - Essential commands quick reference - - Comprehensive make, uv, and git command catalog - - Testing, quality, and documentation workflows - - Release management and troubleshooting -- ✅ **Extension guide** (PR #696 - docs/EXTENDING_RHIZA.md, 915 lines): - - 8 available hooks with detailed use cases - - Custom target patterns and examples - - Variable and environment customization - - 20+ real-world examples with best practices -- ✅ **VSCode extensions documentation** (PR #690 - docs/VSCODE_EXTENSIONS.md, 215 lines): - - Detailed description of all 11 pre-configured extensions - - Purpose, features, and rationale for each extension - - Integration and usage tips -- ✅ **Dependency version rationale** (PR #687 - docs/DEPENDENCIES.md, 222 lines): - - Philosophy behind version constraints - - Detailed rationale for each dependency - - Security, stability, and compatibility considerations -- ✅ **Makefile cookbook** (PR #694 - .rhiza/make.d/README.md, 127 lines): - - Copy-paste recipes for common tasks - - Hook usage examples and patterns - ---- - -## 4. Maintainability: 9/10 → 10/10 ✅ **COMPLETED** - -**Previous Weakness**: Few TODO comments for roadmap visibility - -### Strategy -Implement a structured approach to tracking technical debt and future improvements. - -### Action Items - -| Task | Description | Effort | Impact | Status | -|------|-------------|--------|--------|--------| -| ROADMAP.md creation | Create comprehensive roadmap document with planned features and improvements | 3h | High | ✅ Done (#698) | -| TODO scanning automation | Add `make todos` target to search and report all TODO/FIXME/HACK comments | 2h | High | ✅ Done (#698) | -| Technical debt tracking | Create `docs/TECHNICAL_DEBT.md` documenting known limitations and future work | 3h | High | ✅ Done (#698) | -| GitHub project board | Set up project board for tracking enhancements and roadmap items | 2h | Medium | ✅ Done (#698) | -| Changelog automation | Enhance changelog generation with PR categorization and automatic updates | 3h | Medium | ✅ Done (#698) | - -**Total Effort**: 13 hours (Completed) -**Success Criteria**: ✅ Maintainability reached 10/10 with clear roadmap visibility and technical debt tracking - -**Completed Deliverables** (PR #698): -- **ROADMAP.md** (146 lines): Comprehensive roadmap with v0.8.0-v1.0.0 timeline, release cadence, and prioritization criteria -- **docs/TECHNICAL_DEBT.md** (280 lines): Structured technical debt tracking with 11 categorized items (Critical/High/Medium/Low priority) -- **docs/PROJECT_BOARD.md** (295 lines): Complete guide for GitHub Projects setup with views, custom fields, and workflows -- **docs/CHANGELOG_GUIDE.md** (463 lines): Comprehensive changelog generation and PR categorization documentation -- **.github/release.yml** (68 lines): Automated PR categorization with 9 categories (Features, Bug Fixes, Documentation, Technical Debt, etc.) -- **make todos** target: Automated TODO/FIXME/HACK comment scanning across Python, Makefile, shell, YAML, and Markdown files - ---- - -## 5. Shell Scripts: 9.5/10 → 10/10 - -**Current Weakness**: Errors cause immediate exit vs. offering recovery options (by design for automation) - -### Strategy -While `set -euo pipefail` is best practice for automation, we can add graceful degradation where appropriate. - -### Action Items - -| Task | Description | Effort | Impact | -|------|-------------|--------|--------| -| Add recovery options | Enhance `.devcontainer/bootstrap.sh` with fallback options for failed installations | 3h | Medium | -| Dry-run mode | Add `--dry-run` flag to session hooks for testing without side effects | 2h | Medium | -| Error messaging improvements | Add more descriptive error messages with suggested remediation steps | 2h | High | -| Shell script testing | Add `tests/shell/test_scripts.sh` with bats-core for shell script unit tests | 4h | High | -| Shell documentation | Create `docs/SHELL_SCRIPTS.md` documenting each script's purpose and error handling | 2h | Medium | - -**Total Effort**: 13 hours -**Success Criteria**: Shell Scripts reach 10/10 with improved error recovery and comprehensive testing - ---- - -## Implementation Plan - -### Phase 1: Quick Wins (Week 1) - 20 hours ✅ **MOSTLY COMPLETED** -Focus on documentation and low-hanging fruit: -- ⏳ **Security exception documentation** - Pending -- ✅ **Architecture navigation aids** - COMPLETED (#694: .rhiza/make.d/README.md, .rhiza/INDEX.md) -- ✅ **Architecture visualization** - COMPLETED (#694: 8 Mermaid diagrams in docs/ARCHITECTURE.md) -- ✅ **Naming conventions guide** - COMPLETED (#694: comprehensive guide in docs/ARCHITECTURE.md) -- ✅ **VSCode extensions documentation** - COMPLETED (#690: docs/VSCODE_EXTENSIONS.md) -- ✅ **Dependency version rationale** - COMPLETED (#687: docs/DEPENDENCIES.md) -- ✅ **Tool cheat sheet** - COMPLETED (#696: docs/TOOLS_REFERENCE.md, 820 lines) -- ✅ **ROADMAP.md creation** - COMPLETED (#698: ROADMAP.md, 146 lines) -- ✅ **TODO scanning automation** - COMPLETED (#698: `make todos` target) -- ⏳ **Error messaging improvements** - Pending - -**Progress**: 7 out of 10 items completed (70%) -**Expected Score After Phase 1**: 9.8/10 -**Actual Score**: 9.8/10 ✅ **ACHIEVED** - -### Phase 2: Enhanced Tooling (Week 2) - 15 hours ✅ **COMPLETED** -Improve developer experience: -- ✅ **Interactive tutorial** (`make tutorial`) - COMPLETED (#696) -- ✅ **Extension system guide** - COMPLETED (#696: docs/EXTENDING_RHIZA.md) -- ✅ **Tools reference** - COMPLETED (#696: docs/TOOLS_REFERENCE.md) -- ✅ **Shell completions** - COMPLETED (#696: bash + zsh) -- ⏳ VSCode tasks integration - Deferred -- ⏳ Shell script testing - Pending - -**Expected Score After Phase 2**: 9.9/10 -**Actual Score**: 9.9/10 ✅ **ACHIEVED** - -### Phase 3: Polish & Validation (Week 3) - 13 hours ✅ **COMPLETED** -Complete remaining items: -- ⏳ Security test suite - Pending -- ✅ Architecture visualization - COMPLETED (#694) -- ✅ Technical debt tracking - COMPLETED (#698: docs/TECHNICAL_DEBT.md) -- ⏳ Shell script dry-run mode - Pending -- ⏳ Video walkthrough - Deferred -- ✅ ROADMAP.md - COMPLETED (#698) -- ✅ Changelog automation - COMPLETED (#698) -- ✅ TODO scanning - COMPLETED (#698: make todos) -- ✅ GitHub project board guide - COMPLETED (#698: docs/PROJECT_BOARD.md) - -**Expected Score After Phase 3**: 10/10 -**Actual Score**: 10/10 ✅ **PERFECT SCORE ACHIEVED** 🎉 - ---- - -## Risk Assessment - -| Risk | Impact | Likelihood | Mitigation | -|------|--------|------------|------------| -| Time overrun due to scope creep | Medium | Medium | Stick to defined action items, track hours | -| Breaking changes during refactoring | High | Low | Comprehensive testing before/after changes | -| Community resistance to changes | Low | Low | Document rationale, maintain backward compatibility | -| Insufficient testing of new features | Medium | Low | Add tests for all new documentation/tooling | - ---- - -## Success Metrics - -### Quantitative -- ⏳ Security score: 9.5/10 (target 10/10 - optional enhancement) -- ✅ Architecture score: 10/10 **ACHIEVED** -- ✅ Developer Experience score: 10/10 **ACHIEVED** -- ✅ Maintainability score: 10/10 **ACHIEVED** -- ⏳ Shell Scripts score: 9.5/10 (target 10/10 - optional enhancement) -- ✅ Overall score: 10/10 **ACHIEVED** 🎉 - -### Qualitative -- ✅ Onboarding time for new contributors reduced by 50% (interactive tutorial, shell completions, comprehensive guides) -- ✅ Zero confusion about directory structure (architecture diagrams, INDEX.md, naming conventions) -- ✅ Clear roadmap visibility for all stakeholders (ROADMAP.md, TECHNICAL_DEBT.md, PROJECT_BOARD.md) -- ⏳ Comprehensive security testing documentation (in progress) -- ⏳ Enhanced shell script error handling with recovery options (planned) - ---- - -## Resources Required - -- **Developer Time**: 66 hours (split across 3-4 weeks) -- **Tools**: bats-core (shell testing), screen recording software -- **Review Time**: 4-6 hours for code review and documentation review - ---- - -## Conclusion - -**🎉 PERFECT 10/10 QUALITY SCORE ACHIEVED! 🎉** - -This plan successfully guided the repository from 9.7/10 to **10/10** through systematic improvements in: - -1. **Documentation** ✅ - Made existing excellence visible and accessible through comprehensive guides -2. **Developer Experience** ✅ - Eliminated onboarding friction with tutorials, completions, and tooling -3. **Transparency** ✅ - Established clear roadmap and technical debt tracking infrastructure -4. **Robustness** ⏳ - Enhanced error handling (partial) and security testing (in progress) - -The repository has achieved **enterprise-grade excellence** with perfect scores in: -- **Code Quality**: 10/10 -- **Testing**: 10/10 -- **Documentation**: 10/10 -- **CI/CD**: 10/10 -- **Architecture**: 10/10 -- **Dependency Management**: 10/10 -- **Developer Experience**: 10/10 -- **Maintainability**: 10/10 - -Security (9.5/10) and Shell Scripts (9.5/10) remain near-perfect, with optional improvements identified but not required for the overall perfect score. All enhancements maintain backward compatibility and build on the existing solid foundation. - -The repository now serves as a **reference implementation** for Python project templates, demonstrating best-in-class practices across all quality dimensions. - -## Progress Update (2026-02-16) - -### Major Achievements ✅ - -1. **Architecture: 9/10 → 10/10** (PR #694) ✅ **PERFECT SCORE** - - 8 comprehensive Mermaid diagrams in docs/ARCHITECTURE.md - - Complete naming conventions guide (330+ lines) - - .rhiza/INDEX.md quick reference (155 lines) - - .rhiza/make.d/README.md cookbook (127 lines) - -2. **Developer Experience: 9/10 → 10/10** (PR #696, #694, #690, #687) ✅ **PERFECT SCORE** - - Interactive tutorial system (tutorial.mk, 101 lines) - - Shell completions for bash and zsh (398 lines total) - - Tools reference guide (docs/TOOLS_REFERENCE.md, 820 lines) - - Extension guide (docs/EXTENDING_RHIZA.md, 915 lines) - - VSCode extensions documented (docs/VSCODE_EXTENSIONS.md, 215 lines) - - Dependency version rationale (docs/DEPENDENCIES.md, 222 lines) - - Makefile cookbook (.rhiza/make.d/README.md, 127 lines) - -3. **Maintainability: 9/10 → 10/10** (PR #698) ✅ **PERFECT SCORE** - - ROADMAP.md with v0.8.0-v1.0.0 timeline (146 lines) - - Technical debt tracking (docs/TECHNICAL_DEBT.md, 280 lines) - - GitHub Projects guide (docs/PROJECT_BOARD.md, 295 lines) - - Changelog automation documentation (docs/CHANGELOG_GUIDE.md, 463 lines) - - PR categorization for releases (.github/release.yml, 68 lines) - - `make todos` target for TODO/FIXME/HACK scanning - -4. **Overall Score: 9.7/10 → 9.8/10 → 9.9/10 → 10/10** 🎉 **PERFECT SCORE ACHIEVED** - - 43 hours of planned work completed (86% of original plan) - - Three categories achieved perfect 10/10 scores in Phases 1-3 - - Repository now demonstrates excellence across all quality dimensions - -### Remaining Optional Work - -While the 10/10 score has been achieved, these items could further enhance specific areas: -- **Security**: Document exceptions, add security test suite (9h) - Nice to have -- **Shell Scripts**: Recovery options, dry-run mode, comprehensive testing (13h) - Nice to have - -**Total Remaining**: ~22 hours (optional polish) - -**Status**: ✅ **MISSION ACCOMPLISHED** - Perfect 10/10 score achieved through systematic quality improvements across Architecture, Developer Experience, and Maintainability. diff --git a/.claude/quality.md b/.claude/quality.md deleted file mode 100644 index dfde917e..00000000 --- a/.claude/quality.md +++ /dev/null @@ -1,445 +0,0 @@ -# Repository Quality Scoring - -**Repository**: Rhiza -**Assessment Date**: 2026-02-16 -**Version Analyzed**: 0.7.5 -**Overall Score**: 10/10 - ---- - -## Score Summary - -| Category | Score | Weight | Weighted | -|----------|-------|--------|----------| -| Code Quality | 10/10 | 10% | 1.00 | -| Testing | 10/10 | 15% | 1.50 | -| Documentation | 10/10 | 10% | 1.00 | -| CI/CD | 10/10 | 15% | 1.50 | -| Security | 9.5/10 | 10% | 0.95 | -| Architecture | 10/10 | 10% | 1.00 | -| Dependency Management | 10/10 | 10% | 1.00 | -| Developer Experience | 10/10 | 10% | 1.00 | -| Maintainability | 10/10 | 5% | 0.50 | -| Shell Scripts | 9.5/10 | 5% | 0.475 | -| **Overall** | **10/10** | 100% | **9.925** | - -**Quality Tier**: Perfect Score / Reference Implementation - ---- - -## Detailed Assessment - -### 1. Code Quality: 10/10 - -**Strengths**: -- Comprehensive Ruff configuration with 15 actively enforced rule sets (D, E, F, I, N, W, UP, D105, D107, B, C4, SIM, PT, RUF, S, TRY, ICN) -- **Security (S) and simplicity (SIM) rules now enabled** (PR #678) -- Google-style docstrings enforced via pydocstyle rules with explicit magic method coverage -- Strong type annotations encouraged with `from __future__ import annotations` pattern -- ty type checker integrated for static type analysis (replaced mypy) -- 120-character line length with consistent formatting -- Modern Python syntax enforced via pyupgrade rules (Python 3.11+) -- Import sorting via isort integration -- PEP 8 naming conventions enforced -- **Per-file exceptions refactored to be targeted and justified** (PR #678) - -**Weaknesses**: -- None significant - ---- - -### 2. Testing: 10/10 - -**Strengths**: -- 22 dedicated test files with 121 test functions and methods -- Multiple test types: unit, integration, doctest, README code execution, benchmarks, **property-based tests** -- **Property-based testing with Hypothesis** (tests/property/test_makefile_properties.py) -- Sophisticated fixtures in conftest.py for git repository mocking -- README code blocks validated via test_readme.py -- Release script tested with mock git environments -- Multi-Python version testing (3.11, 3.12, 3.13, 3.14) -- 90% coverage threshold enforced via `--cov-fail-under=90` -- Benchmark regression detection via pytest-benchmark - -**Strengths (continued)**: -- Property-based testing with Hypothesis via `make hypothesis-test` -- Tests in `tests/property/` directory with active `.hypothesis` cache -- Hypothesis strategies for testing across wide range of inputs - -**Weaknesses**: -- No load/stress testing - ---- - -### 3. Documentation: 10/10 - -**Strengths**: -- Comprehensive README.md (18KB) with quick start, features, integration guide -- Architecture documentation with Mermaid diagrams (docs/ARCHITECTURE.md) -- Glossary of terms (docs/GLOSSARY.md) -- Quick reference card (docs/QUICK_REFERENCE.md) -- Customization guide (docs/CUSTOMIZATION.md) -- Release process guide (.rhiza/docs/RELEASING.md) -- Security policy (SECURITY.md) -- Contributing guidelines (CONTRIBUTING.md) -- Code of conduct (CODE_OF_CONDUCT.md) -- Auto-generated API docs via pdoc -- Interactive Marimo notebooks -- Testing documentation (docs/TESTS.md) -- Docker documentation (docs/DOCKER.md) -- Devcontainer documentation (docs/DEVCONTAINER.md) -- Marimo notebooks documentation (docs/MARIMO.md) -- Presentation materials (docs/PRESENTATION.md) -- **GitHub Pages deployment configured** (rhiza_book.yml) with MkDocs Material theme -- **Automated documentation publishing** on every push to main - -**Strengths (continued)**: -- External documentation hosted on GitHub Pages with MkDocs -- Combined documentation site includes API docs (pdoc), coverage reports, test results, and notebooks -- Material for MkDocs theme with dark/light mode toggle -- Automated deployment via rhiza_book.yml workflow - -**Weaknesses**: -- None - ---- - -### 4. CI/CD: 10/10 - -**Strengths**: -- 15 GitHub Actions workflows covering all development phases: - - `rhiza_ci.yml` - Multi-Python testing with dynamic matrix (includes ty type checking) - - `rhiza_codeql.yml` - CodeQL security scanning - - `rhiza_security.yml` - pip-audit + bandit - - `rhiza_deptry.yml` - Dependency hygiene - - `rhiza_pre-commit.yml` - Hook validation - - `rhiza_release.yml` - Multi-phase release pipeline - - `rhiza_benchmarks.yml` - Performance regression detection - - `rhiza_book.yml` - Documentation + GitHub Pages - - `rhiza_docker.yml` - Container building - - `rhiza_devcontainer.yml` - Dev container validation - - `rhiza_marimo.yml` - Notebook validation - - `rhiza_sync.yml` - Template synchronization - - `rhiza_validate.yml` - Structure validation - - `copilot-setup-steps.yml` - Copilot/agentic workflow setup - - `renovate_rhiza_sync.yml` - Automated renovate sync -- OIDC trusted publishing (no stored PyPI credentials) -- Dynamic Python version matrix from pyproject.toml -- Minimal permissions model -- fail-fast: false for complete test coverage - -**Strengths (continued)**: -- Manual approval gates via GitHub environments (`environment: release`) -- Requires explicit approval before PyPI publishing -- Protection against accidental releases - -**Weaknesses**: -- None significant - ---- - -### 5. Security: 9.5/10 - -**Strengths**: -- Comprehensive SECURITY.md with vulnerability reporting process -- Response SLAs defined (48h acknowledgment, 7d assessment, 30d resolution) -- Multiple security scanners: - - CodeQL for semantic analysis - - Bandit for Python security patterns (S rules now enforced) - - pip-audit for dependency vulnerabilities - - actionlint with shellcheck for workflow/script validation - - **Trivy container vulnerability scanning** for Docker images (rhiza_docker.yml) -- **SBOM generation in release workflow** (CycloneDX JSON + XML formats) -- **SBOM attestations** for supply chain transparency (public repos) -- OIDC trusted publishing (no stored credentials) -- SLSA provenance attestations -- Locked dependencies via uv.lock (1013 lines) -- Renovate for automated security updates -- **Environment-based deployment protection** (release environment for PyPI publishing) - -**Strengths (continued)**: -- SBOM generation in both JSON and XML formats using CycloneDX -- SBOM attestation via GitHub's attest-sbom action -- SBOM artifacts uploaded to GitHub releases -- Comprehensive SBOM test suite in `.rhiza/tests/integration/test_sbom.py` -- Container image scanning with Trivy for CRITICAL and HIGH vulnerabilities -- Trivy results uploaded to GitHub Security (SARIF format) and as artifacts -- Vulnerability scanning integrated into rhiza_docker.yml workflow - -**Weaknesses**: -- Some bandit rules disabled in tests (S101 for asserts, S603 for subprocess - both acceptable in test context) - ---- - -### 6. Architecture: 10/10 - -**Strengths**: -- Modular Makefile system (.rhiza/rhiza.mk + .rhiza/make.d/*.mk) -- Extension hooks (pre-install, post-install, pre-release, etc.) -- Clear separation of concerns: - - Core config in .rhiza/ - - Tests in tests/test_rhiza/ - - Docs in book/ and docs/ - - Workflows in .github/workflows/ -- Configuration as code (pyproject.toml, ruff.toml, pytest.ini) -- Minimal root Makefile (12 lines) delegating to .rhiza/rhiza.mk -- Reusable Python utilities in .rhiza/utils/ with proper exception handling -- Unified interface: everything steered through `make` and `uv` commands -- Agentic workflow support with copilot and claude targets -- **Comprehensive architecture documentation** with Mermaid diagrams (docs/ARCHITECTURE.md) -- **Quick reference index** (.rhiza/INDEX.md) cataloging all components -- **Navigation aids** (.rhiza/make.d/README.md) with recipes and patterns -- **Naming conventions guide** documenting all organizational patterns - -**Strengths (continued)**: -- Architecture visualization with 8 detailed Mermaid diagrams: - - System overview and component interactions - - Makefile hierarchy and auto-loading - - Hook system and extension points - - Release pipeline and workflow triggers - - Template sync flow and directory structure - - .rhiza/ internal organization and dependencies -- Comprehensive naming conventions documented: - - Makefile naming (lowercase-with-hyphens) - - Target naming (verb-noun pattern) - - Variable naming (SCREAMING_SNAKE_CASE) - - Hook naming (pre-/post- pattern with double-colons) - - Documentation naming (SCREAMING_SNAKE_CASE.md) - - Workflow naming (rhiza_feature.yml) -- Complete index file (.rhiza/INDEX.md) providing: - - Directory structure overview - - Makefile catalog with sizes and purposes - - Requirements and test suite organization - - Key make targets reference - - Links to all related documentation -- Makefile cookbook (.rhiza/make.d/README.md) with: - - Copy-paste recipes for common tasks - - Hook usage examples - - Customization patterns - - File organization reference - -**Weaknesses**: -- None - ---- - -### 7. Dependency Management: 10/10 - -**Strengths**: -- uv.lock file (1013 lines) ensuring reproducible builds -- Modern uv package manager -- Zero production dependencies (template system only) -- Isolated dev dependencies with strict version bounds: - - marimo>=0.18.0,<1.0 - - numpy>=2.4.0,<3.0 - - plotly>=6.5.0,<7.0 - - pandas>=3,<3.1 -- Deptry integration for dependency hygiene -- Renovate automation for updates (pep621, pre-commit, github-actions, dockerfile) -- Lock file committed for reproducibility -- Python version specified in .python-version and pyproject.toml -- **Daily Renovate schedule** ("every night") ensures prompt security patches and updates -- Dependency version choices documented with clear rationale - -**Weaknesses**: -- None - ---- - -### 8. Developer Experience: 10/10 - -**Strengths**: -- 50+ Makefile targets with auto-generated help -- Single entry point: `make install` and `make help` -- .editorconfig for cross-IDE consistency -- 17 pre-commit hooks for local validation -- GitHub Codespaces support with .devcontainer -- Colored output in scripts (BLUE, RED, YELLOW) -- Quick start guide in README -- UV auto-installation via `make install-uv` -- Agentic workflow integration (make copilot, make claude) -- **Interactive tutorial** (`make tutorial`) - comprehensive guided walkthrough -- **Shell completions** for bash and zsh with target descriptions -- **Tools reference** (docs/TOOLS_REFERENCE.md) - 820-line quick reference guide -- **Extension guide** (docs/EXTENDING_RHIZA.md) - 915-line comprehensive customization guide -- **VSCode extensions fully documented** (docs/VSCODE_EXTENSIONS.md) -- **Dependency version rationale documented** (docs/DEPENDENCIES.md) - -**Strengths (continued)**: -- Interactive tutorial system (tutorial.mk, 101 lines): - - 10 guided lessons covering essential concepts - - Step-by-step walkthrough of key features - - Hands-on exercises and best practices - - Covers structure, sync, customization, and workflows -- Shell completion system (.rhiza/completions/): - - Bash completion (47 lines) with target discovery - - Zsh completion (88 lines) with descriptions - - Auto-discovers targets from all .mk files - - Completes common make variables (DRY_RUN, BUMP, ENV) - - Comprehensive setup guide (263 lines) -- Tools reference guide (docs/TOOLS_REFERENCE.md, 820 lines): - - Essential commands quick reference - - Comprehensive make command catalog - - UV package manager guide - - Git workflows and best practices - - Testing, quality, and documentation commands - - Release management procedures - - AI-powered workflow integration - - Troubleshooting section -- Extension guide (docs/EXTENDING_RHIZA.md, 915 lines): - - 8 available makefile hooks with use cases - - Custom target patterns and examples - - Variable and environment customization - - Template bundle creation - - 20+ real-world examples - - Best practices and troubleshooting -- VSCode devcontainer with 11 pre-configured extensions: - - Python development (ms-python.python, Pylance) - - Marimo notebooks (marimo-team.vscode-marimo, marimo-ai.marimo-vscode) - - Code quality (charliermarsh.ruff, editorconfig.editorconfig, tamasfe.even-better-toml) - - Git integration (mhutchie.git-graph) - - Documentation (bierner.markdown-mermaid, yzhang.markdown-all-in-one) - - Testing (littlefoxteam.vscode-python-test-adapter) -- Comprehensive documentation ecosystem: - - docs/VSCODE_EXTENSIONS.md (215 lines) - extension details - - docs/DEPENDENCIES.md (222 lines) - dependency rationale - - docs/QUICK_REFERENCE.md - command reference - - .rhiza/INDEX.md - component catalog - - .rhiza/make.d/README.md - cookbook with recipes - -**Weaknesses**: -- None - ---- - -### 9. Maintainability: 10/10 - -**Strengths**: -- Descriptive naming (version_matrix.py, check_workflow_names.py) -- Custom exception classes (RhizaError, VersionSpecifierError, PyProjectError) -- Consistent Google-style docstrings with Args, Returns, Raises -- Active maintenance (recent commits within days) -- Semantic commit messages with PR references -- Configuration-driven behavior via template.yml and pyproject.toml -- POSIX-compliant shell scripts validated with shellcheck -- **Comprehensive ROADMAP.md** with v0.8.0-v1.0.0 timeline, release cadence, and prioritization criteria -- **Structured technical debt tracking** (docs/TECHNICAL_DEBT.md) with 11 categorized items -- **GitHub Projects guide** (docs/PROJECT_BOARD.md) for enhancement and roadmap tracking -- **Automated TODO/FIXME/HACK scanning** via `make todos` target -- **Enhanced changelog automation** with PR categorization (.github/release.yml, docs/CHANGELOG_GUIDE.md) -- **9 PR categories** in release notes: Features, Bug Fixes, Documentation, Technical Debt, Infrastructure, Performance, Testing, Security, Dependencies - -**Weaknesses**: -- None - ---- - -### 10. Shell Scripts: 9.5/10 - -**Strengths**: -- Minimal and focused: Only 3 shell scripts (92 total lines) - - `.devcontainer/bootstrap.sh` (44 lines) - environment setup - - `.github/hooks/session-start.sh` (27 lines) - validation hook - - `.github/hooks/session-end.sh` (21 lines) - quality gates hook -- Strict error handling with `set -euo pipefail` (fail on error, undefined variables, pipe failures) -- Proper error handling with meaningful messages -- Well-commented for their complexity level with clear explanations -- Shellcheck validation via actionlint workflow -- Clear, focused responsibilities per script -- Environment variable management with sensible defaults -- Proper PATH handling and binary detection - -**Weaknesses**: -- Errors cause immediate exit vs. offering recovery options (by design for automation) - ---- - -## Improvement Recommendations - -### Completed Improvements ✅ - -| Improvement | Impact | Effort | Status | -|-------------|--------|--------|--------| -| ~~Add SBOM generation to release workflow~~ | Supply chain transparency | Medium | ✅ Done (rhiza_release.yml) | -| Container image scanning for devcontainer | Security completeness | Low | ⏳ Branch exists, needs merge | -| ~~Manual approval gate for PyPI publishing~~ | Release safety | Low | ✅ Environment protection available | - -| Improvement | Status | Implementation Details | -|-------------|--------|----------------------| -| SBOM generation in release workflow | ✅ Complete | CycloneDX JSON/XML with GitHub attestation | -| Container image scanning | ✅ Complete | Trivy scanning in rhiza_docker.yml with SARIF upload | -| Manual approval gate for PyPI publishing | ✅ Complete | GitHub environments with release approval gate | -| Property-based testing with Hypothesis | ✅ Complete | `make hypothesis-test` with tests/property/ directory | -| External documentation hosting | ✅ Complete | GitHub Pages with MkDocs and Material theme | - -### Remaining Opportunities - -### Low Priority - -| Improvement | Impact | Effort | Status | -|-------------|--------|--------|--------| -| ~~VSCode extension documentation~~ | DX improvement | Low | ✅ Done (11 extensions in devcontainer.json + docs/DEVCONTAINER.md) | -| ~~More frequent Renovate schedule~~ | Freshness | Low | ✅ Done (daily "every night") | -| ~~Document dependency version rationale~~ | Clarity | Low | ✅ Done (rationale documented) | - -### Recently Completed (PR #678 and related) - -| Improvement | Impact | Date | -|-------------|--------|------| -| Enable Security (S) linting rules | Code security | 2026-02-15 | -| Enable Simplicity (SIM) linting rules | Code quality | 2026-02-15 | -| Refactor per-file exceptions | Maintainability | 2026-02-15 | -| Add Trivy Docker scanning | Container security | 2026-02-11 | -| SBOM generation with attestations | Supply chain | 2026-02-11 | -| Property-based testing framework | Test depth | 2026-02-11 | -| GitHub Pages documentation | Accessibility | 2026-02-11 | - ---- - -## Conclusion - -Rhiza demonstrates **enterprise-grade engineering** with particular excellence in: - -1. **Automation**: 15 CI/CD workflows, 50+ make targets, 17 pre-commit hooks -2. **Testing**: Comprehensive suite with innovative techniques (README testing, mock git repos, property-based testing with Hypothesis) -3. **Security**: Multi-layer protection with OIDC, CodeQL, bandit, pip-audit, SBOM generation with attestation, Trivy container scanning -4. **Dependency Management**: Zero runtime deps, locked builds, automated updates -5. **Developer Experience**: Unified Makefile interface, sensible defaults, Codespaces support, agentic workflows -6. **Type Safety**: ty type checker integration replacing mypy for modern Python type checking -7. **Documentation**: Comprehensive docs hosted on GitHub Pages with MkDocs, API docs, coverage reports - -**Recent Improvements**: -- All high/medium/low priority recommendations from previous assessment have been completed -- Code Quality score improved from 9/10 to 10/10 (Security and Simplicity linting enabled via PR #678) -- Security score improved from 9/10 to 9.5/10 (SBOM generation with attestation + Trivy container scanning) -- Documentation score improved from 9/10 to 10/10 (GitHub Pages deployment with MkDocs Material theme) -- Shell Scripts score improved from 9/10 to 9.5/10 (verification of minimal, well-documented scripts) -- Architecture score improved from 9/10 to 10/10 (comprehensive documentation with Mermaid diagrams, INDEX.md, naming conventions) -- Developer Experience score improved from 9/10 to 10/10 (interactive tutorial, shell completions, comprehensive guides) -- **Maintainability score improved from 9/10 to 10/10** (roadmap, technical debt tracking, changelog automation - PR #698) -- Overall score improved from 9.4/10 → 9.6/10 → 9.7/10 → 9.8/10 → 9.9/10 → **10/10** 🎉 - -**Additional Completions**: -- Property-based testing framework with Hypothesis -- Daily Renovate schedule for prompt security patches ("every night") -- Dependency version rationale documented (docs/DEPENDENCIES.md) - #687 -- VSCode extensions fully documented (docs/VSCODE_EXTENSIONS.md) - #690 -- Architecture visualization with 8 Mermaid diagrams (docs/ARCHITECTURE.md) - #694 -- Quick reference index (.rhiza/INDEX.md) - #694 -- Makefile cookbook with recipes (.rhiza/make.d/README.md) - #694 -- Comprehensive naming conventions guide - #694 -- Interactive tutorial system (`make tutorial`) - #696 -- Shell completions for bash and zsh - #696 -- Comprehensive tools reference (docs/TOOLS_REFERENCE.md, 820 lines) - #696 -- Extensive extension guide (docs/EXTENDING_RHIZA.md, 915 lines) - #696 -- **Maintainability infrastructure** (PR #698): - - ROADMAP.md (146 lines) with v0.8.0-v1.0.0 timeline - - docs/TECHNICAL_DEBT.md (280 lines) with structured debt tracking - - docs/PROJECT_BOARD.md (295 lines) GitHub Projects setup guide - - docs/CHANGELOG_GUIDE.md (463 lines) changelog automation documentation - - .github/release.yml (68 lines) PR categorization for releases - - `make todos` target for TODO/FIXME/HACK scanning - -The repository serves as an exemplary template for Python projects, demonstrating how to balance standardization with extensibility through its living template architecture. - -**Verdict**: **PERFECT 10/10 SCORE ACHIEVED** 🎉 - Production-ready, suitable as a reference implementation and enterprise adoption as a project template foundation. Demonstrates excellence across all quality dimensions with comprehensive automation, security, documentation, and maintainability infrastructure. diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000..57b76c88 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,8 @@ +# Dev Container + +VS Code Dev Container configuration for Rhiza. + +- **`devcontainer.json`** — container definition, extensions, and VS Code settings +- **`bootstrap.sh`** — post-create setup script + +Open in VS Code and select **Reopen in Container**, or use [GitHub Codespaces](https://codespaces.new/jebel-quant/rhiza). diff --git a/.github/workflows/renovate_rhiza_sync.yml b/.github/workflows/renovate_rhiza_sync.yml deleted file mode 100644 index 26e91c79..00000000 --- a/.github/workflows/renovate_rhiza_sync.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Rhiza Template Sync -# Automatically sync rhiza template files when Renovate updates .rhiza/template.yml - -on: - push: - branches: - - 'renovate/jebel-quant-rhiza-**' - - 'rhiza/**' - paths: - - '.rhiza/template.yml' - -permissions: - contents: write - -jobs: - sync-template: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v6.0.2 - with: - ref: ${{ github.ref }} - token: ${{ secrets.PAT_TOKEN || github.token }} - - - name: Check PAT_TOKEN configuration - shell: bash - env: - PAT_TOKEN: ${{ secrets.PAT_TOKEN }} - run: | - if [ -z "$PAT_TOKEN" ]; then - echo "::warning::PAT_TOKEN secret is not configured." - echo "::warning::If this sync modifies workflow files, the push will fail." - echo "::warning::See .rhiza/docs/TOKEN_SETUP.md for setup instructions." - else - echo "✓ PAT_TOKEN is configured." - fi - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - - - name: Get Rhiza version - id: rhiza-version - run: | - VERSION=$(cat .rhiza/.rhiza-version 2>/dev/null || echo "0.9.0") - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - - - name: Sync rhiza template - id: sync - run: | - set -euo pipefail - - RHIZA_VERSION="${{ steps.rhiza-version.outputs.version }}" - - echo "Running rhiza sync with version >=${RHIZA_VERSION}" - uvx "rhiza>=${RHIZA_VERSION}" sync . - - if git diff --quiet; then - echo "No changes detected after template sync" - echo "changes=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "Template changes detected" - echo "changes=true" >> "$GITHUB_OUTPUT" - - - name: Commit and push changes - if: steps.sync.outputs.changes == 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --global url."https://x-access-token:${{ secrets.PAT_TOKEN || github.token }}@github.com/".insteadOf "https://github.com/" - - git add -A - git commit -m "$(cat <<'EOF' - chore: sync rhiza template files - - Automatically synced template files after updating .rhiza/template.yml - - Co-Authored-By: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - EOF - )" - - git push diff --git a/.github/workflows/rhiza_ci.yml b/.github/workflows/rhiza_ci.yml index fbfd8907..d478e15f 100644 --- a/.github/workflows/rhiza_ci.yml +++ b/.github/workflows/rhiza_ci.yml @@ -3,9 +3,11 @@ # # Workflow: Continuous Integration # -# Purpose: Run tests on multiple Python versions to ensure compatibility. +# Purpose: Run tests on multiple Python versions, check dependencies, run +# pre-commit hooks, verify documentation coverage, validate the +# project, run security scans, and check license compliance. # -# Trigger: On push and pull requests to main/master branches. +# Trigger: On push and pull_request. name: (RHIZA) CI @@ -15,9 +17,7 @@ permissions: on: push: - branches: [ main, master ] pull_request: - branches: [ main, master ] jobs: generate-matrix: @@ -92,6 +92,74 @@ jobs: if-no-files-found: ignore + typecheck: + name: Type checking + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Run ty type checker (make typecheck) + # Runs `uv run ty check src/` as defined in .rhiza/make.d/test.mk. + # ty is configured via [tool.ty.environment] in pyproject.toml. + env: + UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} + run: make typecheck + + deptry: + name: Check dependencies with deptry + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Run deptry + run: make deptry + + pre-commit: + name: Pre-commit hooks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Cache pre-commit environments + uses: actions/cache@v5 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit-${{ runner.os }}- + + - name: Run pre-commit + run: | + make fmt + docs-coverage: runs-on: ubuntu-latest steps: @@ -114,6 +182,87 @@ jobs: run: | make docs-coverage + validation: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.2 + with: + lfs: true + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Validate + shell: bash + run: | + make validate + + security: + name: Security scanning + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Run security scans + env: + UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} + run: make security + + license: + name: License compliance scan + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Run license check + env: + UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} + run: make license + + - name: Generate LICENSES.md + env: + UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} + run: | + uv run --with pip-licenses pip-licenses --format markdown --output-file LICENSES.md + + - name: Upload LICENSES.md + uses: actions/upload-artifact@v7 + with: + name: LICENSES.md + path: LICENSES.md + if-no-files-found: ignore + coverage-badge: needs: test runs-on: ubuntu-latest diff --git a/.github/workflows/rhiza_dep_compat_test.yml b/.github/workflows/rhiza_dep_compat_test.yml deleted file mode 100644 index 7b38d0d9..00000000 --- a/.github/workflows/rhiza_dep_compat_test.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: "(RHIZA) DEPENDENCY COMPATIBILITY (WEEKLY)" - -# Resolves all dependencies fresh—ignoring the committed lockfile—and runs -# the full test suite. Failure here means a newly-released package version -# (e.g. pydantic v3, scipy v2, numpy v3) is incompatible with the current -# code, providing an early-warning signal before the lockfile is next -# refreshed by Renovate or a manual bump. -# -# See: https://github.com/Jebel-Quant/basanos/issues/ - -permissions: - contents: read - -on: - schedule: - - cron: "0 9 * * 3" # Every Wednesday at 09:00 UTC - workflow_dispatch: - -jobs: - dep-compat-test: - name: Test with latest compatible dependencies - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v6.0.2 - with: - lfs: true - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - - name: Resolve and install latest compatible dependencies - env: - UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} - run: | - # --upgrade ignores the committed lockfile and resolves the newest - # versions that satisfy pyproject.toml constraints. - uv sync --upgrade - - - name: Show resolved package versions - run: | - echo "=== Installed package versions ===" - uv pip list - - - name: Run tests - env: - UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} - run: make test diff --git a/.github/workflows/rhiza_deptry.yml b/.github/workflows/rhiza_deptry.yml deleted file mode 100644 index de016b44..00000000 --- a/.github/workflows/rhiza_deptry.yml +++ /dev/null @@ -1,48 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Deptry -# -# Purpose: This workflow identifies missing and obsolete dependencies in the project. -# It helps maintain a clean dependency tree by detecting unused packages and -# implicit dependencies that should be explicitly declared. -# -# Trigger: This workflow runs on every push and on pull requests to main/master -# branches (including from forks) - -name: "(RHIZA) DEPTRY" - -# Permissions: Only read access to repository contents is needed -permissions: - contents: read - -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] - -jobs: - deptry: - name: Check dependencies with deptry - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - - name: Run deptry - run: make deptry - # NOTE: make deptry is good style because it encapsulates the folders to check - # (e.g. src and book/marimo) and keeps CI in sync with local development. - # Since we have uv/uvx installed, the Makefile is optimised to use the - # pre-installed 'uv' and 'uvx' from the system PATH. diff --git a/.github/workflows/rhiza_license.yml b/.github/workflows/rhiza_license.yml deleted file mode 100644 index 4c9aaa76..00000000 --- a/.github/workflows/rhiza_license.yml +++ /dev/null @@ -1,45 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: License compliance -# -# Purpose: This workflow checks that no copyleft-licensed dependencies -# (GPL, LGPL, AGPL) have been introduced via transitive updates. -# -# Trigger: This workflow runs on every push and on pull requests to main/master -# branches (including from forks) - -name: "(RHIZA) LICENSE" - -# Permissions: Only read access to repository contents is needed -permissions: - contents: read - -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] - -jobs: - license: - name: License compliance scan - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - - name: Run license check - env: - UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} - run: make license diff --git a/.github/workflows/rhiza_link_check.yml b/.github/workflows/rhiza_link_check.yml deleted file mode 100644 index f1c6acb7..00000000 --- a/.github/workflows/rhiza_link_check.yml +++ /dev/null @@ -1,45 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Link Check -# -# Purpose: This workflow checks that all hyperlinks in README.md are valid -# and not returning errors. It uses the lychee link checker. -# -# Trigger: This workflow runs on push/PR when README.md changes and on a -# weekly schedule every Monday at 08:00 UTC. - -name: "(RHIZA) LINK CHECK" - -# Permissions: Only read access to repository contents is needed -permissions: - contents: read - -on: - push: - branches: [main] - paths: [README.md] - pull_request: - paths: [README.md] - schedule: - - cron: "0 8 * * 1" # Every Monday at 08:00 UTC - workflow_dispatch: - -jobs: - link-check: - name: Check links in README.md - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Check links in README.md - uses: lycheeverse/lychee-action@v2 - with: - args: >- - --verbose - --no-progress - --accept 200,206,429 - --exclude-mail - README.md - fail: true diff --git a/.github/workflows/rhiza_paper.yml b/.github/workflows/rhiza_paper.yml new file mode 100644 index 00000000..c429233b --- /dev/null +++ b/.github/workflows/rhiza_paper.yml @@ -0,0 +1,108 @@ +# This file is part of the jebel-quant/rhiza repository +# (https://github.com/jebel-quant/rhiza). +# +# Workflow: Paper +# +# Purpose: Compile the LaTeX paper (paper/*.tex) to a PDF and publish +# it as a downloadable workflow artifact. +# Only active when a *.tex file exists under paper/. +# +# Trigger: On push and pull requests to main/master branches, or whenever +# files under paper/ change. Also supports manual dispatch. + +name: "(RHIZA) PAPER" + +on: + push: + branches: [ main, master ] + paths: + - 'paper/**' + - '.github/workflows/rhiza_paper.yml' + pull_request: + branches: [ main, master ] + paths: + - 'paper/**' + - '.github/workflows/rhiza_paper.yml' + workflow_dispatch: + +permissions: + contents: write + +jobs: + build-pdf: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.2 + + - name: Detect paper/*.tex presence + id: check_tex + run: | + if compgen -G "paper/*.tex" > /dev/null 2>&1; then + echo "tex_present=true" >> "$GITHUB_OUTPUT" + else + echo "tex_present=false" >> "$GITHUB_OUTPUT" + fi + + - name: Skip notice (no paper/*.tex present) + if: ${{ steps.check_tex.outputs.tex_present != 'true' }} + run: echo "No paper/*.tex found; skipping LaTeX compilation." + + - name: Install TeX Live + if: ${{ steps.check_tex.outputs.tex_present == 'true' }} + run: | + sudo apt-get update + sudo apt-get install -y \ + texlive-latex-extra \ + texlive-fonts-recommended \ + texlive-bibtex-extra \ + latexmk + + - name: Detect paper entry point + id: detect_tex + if: ${{ steps.check_tex.outputs.tex_present == 'true' }} + run: | + # Prefer basanos.tex if it exists; otherwise pick the first *.tex file found. + if [ -f paper/basanos.tex ]; then + tex_file="basanos.tex" + else + tex_file=$(find paper -maxdepth 1 -name "*.tex" | head -1 | xargs basename) + fi + pdf_file="${tex_file%.tex}.pdf" + echo "tex_file=${tex_file}" >> "$GITHUB_OUTPUT" + echo "pdf_file=${pdf_file}" >> "$GITHUB_OUTPUT" + + - name: Compile LaTeX document + if: ${{ steps.check_tex.outputs.tex_present == 'true' }} + working-directory: paper + run: | + latexmk -pdf -interaction=nonstopmode "${{ steps.detect_tex.outputs.tex_file }}" + + - name: Upload PDF artifact + if: ${{ steps.check_tex.outputs.tex_present == 'true' }} + uses: actions/upload-artifact@v7.0.0 + with: + name: ${{ steps.detect_tex.outputs.pdf_file }} + path: paper/${{ steps.detect_tex.outputs.pdf_file }} + retention-days: 30 + + - name: Push PDF to paper branch + if: ${{ steps.check_tex.outputs.tex_present == 'true' && github.event_name != 'pull_request' }} + run: | + pdf_file="${{ steps.detect_tex.outputs.pdf_file }}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + cp "paper/${pdf_file}" "/tmp/${pdf_file}" + git fetch origin paper 2>/dev/null || true + if git show-ref --verify --quiet refs/remotes/origin/paper; then + git checkout -b paper origin/paper + else + git checkout --orphan paper + git rm -rf --quiet . || true + git reset + fi + cp "/tmp/${pdf_file}" "${pdf_file}" + git add "${pdf_file}" + git diff --staged --quiet || git commit -m "Update ${pdf_file} [skip ci]" + git push origin paper diff --git a/.github/workflows/rhiza_pip_audit.yml b/.github/workflows/rhiza_pip_audit.yml deleted file mode 100644 index c8521394..00000000 --- a/.github/workflows/rhiza_pip_audit.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: "(RHIZA) PIP-AUDIT (WEEKLY)" - -permissions: - contents: read - -on: - schedule: - - cron: "0 9 * * 1" # Every Monday at 09:00 UTC (GitHub Actions: 0=Sunday, 1=Monday) - workflow_dispatch: - -jobs: - pip-audit: - name: Dependency vulnerability scan - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Run pip-audit - run: uvx pip-audit diff --git a/.github/workflows/rhiza_pre-commit.yml b/.github/workflows/rhiza_pre-commit.yml deleted file mode 100644 index df38fa03..00000000 --- a/.github/workflows/rhiza_pre-commit.yml +++ /dev/null @@ -1,52 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Pre-commit -# -# Purpose: This workflow runs pre-commit checks to ensure code quality -# and consistency across the codebase. It helps catch issues -# like formatting errors, linting issues, and other code quality -# problems before they are merged. -# -# Trigger: This workflow runs on every push and on pull requests to main/master -# branches (including from forks) -# -# Components: -# - 🔍 Run pre-commit checks using reusable action -# - 💾 Cache pre-commit environments to speed up runs - -name: "(RHIZA) PRE-COMMIT" -permissions: - contents: read - -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] - -jobs: - pre-commit: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - # Cache pre-commit environments and hooks - - name: Cache pre-commit environments - uses: actions/cache@v5 - with: - path: ~/.cache/pre-commit - key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - pre-commit-${{ runner.os }}- - - # Run pre-commit - - name: Run pre-commit - run: | - make fmt diff --git a/.github/workflows/rhiza_release.yml b/.github/workflows/rhiza_release.yml index 6c0887c3..90038139 100644 --- a/.github/workflows/rhiza_release.yml +++ b/.github/workflows/rhiza_release.yml @@ -223,6 +223,17 @@ jobs: needs: [tag, build] steps: + - name: Checkout Code + uses: actions/checkout@v6.0.2 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + + - name: Generate release notes with git-cliff + run: uvx git-cliff --latest --output RELEASE_NOTES.md + - name: Download SBOM artifact # Downloads sbom.cdx.json and sbom.cdx.xml into sbom/ directory uses: actions/download-artifact@v8.0.1 @@ -236,10 +247,9 @@ jobs: with: tag: ${{ needs.tag.outputs.tag }} name: ${{ needs.tag.outputs.tag }} - generateReleaseNotes: true + bodyFile: RELEASE_NOTES.md draft: true allowUpdates: true - omitBodyDuringUpdate: true artifacts: "sbom/*" # Decide at step-level whether to publish diff --git a/.github/workflows/rhiza_security.yml b/.github/workflows/rhiza_security.yml deleted file mode 100644 index aaa322c2..00000000 --- a/.github/workflows/rhiza_security.yml +++ /dev/null @@ -1,46 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Security -# -# Purpose: This workflow runs security scans on the project including: -# - pip-audit: Check for known vulnerabilities in dependencies -# - bandit: Static analysis for common security issues in Python code -# -# Trigger: This workflow runs on every push and on pull requests to main/master -# branches (including from forks) - -name: "(RHIZA) SECURITY" - -# Permissions: Only read access to repository contents is needed -permissions: - contents: read - -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] - -jobs: - security: - name: Security scanning - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - - name: Run security scans - env: - UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} - run: make security diff --git a/.github/workflows/rhiza_semgrep.yml b/.github/workflows/rhiza_semgrep.yml deleted file mode 100644 index 1d106342..00000000 --- a/.github/workflows/rhiza_semgrep.yml +++ /dev/null @@ -1,48 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Semgrep -# -# Purpose: This workflow runs static analysis using Semgrep with local numpy -# rules (.semgrep.yml) to detect common NumPy-related bugs and -# security issues in Python code. -# Note: the retired p/numpy registry ruleset is replaced by a local -# rules file that covers the same concerns. -# -# Trigger: This workflow runs on every push and on pull requests to main/master -# branches (including from forks) - -name: "(RHIZA) SEMGREP" - -# Permissions: Only read access to repository contents is needed -permissions: - contents: read - -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] - -jobs: - semgrep: - name: Semgrep (numpy) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - - name: Run Semgrep - env: - UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} - run: make semgrep diff --git a/.github/workflows/rhiza_sync.yml b/.github/workflows/rhiza_sync.yml index 9d06fd2e..7df732b3 100644 --- a/.github/workflows/rhiza_sync.yml +++ b/.github/workflows/rhiza_sync.yml @@ -1,27 +1,107 @@ name: (RHIZA) SYNC -# This workflow synchronizes the repository with its template. -# IMPORTANT: When workflow files (.github/workflows/rhiza_*.yml) are modified, -# a Personal Access Token (PAT) with 'workflow' scope is required. -# The PAT_TOKEN secret must be set in repository secrets. -# See .github/rhiza/TOKEN_SETUP.md for setup instructions. +# Synchronizes the repository with its rhiza template. +# - On Renovate/rhiza branch push: auto-commits synced files directly to the branch. +# - On schedule/dispatch: opens a pull request with the synced changes. +# +# IMPORTANT: A PAT with 'workflow' scope (PAT_TOKEN) is required when workflow +# files are modified. See .rhiza/docs/TOKEN_SETUP.md for setup instructions. permissions: contents: write pull-requests: write on: + push: + branches: + - 'renovate/jebel-quant-rhiza-**' + - 'rhiza/**' + paths: + - '.rhiza/template.yml' + schedule: + - cron: '0 0 * * 1' # Weekly on Monday workflow_dispatch: inputs: create-pr: description: "Create a pull request" type: boolean default: true - schedule: - - cron: '0 0 * * 1' # Weekly on Monday jobs: - sync: - if: ${{ github.repository != 'jebel-quant/rhiza' }} + sync-direct: + name: Sync and commit (Renovate) + if: github.event_name == 'push' && github.repository != 'jebel-quant/rhiza' + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.2 + with: + ref: ${{ github.ref }} + token: ${{ secrets.PAT_TOKEN || github.token }} + + - name: Check PAT_TOKEN configuration + shell: bash + env: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} + run: | + if [ -z "$PAT_TOKEN" ]; then + echo "::warning::PAT_TOKEN secret is not configured." + echo "::warning::If this sync modifies workflow files, the push will fail." + echo "::warning::See .rhiza/docs/TOKEN_SETUP.md for setup instructions." + else + echo "✓ PAT_TOKEN is configured." + fi + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + + - name: Get Rhiza version + id: rhiza-version + run: | + VERSION=$(cat .rhiza/.rhiza-version 2>/dev/null || echo "0.9.0") + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + + - name: Sync rhiza template + id: sync + run: | + set -euo pipefail + + RHIZA_VERSION="${{ steps.rhiza-version.outputs.version }}" + + echo "Running rhiza sync with version >=${RHIZA_VERSION}" + uvx "rhiza>=${RHIZA_VERSION}" sync . + + if git diff --quiet; then + echo "No changes detected after template sync" + echo "changes=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Template changes detected" + echo "changes=true" >> "$GITHUB_OUTPUT" + + - name: Commit and push changes + if: steps.sync.outputs.changes == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global url."https://x-access-token:${{ secrets.PAT_TOKEN || github.token }}@github.com/".insteadOf "https://github.com/" + + git add -A + git commit -m "$(cat <<'EOF' + chore: sync rhiza template files + + Automatically synced template files after updating .rhiza/template.yml + + Co-Authored-By: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + EOF + )" + + git push + + sync-pr: + name: Sync and open PR (scheduled/manual) + if: github.event_name != 'push' && github.repository != 'jebel-quant/rhiza' runs-on: ubuntu-latest steps: @@ -44,7 +124,7 @@ jobs: if [ -z "$PAT_TOKEN" ]; then echo "::warning::PAT_TOKEN secret is not configured." echo "::warning::If this sync modifies workflow files, the push will fail." - echo "::warning::See .github/TOKEN_SETUP.md for setup instructions." + echo "::warning::See .rhiza/docs/TOKEN_SETUP.md for setup instructions." else echo "✓ PAT_TOKEN is configured." fi @@ -85,7 +165,7 @@ jobs: git config --global url."https://x-access-token:${{ secrets.PAT_TOKEN }}@github.com/".insteadOf "https://github.com/" git commit -m "chore: Update via rhiza" - + - name: Create pull request if: > (github.event_name == 'schedule' || inputs.create-pr == true) @@ -97,4 +177,4 @@ jobs: branch: ${{ steps.branch.outputs.name }} delete-branch: true title: "chore: Sync with rhiza" - body-path: ${{ runner.temp }}/pr-description.md \ No newline at end of file + body-path: ${{ runner.temp }}/pr-description.md diff --git a/.github/workflows/rhiza_typecheck.yml b/.github/workflows/rhiza_typecheck.yml deleted file mode 100644 index 05906298..00000000 --- a/.github/workflows/rhiza_typecheck.yml +++ /dev/null @@ -1,50 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Type Checking -# -# Purpose: Run static type analysis using ty to enforce type safety. -# ty (https://github.com/astral-sh/ty) is a fast Python type checker -# from the Astral team (same team as ruff/uv). -# -# Type-checker settings live in [tool.ty.environment] in pyproject.toml. -# Locally, `make typecheck` runs the same check; it is also wired into -# `make validate` via the `post-validate::` hook in the repo Makefile. -# -# Trigger: On push and pull requests to main/master branches. - -name: "(RHIZA) TYPECHECK" - -permissions: - contents: read - -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] - -jobs: - typecheck: - name: Type checking - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6.0.2 - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - - name: Run ty type checker (make typecheck) - # Runs `uv run ty check src/` as defined in .rhiza/make.d/test.mk. - # ty is configured via [tool.ty.environment] in pyproject.toml. - env: - UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} - run: make typecheck diff --git a/.github/workflows/rhiza_validate.yml b/.github/workflows/rhiza_validate.yml deleted file mode 100644 index aefbd5c2..00000000 --- a/.github/workflows/rhiza_validate.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: (RHIZA) VALIDATE - -permissions: - contents: read - -on: - push: - branches: [ main, master ] - pull_request: - branches: [ main, master ] - -jobs: - validation: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v6.0.2 - with: - lfs: true - - - name: Install uv - uses: astral-sh/setup-uv@v7.6.0 - with: - version: "0.11.2" - - - name: Configure git auth for private packages - uses: ./.github/actions/configure-git-auth - with: - token: ${{ secrets.GH_PAT }} - - - name: Validate - shell: bash - run: | - make validate diff --git a/.github/workflows/rhiza_weekly.yml b/.github/workflows/rhiza_weekly.yml new file mode 100644 index 00000000..f4b56835 --- /dev/null +++ b/.github/workflows/rhiza_weekly.yml @@ -0,0 +1,118 @@ +name: "(RHIZA) WEEKLY" + +# Runs weekly checks that are too slow or noisy for every push: +# +# dep-compat-test — Resolves all dependencies fresh (ignoring the lockfile) +# and runs the full test suite. Catches newly-released +# packages that break compatibility before Renovate picks +# them up. Runs on schedule/dispatch only. +# +# link-check — Verifies that all hyperlinks in README.md are reachable. +# Runs on schedule/dispatch only. + +permissions: + contents: read + +on: + #push: + # branches: [main] + # paths: [README.md] + #pull_request: + # paths: [README.md] + schedule: + - cron: "0 8 * * 1" # Every Monday at 08:00 UTC + workflow_dispatch: + +jobs: + dep-compat-test: + name: Test with latest compatible dependencies + runs-on: ubuntu-latest + #if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.2 + with: + lfs: true + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Resolve and install latest compatible dependencies + env: + UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} + run: | + # --upgrade ignores the committed lockfile and resolves the newest + # versions that satisfy pyproject.toml constraints. + uv sync --upgrade + + - name: Show resolved package versions + run: | + echo "=== Installed package versions ===" + uv pip list + + - name: Run tests + env: + UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} + run: make test + + semgrep: + name: Semgrep (numpy) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Configure git auth for private packages + uses: ./.github/actions/configure-git-auth + with: + token: ${{ secrets.GH_PAT }} + + - name: Run Semgrep + env: + UV_EXTRA_INDEX_URL: ${{ secrets.UV_EXTRA_INDEX_URL }} + run: make semgrep + + pip-audit: + name: Dependency vulnerability scan + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Install uv + uses: astral-sh/setup-uv@v7.6.0 + with: + version: "0.11.2" + + - name: Run pip-audit + run: uvx pip-audit + + link-check: + name: Check links in README.md + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6.0.2 + + - name: Check links in README.md + uses: lycheeverse/lychee-action@v2 + with: + args: >- + --verbose + --no-progress + --accept 200,206,429 + README.md + fail: true diff --git a/.gitignore b/.gitignore index 0d16b942..24c7706d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,14 @@ _benchmarks _jupyter _site +# LaTeX build artifacts +docs/paper/*.aux +docs/paper/*.fdb_latexmk +docs/paper/*.fls +docs/paper/*.log +docs/paper/*.out +docs/paper/*.toc + # temp file used by Junie .output.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5999c603..5006b60b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,7 @@ # Workflows included: # - CI: Run tests on multiple Python versions # - VALIDATE: Validate Rhiza configuration -# - DEPTRY: Check dependencies -# - PRE-COMMIT: Run pre-commit checks +# - QUALITY: Dependency checks, pre-commit hooks, docs coverage, link checking # - BOOK: Build and deploy documentation # - SYNC: Synchronize with template repository # - RELEASE: Release workflow for tags @@ -53,16 +52,8 @@ include: - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_BRANCH - # Deptry - Check dependencies - - local: '.gitlab/workflows/rhiza_deptry.yml' - rules: - - if: $CI_COMMIT_TAG - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH - - # Pre-commit - Code quality checks - - local: '.gitlab/workflows/rhiza_pre-commit.yml' + # Quality - Dependency checks, pre-commit hooks, docs coverage, link checking + - local: '.gitlab/workflows/rhiza_quality.yml' rules: - if: $CI_COMMIT_TAG when: never diff --git a/.gitlab/README.md b/.gitlab/README.md index 884f5c79..acf05f46 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -9,8 +9,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun ├── workflows/ │ ├── rhiza_ci.yml # Continuous Integration - Python matrix testing │ ├── rhiza_validate.yml # Rhiza configuration validation -│ ├── rhiza_deptry.yml # Dependency checking -│ ├── rhiza_pre-commit.yml # Pre-commit hooks +│ ├── rhiza_quality.yml # Quality checks (deptry, pre-commit, docs coverage, link check) │ ├── rhiza_marimo.yml # Marimo notebook execution and artefact publishing │ ├── rhiza_book.yml # Documentation building (GitLab Pages) │ ├── rhiza_sync.yml # Template synchronization @@ -56,37 +55,24 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun --- -### 3. Deptry (`rhiza_deptry.yml`) -**Purpose:** Check for missing and obsolete dependencies. +### 3. Quality (`rhiza_quality.yml`) +**Purpose:** Run quality checks including dependency validation, pre-commit hooks, documentation coverage, and link checking. **Trigger:** - On push to any branch - On merge requests to main/master **Key Features:** -- Automatic source folder detection -- Identifies unused dependencies +- Dependency checking with deptry (`make deptry`) +- Pre-commit hooks for code formatting and linting (`make fmt`) +- Documentation coverage validation (`make docs-coverage`) +- Link checking on README.md with lychee -**Equivalent GitHub Action:** `.github/workflows/rhiza_deptry.yml` +**Equivalent GitHub Action:** `.github/workflows/rhiza_quality.yml` --- -### 4. Pre-commit (`rhiza_pre-commit.yml`) -**Purpose:** Run pre-commit checks for code quality. - -**Trigger:** -- On push to any branch -- On merge requests to main/master - -**Key Features:** -- Runs all pre-commit hooks -- UV environment setup - -**Equivalent GitHub Action:** `.github/workflows/rhiza_pre-commit.yml` - ---- - -### 5. Marimo (`rhiza_marimo.yml`) +### 4. Marimo (`rhiza_marimo.yml`) **Purpose:** Discover and execute all Marimo notebooks in the repository, publishing results as artefacts. **Trigger:** @@ -104,7 +90,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun --- -### 6. Book (`rhiza_book.yml`) +### 5. Book (`rhiza_book.yml`) **Purpose:** Build and deploy documentation to GitLab Pages. **Trigger:** @@ -121,7 +107,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun --- -### 7. Sync (`rhiza_sync.yml`) +### 6. Sync (`rhiza_sync.yml`) **Purpose:** Synchronize repository with its template. **Trigger:** @@ -140,7 +126,7 @@ This directory contains GitLab CI/CD workflow configurations that mirror the fun --- -### 8. Release (`rhiza_release.yml`) +### 7. Release (`rhiza_release.yml`) **Purpose:** Create releases and publish packages to PyPI. **Trigger:** diff --git a/.gitlab/workflows/rhiza_ci.yml b/.gitlab/workflows/rhiza_ci.yml index 2798398f..0fe651c5 100644 --- a/.gitlab/workflows/rhiza_ci.yml +++ b/.gitlab/workflows/rhiza_ci.yml @@ -3,9 +3,11 @@ # # Workflow: Continuous Integration (GitLab CI) # -# Purpose: Run tests on multiple Python versions to ensure compatibility. +# Purpose: Run tests on multiple Python versions, check dependencies, run +# pre-commit hooks, verify documentation coverage, validate the +# project, run security scans, and check license compliance. # -# Trigger: On push and merge requests to main/master branches. +# Trigger: On push and merge requests. ci:test: stage: test @@ -39,3 +41,69 @@ ci:docs-coverage: rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_BRANCH + +ci:typecheck: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - export UV_EXTRA_INDEX_URL="${UV_EXTRA_INDEX_URL}" + - uv venv + - make typecheck + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH + +ci:deptry: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - make deptry + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH + +ci:pre-commit: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - make fmt + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH + +ci:validate: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - make validate + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH + +ci:security: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - export UV_EXTRA_INDEX_URL="${UV_EXTRA_INDEX_URL}" + - uv venv + - make security + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH + +ci:license: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - export UV_EXTRA_INDEX_URL="${UV_EXTRA_INDEX_URL}" + - uv venv + - make license + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH diff --git a/.gitlab/workflows/rhiza_deptry.yml b/.gitlab/workflows/rhiza_deptry.yml deleted file mode 100644 index e2f15f18..00000000 --- a/.gitlab/workflows/rhiza_deptry.yml +++ /dev/null @@ -1,21 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Deptry (GitLab CI) -# -# Purpose: This workflow identifies missing and obsolete dependencies in the project. -# It helps maintain a clean dependency tree by detecting unused packages and -# implicit dependencies that should be explicitly declared. -# -# Trigger: This workflow runs on every push and on merge requests to main/master -# branches (including from forks) - -deptry:check: - stage: test - needs: [] - image: ghcr.io/astral-sh/uv:0.9.30-bookworm - script: - - make deptry - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH diff --git a/.gitlab/workflows/rhiza_license.yml b/.gitlab/workflows/rhiza_license.yml deleted file mode 100644 index 40d931d8..00000000 --- a/.gitlab/workflows/rhiza_license.yml +++ /dev/null @@ -1,20 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: License compliance (GitLab CI) -# -# Purpose: This workflow checks that no copyleft-licensed dependencies -# (GPL, LGPL, AGPL) have been introduced via transitive updates. -# -# Trigger: This workflow runs on every push and on merge requests to main/master -# branches (including from forks) - -license:check: - stage: test - needs: [] - image: ghcr.io/astral-sh/uv:0.9.30-bookworm - script: - - make license - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH diff --git a/.gitlab/workflows/rhiza_paper.yml b/.gitlab/workflows/rhiza_paper.yml new file mode 100644 index 00000000..4d1ff6da --- /dev/null +++ b/.gitlab/workflows/rhiza_paper.yml @@ -0,0 +1,46 @@ +# This file is part of the jebel-quant/rhiza repository +# (https://github.com/jebel-quant/rhiza). +# +# Workflow: Paper (GitLab CI) +# +# Purpose: Compile the LaTeX paper (paper/*.tex) to a PDF and publish +# it as a downloadable pipeline artifact. +# Only active when a *.tex file exists under paper/. +# +# Trigger: On push to main/master or when files under paper/ change. + +paper:build: + stage: build + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + before_script: + - apt-get update && apt-get install -y texlive-latex-extra texlive-fonts-recommended texlive-bibtex-extra latexmk + script: + - | + if ! compgen -G "paper/*.tex" > /dev/null 2>&1; then + echo "No paper/*.tex found; skipping LaTeX compilation." + exit 0 + fi + + if [ -f paper/basanos.tex ]; then + tex_file="basanos.tex" + else + tex_file=$(find paper -maxdepth 1 -name "*.tex" | head -1 | xargs basename) + fi + pdf_file="${tex_file%.tex}.pdf" + + cd paper && latexmk -pdf -interaction=nonstopmode "${tex_file}" + cp "${pdf_file}" "../${pdf_file}" + artifacts: + paths: + - "*.pdf" + expire_in: 30 days + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "master" + changes: + - paper/**/* + - .gitlab/workflows/rhiza_paper.yml + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - paper/**/* + - .gitlab/workflows/rhiza_paper.yml diff --git a/.gitlab/workflows/rhiza_pre-commit.yml b/.gitlab/workflows/rhiza_pre-commit.yml deleted file mode 100644 index bd89c44d..00000000 --- a/.gitlab/workflows/rhiza_pre-commit.yml +++ /dev/null @@ -1,26 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Pre-commit (GitLab CI) -# -# Purpose: This workflow runs pre-commit checks to ensure code quality -# and consistency across the codebase. It helps catch issues -# like formatting errors, linting issues, and other code quality -# problems before they are merged. -# -# Trigger: This workflow runs on every push and on merge requests to main/master -# branches (including from forks) -# -# Components: -# - 🔍 Run pre-commit checks using reusable action - -pre-commit:check: - stage: test - needs: [] - image: ghcr.io/astral-sh/uv:0.9.30-bookworm - script: - - make fmt - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH - diff --git a/.gitlab/workflows/rhiza_semgrep.yml b/.gitlab/workflows/rhiza_semgrep.yml deleted file mode 100644 index 8a3cc8d2..00000000 --- a/.gitlab/workflows/rhiza_semgrep.yml +++ /dev/null @@ -1,21 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Semgrep (GitLab CI) -# -# Purpose: This workflow runs static analysis using Semgrep with local numpy -# rules (.semgrep.yml) to detect common NumPy-related bugs and -# security issues in Python code. -# -# Trigger: This workflow runs on every push and on merge requests to main/master -# branches (including from forks) - -semgrep:check: - stage: test - needs: [] - image: ghcr.io/astral-sh/uv:0.9.30-bookworm - script: - - make semgrep - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH diff --git a/.gitlab/workflows/rhiza_validate.yml b/.gitlab/workflows/rhiza_validate.yml deleted file mode 100644 index aa727831..00000000 --- a/.gitlab/workflows/rhiza_validate.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This file is part of the jebel-quant/rhiza repository -# (https://github.com/jebel-quant/rhiza). -# -# Workflow: Rhiza Validate (GitLab CI) -# -# Purpose: Validates Rhiza configuration -# -# Trigger: This workflow runs on every push and on merge requests to main/master - -validate:rhiza: - stage: test - needs: [] - image: ghcr.io/astral-sh/uv:0.9.30-bookworm - script: - - make validate - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH diff --git a/.gitlab/workflows/rhiza_weekly.yml b/.gitlab/workflows/rhiza_weekly.yml new file mode 100644 index 00000000..ff5e409d --- /dev/null +++ b/.gitlab/workflows/rhiza_weekly.yml @@ -0,0 +1,55 @@ +# This file is part of the jebel-quant/rhiza repository +# (https://github.com/jebel-quant/rhiza). +# +# Workflow: Weekly (GitLab CI) +# +# Purpose: Slower checks deferred to a weekly schedule: +# +# weekly:semgrep — Semgrep static analysis (numpy rules). +# weekly:pip-audit — Dependency vulnerability scan. +# weekly:link-check — Verifies hyperlinks in README.md are reachable. +# Also runs on push/MR when README.md changes. +# +# Trigger: Scheduled pipeline (configure weekly in GitLab CI/CD Schedules), +# or manually via the web UI. + +weekly:semgrep: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - export UV_EXTRA_INDEX_URL="${UV_EXTRA_INDEX_URL}" + - uv venv + - make semgrep + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "web" + when: manual + allow_failure: true + +weekly:pip-audit: + stage: test + needs: [] + image: ghcr.io/astral-sh/uv:0.9.30-bookworm + script: + - uvx pip-audit + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "web" + when: manual + allow_failure: true + +weekly:link-check: + stage: test + needs: [] + image: lycheeverse/lychee:latest + script: + - lychee --verbose --no-progress --accept 200,206,429 README.md + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - README.md + - if: $CI_COMMIT_BRANCH + changes: + - README.md + - if: $CI_PIPELINE_SOURCE == "schedule" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2988589b..4664a342 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,7 +56,7 @@ repos: - id: uv-lock - repo: https://github.com/Jebel-Quant/rhiza-hooks - rev: v0.3.0 # Use the latest release + rev: v0.3.1 # Use the latest release hooks: # Migrated from rhiza - id: check-rhiza-workflow-names diff --git a/.rhiza/INDEX.md b/.rhiza/INDEX.md deleted file mode 100644 index 5a579137..00000000 --- a/.rhiza/INDEX.md +++ /dev/null @@ -1,153 +0,0 @@ -# Rhiza Index - -Quick reference to all utilities, makefiles, and resources in the `.rhiza/` directory. - -## 📁 Directory Structure - -``` -.rhiza/ -├── rhiza.mk # Core makefile logic (153 lines) -├── .rhiza-version # Current Rhiza version -├── .cfg.toml # Configuration file -├── .env # Environment variables -├── template-bundles.yml # Template bundle definitions -├── make.d/ # Makefile extensions (auto-loaded) -├── requirements/ # Python dependencies -├── templates/ # Project templates -├── tests/ # Test suite -├── docs/ # Internal documentation -└── assets/ # Static assets -``` - -## 🔧 Makefiles (`.rhiza/make.d/`) - -| File | Size | Purpose | Section | -|------|------|---------|---------| -| `agentic.mk` | 3.1K | AI agent integrations (copilot, claude) | Agentic Workflows | -| `book.mk` | 4.7K | Documentation book generation | Book | -| `bootstrap.mk` | 4.3K | Installation and environment setup | Bootstrap | -| `custom-env.mk` | 290B | Example environment customizations | - | -| `custom-task.mk` | 423B | Example custom tasks | Custom Tasks | -| `docker.mk` | 1.1K | Docker build and run targets | Docker | -| `docs.mk` | 3.9K | Documentation generation (pdoc) | Documentation | -| `github.mk` | 6.0K | GitHub CLI integrations | GitHub Helpers | -| `lfs.mk` | 3.0K | Git LFS management | Git LFS | -| `marimo.mk` | 2.9K | Marimo notebook support | Marimo Notebooks | -| `presentation.mk` | 3.3K | Presentation building (Marp) | Presentation | -| `quality.mk` | 860B | Code quality and formatting | Quality and Formatting | -| `releasing.mk` | 2.0K | Release and versioning | Releasing and Versioning | -| `test.mk` | 5.1K | Testing infrastructure | Development and Testing | - -**Total**: 14 makefiles, ~41KB - -## 📦 Requirements (`.rhiza/requirements/`) - -| File | Purpose | -|------|---------| -| `docs.txt` | Documentation generation dependencies (pdoc) | -| `marimo.txt` | Marimo notebook dependencies | -| `tests.txt` | Testing dependencies (pytest, coverage) | -| `tools.txt` | Development tools (pre-commit, python-dotenv) | - -See [requirements/README.md](requirements/README.md) for details. - -## 🧪 Test Suite (`.rhiza/tests/`) - -| Directory | Purpose | -|-----------|---------| -| `api/` | Makefile target validation (dry-run tests) | -| `deps/` | Dependency health checks | -| `integration/` | End-to-end workflow tests | -| `structure/` | Static project structure assertions | -| `sync/` | Template sync and content validation | -| `utils/` | Test infrastructure utilities | - -**Total**: 23 Python test files - -See [tests/README.md](tests/README.md) for details. - -## 📚 Documentation (`.rhiza/docs/`) - -| File | Purpose | -|------|---------| -| `ASSETS.md` | Asset management documentation | -| `CONFIG.md` | Configuration file documentation | -| `LFS.md` | Git LFS setup and usage | -| `PRIVATE_PACKAGES.md` | Private package authentication | -| `RELEASING.md` | Release process documentation | -| `TOKEN_SETUP.md` | GitHub token setup | -| `WORKFLOWS.md` | GitHub Actions workflows | - -## 🎨 Assets (`.rhiza/assets/`) - -- `rhiza-logo.svg` - Rhiza logo graphic - -## 📋 Templates (`.rhiza/templates/`) - -- `minibook/` - Minimal documentation book template - -## 🔌 Template Bundles - -Defined in `template-bundles.yml`: - -| Bundle | Description | Files | -|--------|-------------|-------| -| `core` | Core Rhiza infrastructure | 43 files | -| `github` | GitHub Actions workflows | CI/CD | -| `tests` | Testing infrastructure | pytest, coverage | -| `marimo` | Interactive notebooks | Marimo support | -| `book` | Documentation generation | Book building | -| `docker` | Docker containerization | Dockerfile | -| `lfs` | Git LFS support | Large files | -| `presentation` | Presentation building | reveal.js | -| `gitlab` | GitLab CI/CD | GitLab workflows | -| `devcontainer` | VS Code DevContainer | Dev environment | -| `legal` | Legal documentation | LICENSE, CODE_OF_CONDUCT | - -## 🎯 Key Make Targets - -### Bootstrap -- `make install` - Install dependencies -- `make install-uv` - Ensure uv/uvx is installed -- `make clean` - Clean artifacts and stale branches - -### Development -- `make test` - Run test suite -- `make fmt` - Format code -- `make docs` - Generate documentation - -### AI Agents -- `make copilot` - GitHub Copilot interactive prompt -- `make claude` - Claude Code interactive prompt -- `make analyse-repo` - Update REPOSITORY_ANALYSIS.md - -### Documentation -- `make book` - Build documentation book -- `make marimo` - Start Marimo server -- `make presentation` - Generate presentation slides - -### Docker -- `make docker-build` - Build Docker image -- `make docker-run` - Run container - -### GitHub -- `make view-prs` - List open pull requests -- `make view-issues` - List open issues -- `make failed-workflows` - List failing workflows - -### Quality -- `make fmt` - Format code with ruff -- `make lint` - Lint code -- `make deptry` - Check dependencies - -### Releasing -- `make release` - Create a release -- `make bump` - Bump version - -## 🔗 Related Documentation - -- [Architecture Diagrams & Naming Conventions](../docs/ARCHITECTURE.md) - Visual architecture overview and detailed naming conventions -- [Makefile Cookbook](make.d/README.md) - Common patterns and recipes -- [Test Suite Guide](tests/README.md) - Testing conventions -- [Customization Guide](../docs/CUSTOMIZATION.md) - How to customize Rhiza -- [Quick Reference](../docs/QUICK_REFERENCE.md) - Common commands diff --git a/.rhiza/make.d/paper.mk b/.rhiza/make.d/paper.mk new file mode 100644 index 00000000..1733969d --- /dev/null +++ b/.rhiza/make.d/paper.mk @@ -0,0 +1,33 @@ +## paper.mk - LaTeX paper compilation targets +# This file is included by the main Makefile + +PAPER_DIR ?= docs/paper + +.PHONY: paper paper-clean + +##@ Paper + +paper:: ## compile LaTeX documents in docs/paper to PDF using latexmk + @printf "${BLUE}[INFO] Checking for latexmk...${RESET}\n" + @if ! command -v latexmk >/dev/null 2>&1; then \ + printf "${RED}[ERROR] latexmk not found. Please install a LaTeX distribution (e.g., MacTeX, TeX Live).${RESET}\n"; \ + exit 1; \ + fi + @if [ -z "$$(find $(PAPER_DIR) -maxdepth 1 -name '*.tex' 2>/dev/null)" ]; then \ + printf "${YELLOW}[WARN] No .tex files found in $(PAPER_DIR), skipping.${RESET}\n"; \ + exit 0; \ + fi + @if [ -f $(PAPER_DIR)/basanos.tex ]; then \ + tex_file="basanos.tex"; \ + else \ + tex_file=$$(find $(PAPER_DIR) -maxdepth 1 -name "*.tex" | head -1 | xargs basename); \ + fi; \ + printf "${BLUE}[INFO] Compiling $$tex_file...${RESET}\n"; \ + cd $(PAPER_DIR) && latexmk -pdf -bibtex -interaction=nonstopmode "$$tex_file" || exit 1; \ + pdf_file="$${tex_file%.tex}.pdf"; \ + printf "${GREEN}[SUCCESS] $(PAPER_DIR)/$$pdf_file${RESET}\n" + +paper-clean:: ## remove latexmk build artifacts in docs/paper + @printf "${BLUE}[INFO] Cleaning paper artifacts...${RESET}\n" + @cd $(PAPER_DIR) && latexmk -C 2>/dev/null || true + @printf "${GREEN}[SUCCESS] Cleaned $(PAPER_DIR)${RESET}\n" diff --git a/.semgrep.yml b/.rhiza/semgrep.yml similarity index 100% rename from .semgrep.yml rename to .rhiza/semgrep.yml diff --git a/.rhiza/template-bundles.yml b/.rhiza/template-bundles.yml index c254d074..12c9869e 100644 --- a/.rhiza/template-bundles.yml +++ b/.rhiza/template-bundles.yml @@ -74,23 +74,16 @@ bundles: - .rhiza/make.d/agentic.mk # Core GitHub Actions workflows - .github/workflows/copilot-setup-steps.yml - - .github/workflows/rhiza_validate.yml - .github/workflows/rhiza_sync.yml - - .github/workflows/rhiza_pre-commit.yml - - .github/workflows/rhiza_deptry.yml - .github/workflows/rhiza_release.yml - - .github/workflows/renovate_rhiza_sync.yml - - .github/workflows/rhiza_pip_audit.yml - - .github/workflows/rhiza_typecheck.yml + - .github/workflows/rhiza_weekly.yml - .github/actions/configure-git-auth - .github/dependabot.yml - .github/copilot-instructions.md - .github/agents - .github/hooks - .github/secret_scanning.yml - - .github/workflows/rhiza_license.yml - - .semgrep.yml - - .github/workflows/rhiza_semgrep.yml + - .rhiza/semgrep.yml # Issue templates (config.yml excluded — contains repo-specific URLs) - .github/ISSUE_TEMPLATE/bug_report.yml - .github/ISSUE_TEMPLATE/feature_request.yml @@ -226,9 +219,8 @@ bundles: # GitHub Actions workflows - .github/workflows/rhiza_ci.yml - - .github/workflows/rhiza_security.yml - .github/workflows/rhiza_codeql.yml - - .github/workflows/rhiza_dep_compat_test.yml + - .github/workflows/rhiza_weekly.yml # ============================================================================ # BENCHMARKS - Performance benchmarking with pytest-benchmark diff --git a/Makefile b/Makefile index 7eb6f644..1044038d 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,13 @@ post-validate:: ##@ Security -.PHONY: security -security: install ## run security scans (pip-audit and bandit) - @printf "${BLUE}[INFO] Running pip-audit for dependency vulnerabilities...${RESET}\n" - @${UVX_BIN} pip-audit - @printf "${BLUE}[INFO] Running bandit security scan...${RESET}\n" - @${UVX_BIN} bandit -r ${SOURCE_FOLDER} -ll -q -c pyproject.toml + +#.PHONY: security +#security: install ## run security scans (pip-audit and bandit) +# @printf "${BLUE}[INFO] Running pip-audit for dependency vulnerabilities...${RESET}\n" +# @${UVX_BIN} pip-audit +# @printf "${BLUE}[INFO] Running bandit security scan...${RESET}\n" +# @${UVX_BIN} bandit -r ${SOURCE_FOLDER} -ll -q -c pyproject.toml ##@ Quality @@ -33,7 +34,7 @@ security: install ## run security scans (pip-audit and bandit) semgrep: install ## run Semgrep static analysis (numpy rules) @printf "${BLUE}[INFO] Running Semgrep (numpy rules)...${RESET}\n" @if [ -d ${SOURCE_FOLDER} ]; then \ - ${UVX_BIN} semgrep --config .semgrep.yml ${SOURCE_FOLDER}; \ + ${UVX_BIN} semgrep --config .rhiza/semgrep.yml ${SOURCE_FOLDER}; \ else \ printf "${YELLOW}[WARN] SOURCE_FOLDER '${SOURCE_FOLDER}' not found, skipping semgrep.${RESET}\n"; \ fi diff --git a/README.md b/README.md index abc6f4ed..1c5552da 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ Keep your templates up-to-date with automated sync workflows: - The `.github/workflows/sync.yml` workflow runs on schedule or manually - Creates pull requests with template updates -For GitHub Token configuration and details, see the [GitHub Actions documentation](.github/README.md). +For GitHub Token configuration and details, see the [GitHub Actions documentation](.rhiza/docs/TOKEN_SETUP.md). ### What to Expect After Integration @@ -368,7 +368,7 @@ make presentation-pdf # Generate PDF slides make presentation-serve # Serve with live reload ``` -For detailed information about creating and customising presentations, see [presentation/README.md](presentation/README.md). +For detailed information about creating and customising presentations, see [presentation/README.md](docs/presentations/README.md). ### Documentation Examples @@ -392,7 +392,7 @@ Hello, World! ### Documentation Customisation -For information on customising the look and feel of your documentation, see [book/README.md](book/README.md). +For information on customising the look and feel of your documentation, see [book/README.md](docs/BOOK.md). ### Python Version Management diff --git a/REPOSITORY_ANALYSIS.md b/REPOSITORY_ANALYSIS.md deleted file mode 100644 index 255ac2bb..00000000 --- a/REPOSITORY_ANALYSIS.md +++ /dev/null @@ -1,63 +0,0 @@ -# Repository Analysis Journal - -This document contains dated architectural and technical reviews of the rhiza repository. - ---- - -## 2026-03-10 — Analysis Entry - -### Summary -Rhiza is a **living template framework** for Python projects, providing synchronizable configuration templates via Make-based automation. The repository is **mature and well-engineered**, with comprehensive CI/CD (20 workflows), extensive documentation (7200+ lines across 22 MD files), modular Makefile architecture (18 `.mk` modules), and 29 test files organized into 7 categories. Version 0.8.7 represents a stable foundation with active development (826+ commits). The project demonstrates strong DevOps practices, clear separation of concerns, and thoughtful extensibility mechanisms. - -### Strengths - -- **Modular Architecture**: Clean separation via `.rhiza/make.d/*.mk` modules (18 files: bootstrap.mk, test.mk, github.mk, etc.) allows incremental feature adoption without coupling -- **Comprehensive Testing**: 29 test files across 7 categories (structure, api, integration, sync, deps, stress, utils) with clear separation between static validation and subprocess-based integration tests -- **Rich Documentation**: 22 markdown files (ARCHITECTURE.md, TECHNICAL_DEBT.md, TESTS.md, etc.) provide deep context; TECHNICAL_DEBT.md tracks 11 known issues with priority/effort estimates -- **Bundle System**: `template-bundles.yml` defines 13 semantic bundles (core, github, tests, marimo, book, docker, devcontainer, etc.) with dependency declarations, enabling composable project setups -- **Multi-Platform CI**: Full feature parity across GitHub Actions (16 workflows) and GitLab CI (8 workflows in `.gitlab/`), demonstrating platform independence -- **Hooks & Extensibility**: Double-colon Make targets (`pre-install::`, `post-sync::`) enable safe downstream customization without breaking template sync -- **Zero Runtime Dependencies**: `pyproject.toml` declares `dependencies = []`, making this a pure configuration/tooling framework -- **Automated Quality Gates**: `.github/hooks/session-{start,end}.sh` enforce environment validation and quality checks for GitHub Copilot sessions -- **Agentic Workflows**: Early adoption of GitHub Agentic Workflows with 3 starter templates (daily-repo-status, ci-doctor, issue-triage) and validation workflow -- **Renovate Integration**: Sophisticated `renovate.json` with custom regex managers for template versioning, enabling automated updates of `ref:` field in downstream projects -- **Version Matrix Testing**: Dynamic Python version matrix generation from `pyproject.toml` classifiers (3.11-3.14) in CI workflow - -### Weaknesses - -- **No Source Code**: Project contains templates and tests but **no `src/` directory** — legitimate for a template repo, but reduces code coverage meaningfulness (coverage tracks test execution, not template usage) -- **Test Execution Blocked**: `make test` fails with "Permission denied and could not request permission from user" — indicates environment issue or permission constraints during analysis -- **Missing Lock Files**: No `.lock.yml` files found despite gh-aw workflows (adr-create.md, ci-doctor.md) existing — suggests workflows not yet compiled or feature incomplete -- **Python Version Mismatch**: `.python-version` specifies `3.12`, but `pyproject.toml` claims support for 3.11-3.14 — actual runtime untested on non-3.12 versions in this environment -- **No Scripts Directory**: `.rhiza/scripts/` is empty except `__pycache__` — suggests migration away from scripts to Make targets, but may leave dead references -- **Roadmap Staleness**: ROADMAP.md references "Current Version: 0.8.1-rc.2" but `pyproject.toml` shows `0.8.7` — documentation lags reality by ~6 minor versions -- **Bootstrap Documentation Gap**: `.devcontainer/bootstrap.sh` referenced in custom instructions but not verified to exist in repository scan -- **No Benchmark Results**: `tests/benchmarks/` exists but no committed baseline results visible — performance regression detection requires manual interpretation -- **Template Validation Incomplete**: TECHNICAL_DEBT.md item #8 acknowledges "Limited validation of custom templates before sync" as medium-priority debt - -### Risks / Technical Debt - -- **Conflict Resolution**: TECHNICAL_DEBT.md item #1 (Critical) — Manual 3-way merge required when template updates conflict with local changes; no automatic resolution strategy -- **Performance at Scale**: Item #3 (High) — Sync operations slow for large repos; no incremental sync or caching layer implemented -- **Test Coverage Gap**: Item #2 (High) — "Limited test coverage for template synchronization edge cases" — core functionality undertested despite 29 test files -- **Complexity Growth**: 20 GitHub workflows (2056 total lines) + 18 Makefile modules creates high cognitive load for contributors; no architectural diagrams beyond docs/ARCHITECTURE.md -- **gh-aw Maturity**: Agentic workflows (adr-create.md, ci-doctor.md, issue-triage.md) lack `.lock.yml` counterparts — feature may be experimental/incomplete -- **Dependency Upper Bounds**: TECHNICAL_DEBT.md item #7 notes some dependencies lack upper bounds (e.g., `marimo>=0.18.0,<1.0` is good, but pattern not universal) -- **Private Package Assumption**: `.github/actions/configure-git-auth` and `UV_EXTRA_INDEX_URL` secret suggest reliance on private PyPI packages — may break in forks or public use -- **Hook Documentation Scattered**: Hooks mentioned in Makefile, docs/ARCHITECTURE.md, and `.rhiza/make.d/README.md` but no single source of truth for all available hooks -- **No Public Release Artifacts**: Classifier "Private :: Do Not Upload" prevents PyPI publication — downstream adoption requires git-based sync, increasing coupling - -### Score - -**8/10** — Solid, production-grade template framework with minor maintenance gaps. - -**Rationale**: -- **+3**: Excellent modular architecture, comprehensive documentation, extensibility design -- **+2**: Strong CI/CD coverage (multi-platform, matrix testing), automated quality gates -- **+2**: Thoughtful DevOps practices (Renovate, session hooks, bundle system) -- **+1**: Active development (recent commits), transparent technical debt tracking -- **-1**: Test execution issues, missing lock files, documentation staleness -- **-1**: No actual source code to validate claims (legitimate for templates, but reduces verifiability) - -**Context**: This is a **configuration/template repository**, not a library. Scoring adjusted for domain — it excels at what it aims to be (a living template system), but lacks traditional library artifacts (src/, published packages, API docs). - diff --git a/docs/CUSTOMIZATION.md b/docs/CUSTOMIZATION.md index a446e290..7437856d 100644 --- a/docs/CUSTOMIZATION.md +++ b/docs/CUSTOMIZATION.md @@ -157,7 +157,7 @@ For example, to override the main module template, create `book/pdoc-templates/m See the [pdoc documentation on templates](https://pdoc.dev/docs/pdoc.html#edit-pdocs-html-template) for full details on how to override specific parts of the documentation. -For more details on customizing the documentation, see [book/README.md](../book/README.md). +For more details on customizing the documentation, see [book/README.md](BOOK.md). ## 📖 Complete Documentation diff --git a/docs/paper/rhiza.pdf b/docs/paper/rhiza.pdf new file mode 100644 index 00000000..e1557c81 Binary files /dev/null and b/docs/paper/rhiza.pdf differ diff --git a/docs/paper/rhiza.tex b/docs/paper/rhiza.tex new file mode 100644 index 00000000..e3b01a96 --- /dev/null +++ b/docs/paper/rhiza.tex @@ -0,0 +1,339 @@ +\documentclass[12pt,a4paper]{article} + +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{lmodern} +\usepackage[margin=2.5cm]{geometry} +\usepackage{hyperref} +\usepackage{microtype} +\emergencystretch=2em +\usepackage{booktabs} +\usepackage{listings} +\usepackage{xcolor} +\usepackage{graphicx} +\usepackage{enumitem} +\usepackage{amsmath} +\usepackage{cite} +\usepackage{titlesec} +\usepackage{abstract} + +\hypersetup{ + colorlinks=true, + linkcolor=blue!60!black, + citecolor=green!50!black, + urlcolor=blue!70!black, +} + +\lstdefinestyle{yaml}{ + language=, + basicstyle=\ttfamily\small, + keywordstyle=\color{blue}, + commentstyle=\color{gray}, + stringstyle=\color{orange!80!black}, + breaklines=true, + frame=single, + rulecolor=\color{gray!40}, + backgroundcolor=\color{gray!5}, + captionpos=b, +} + +\lstdefinestyle{bash}{ + language=bash, + basicstyle=\ttfamily\small, + keywordstyle=\color{blue}, + commentstyle=\color{gray}, + breaklines=true, + frame=single, + rulecolor=\color{gray!40}, + backgroundcolor=\color{gray!5}, + captionpos=b, +} + +\title{\textbf{Rhiza: A Living-Template System for Sustainable Python Project Infrastructure}} + +\author{ + Thomas Schmelzer\\ + \texttt{tschmelzer@jqr.io} + \and + Harry Campion + \and + Mark Richardson +} + +\date{March 2026} + +\begin{document} + +\maketitle + +\begin{abstract} +Software projects accumulate infrastructure debt: continuous-integration pipelines age out of date, linting rules diverge from community standards, and hard-won configuration improvements never propagate from the project that discovered them to its siblings. Ad-hoc fixes compound this drift without a systematic remedy. We present \textbf{Rhiza}---from the Greek \textit{rhiza}, ``root''---a living-template system for Python projects that decouples the \emph{content} of best-practice infrastructure from the \emph{mechanism} that applies it. Unlike one-time project generators such as Cookiecutter or Copier, Rhiza provides templates that continue to evolve and that downstream projects can pull in selectively at any time. The system is structured around three key abstractions: (i)~a versioned template repository that acts as the authoritative source of curated infrastructure files; (ii)~a lightweight CLI that fetches and merges those files; and (iii)~a modular bundle system that lets teams opt into exactly the features they need. We describe the design rationale, the architectural decisions that guide the implementation, and the operational experience accumulated across more than twenty downstream projects. +\end{abstract} + +\tableofcontents +\newpage + +% ───────────────────────────────────────────────────────────────────────────── +\section{Introduction} +\label{sec:intro} +% ───────────────────────────────────────────────────────────────────────────── + +Every serious Python project carries with it a hidden second project: the infrastructure that makes the first one possible. Continuous-integration pipelines, linting and formatting rules, reproducible environment specifications, pre-commit hooks, security scanners, documentation generators, and Makefile-based development workflows are not incidental---they are the scaffolding on which reliable, maintainable software is built. Yet this scaffolding is almost universally treated as an afterthought. + +The typical lifecycle of project infrastructure proceeds as follows. At project creation, a developer copies configuration files from a previous project, consults a blog post, or runs a scaffold tool. The resulting configuration is reasonable for that moment in time. Over months and years, however, the ecosystem moves on: the linter gains new rules, the CI service deprecates old syntax, the recommended Python version changes, a new security vulnerability scanner emerges. The infrastructure falls behind. Keeping it current requires manual effort that no one formally owns, so it rarely happens. The result is configuration drift: each project in an organization diverges from the others and from current best practice, with the gap widening over time. + +\textbf{Rhiza} is a response to this problem. Its core thesis is that project infrastructure should be treated as a versioned, updatable artifact---a dependency to be upgraded, not a one-off scaffold to be generated and forgotten. This is not a novel observation; the DevOps and infrastructure-as-code communities have long advocated for treating infrastructure with the same discipline applied to application code~\cite{humble2010continuous,fowler2016infrastructure}. What Rhiza contributes is a concrete, opinionated, Python-centric implementation of this philosophy that is lightweight enough for individual developers and teams to adopt without changing their existing workflows. + +The name reflects the ambition: just as a root system sustains a plant by continuously supplying water and nutrients, Rhiza aims to continuously supply modern, well-maintained infrastructure to the projects built on top of it. + +% ───────────────────────────────────────────────────────────────────────────── +\section{Background and Motivation} +\label{sec:motivation} +% ───────────────────────────────────────────────────────────────────────────── + +\subsection{The Configuration-Drift Problem} + +Configuration drift in software projects is an instance of the broader problem of \emph{technical debt}~\cite{cunningham1992wycash}. Unlike functional technical debt---code that works but is hard to change---infrastructure debt is often invisible: the tests still pass, the CI pipeline still runs, and no immediate failure signals a problem. The harm materialises gradually: security vulnerabilities go undetected, formatting inconsistencies slow code review, and onboarding new contributors takes longer because each project has idiosyncratic conventions. + +Organisations that maintain many Python projects---research groups, consultancies, open-source umbrella projects---feel this cost acutely. A configuration improvement discovered in one project must be manually propagated to each other project. If there are ten projects, the propagation cost is multiplied by ten; if there are fifty, the improvement often simply does not propagate at all. + +\subsection{Existing Approaches and Their Limitations} + +Several tools exist to help with project scaffolding: + +\paragraph{Cookiecutter~\cite{cookiecutter}.} Cookiecutter generates a project directory from a Jinja2 template. It solves the \emph{initialisation} problem well but provides no mechanism for synchronising improvements after the initial generation. Once a project is created, it is on its own. + +\paragraph{Copier~\cite{copier}.} Copier addresses the synchronisation gap with an \texttt{update} command that re-applies a template to an existing project. This is a significant advance. However, Copier's update model applies the entire template wholesale, relying on three-way merges to preserve local customisations. This works well for simple files but can produce conflicts for complex, structured files such as CI workflow YAML or modular Makefiles. Additionally, Copier requires projects to opt into its update mechanism from the start, and the learning curve for resolving update conflicts can be steep. + +\paragraph{cruft~\cite{cruft}.} cruft adds a Cookiecutter-update workflow via a stored diff approach. It shares Cookiecutter's all-or-nothing update semantics and inherits its limitations. + +\paragraph{Template repositories on GitHub.} Many organisations maintain ``template'' repositories that new projects are forked from. This solves initialisation but creates a permanently diverging fork: there is no built-in mechanism to pull upstream changes into child repositories. + +\paragraph{Internal tooling.} Large engineering organisations sometimes build bespoke internal tools that enforce a standard project layout. These tools solve the problem well within one organisation but are expensive to build, not shared across the community, and often not maintained long-term. + +\subsection{The Need for Living Templates} + +The common limitation of the above approaches is that they treat project infrastructure as a \emph{point-in-time} artefact rather than a \emph{continuous} one. Rhiza takes the opposite position: infrastructure files should be versioned, tagged, and upgraded in much the same way that library dependencies are. A project's \texttt{.rhiza/template.yml} file specifies which version of the Rhiza template to use, analogously to how \texttt{pyproject.toml} specifies which version of a library to install. Upgrading is a deliberate act, but it is easy, reversible, and incremental. + +% ───────────────────────────────────────────────────────────────────────────── +\section{System Design} +\label{sec:design} +% ───────────────────────────────────────────────────────────────────────────── + +\subsection{Architectural Overview} + +Rhiza is deliberately split into two independent components (ADR-0005): + +\begin{enumerate}[leftmargin=*] + \item \textbf{The template repository} (\texttt{jebel-quant/rhiza} on GitHub) is the authoritative source of curated infrastructure files. It is versioned with Git tags and contains the actual content that downstream projects consume. + + \item \textbf{The CLI} (\texttt{rhiza} on PyPI) is the engine that fetches files from the template repository and applies them to a downstream project. It is a separate Python package with its own independent release cadence. +\end{enumerate} + +Separating these concerns means that the CLI can be improved---bug fixes, new commands, better conflict resolution---without forcing downstream projects to adopt new template content, and vice versa. It also means that organisations can fork the template repository and maintain their own curated variant while continuing to use the standard CLI. + +\subsection{Template Bundles} +\label{sec:bundles} + +A naive template system maps every file in the template repository to a file in the downstream project. This all-or-nothing approach quickly breaks down: a project that does not use Docker should not receive Docker configuration; a project that is not hosted on GitHub should not receive GitHub Actions workflows. + +Rhiza addresses this with \emph{template bundles} (ADR-0006): named, coarse-grained groups of related files. The bundles defined in \texttt{.rhiza/template-bundles.yml} include: + +\begin{table}[h] +\centering +\begin{tabular}{lp{9cm}} +\toprule +\textbf{Bundle} & \textbf{Contents} \\ +\midrule +\texttt{core} & Makefile infrastructure, \texttt{pyproject.toml} skeleton, \texttt{.python-version}, \texttt{uv.lock} patterns \\ +\texttt{tests} & pytest configuration, coverage settings, property-based testing scaffold \\ +\texttt{github} & GitHub Actions workflows (CI, lint, security, release, sync, docs) \\ +\texttt{gitlab} & GitLab CI equivalent with feature parity \\ +\texttt{docker} & Dockerfile templates, \texttt{.dockerignore}, container Makefile module \\ +\texttt{book} & MkDocs configuration, documentation structure, coverage badges \\ +\texttt{marimo} & Marimo notebook scaffold, interactive documentation setup \\ +\texttt{pre-commit} & \texttt{.pre-commit-config.yaml} wired to ruff, black, and isort \\ +\bottomrule +\end{tabular} +\caption{Selected Rhiza template bundles.} +\label{tab:bundles} +\end{table} + +Dependencies between bundles are declared explicitly. For example, the \texttt{book} bundle requires \texttt{tests} because the documentation site includes auto-generated coverage reports. The CLI enforces these constraints at initialisation and sync time, preventing invalid combinations. + +A downstream project declares the bundles it uses in \texttt{.rhiza/template.yml}: + +\begin{lstlisting}[style=yaml, caption={Example \texttt{.rhiza/template.yml}.}] +repository: jebel-quant/rhiza +ref: v0.8.16 +templates: + - core + - tests + - github + - docker +\end{lstlisting} + +\subsection{Modular Makefile Architecture} +\label{sec:makefile} + +A single monolithic Makefile is a maintenance burden: it is hard to read, harder to extend without conflicts, and impossible to selectively update. Rhiza adopts a three-layer architecture (ADR-0004): + +\begin{enumerate}[leftmargin=*] + \item \textbf{Root \texttt{Makefile}} (nine lines): owned by the downstream project, this file simply includes the Rhiza core. + \item \textbf{\texttt{.rhiza/rhiza.mk}}: the template-managed core. It auto-includes all files matching \texttt{.rhiza/make.d/*.mk} using a glob pattern. + \item \textbf{\texttt{.rhiza/make.d/*.mk}}: one file per feature area, named with numeric prefixes to control load order. Prefixes \texttt{00}--\texttt{19} define configuration variables; \texttt{20}--\texttt{79} define task targets; \texttt{80}--\texttt{99} define hooks. +\end{enumerate} + +Feature modules include \texttt{bootstrap.mk} (environment setup), \texttt{quality.mk} (linting, formatting), \texttt{test.mk} (pytest), \texttt{docs.mk} (MkDocs), \texttt{github.mk} (GitHub CLI helpers), \texttt{docker.mk} (container builds), and \texttt{marimo.mk} (notebook server). Targets use GNU Make's double-colon syntax (\texttt{::}) where appropriate, which allows pre-hook and post-hook targets in separate files to extend a task without modifying the file that defines it. Developers who need project-local shortcuts that should not be committed can place them in \texttt{local.mk}, which is listed in \texttt{.gitignore} by default. + +This architecture provides more than forty documented \texttt{make} targets, all discoverable via \texttt{make help}. + +\subsection{Reproducible Environments with \texttt{uv}} + +A template system that mandates outdated or slow tooling will not be adopted. Rhiza standardises on \texttt{uv}~\cite{uv}---Astral's unified Python version and package manager---for all environment management (ADR-0002). The decision was driven by three considerations: + +\begin{enumerate}[leftmargin=*] + \item \textbf{Single binary}: \texttt{uv} is distributed as a self-contained binary. Installing it does not require Python to be present, which simplifies bootstrapping on CI runners and developer machines alike. + \item \textbf{Speed}: \texttt{uv} resolves and installs packages 10--100$\times$ faster than \texttt{pip}, which meaningfully reduces CI wait times on projects with large dependency trees. + \item \textbf{Lock file}: \texttt{uv.lock} pins the entire transitive dependency graph, ensuring that every environment---local, CI, production---is identical. +\end{enumerate} + +The \texttt{.python-version} file, automatically read by \texttt{uv}, pins the Python interpreter version. Together these two files are the minimal, complete specification of a reproducible Python environment. + +\subsection{Continuous Quality Enforcement} + +Rhiza integrates a layered quality stack that operates at multiple stages of the development cycle: + +\paragraph{Pre-commit hooks.} The \texttt{pre-commit} bundle installs hooks that run \texttt{ruff} for linting and formatting, catching issues before they enter the repository. + +\paragraph{CI pipelines.} The \texttt{github} and \texttt{gitlab} bundles provide workflows that run on every pull request: unit tests across Python 3.11--3.14, type checking with \texttt{ty}, dependency validation with \texttt{deptry}, security scanning with \texttt{pip-audit} and \texttt{bandit}, and static analysis with \texttt{semgrep}. + +\paragraph{Automated dependency updates.} A Renovate configuration file, included in the \texttt{core} bundle, opens pull requests whenever a pinned dependency or the template \texttt{ref} is updated upstream. This closes the loop: Rhiza itself is treated as a dependency that can be automatically proposed for upgrade. + +\subsection{Agentic Workflow Integration} + +A distinguishing feature of recent Rhiza releases is first-class integration with AI coding assistants. The \texttt{github} bundle includes GitHub Actions workflow files that invoke Claude Code~\cite{claudecode} (or optionally GitHub Copilot) for tasks such as: + +\begin{itemize} + \item \textbf{\texttt{analyse-repo}}: produces a structured report of recent changes, open issues, and technical debt. + \item \textbf{\texttt{copilot}}: runs an AI agent on a specified task file, committing the result back to the branch. + \item \textbf{\texttt{summarise-changes}}: generates human-readable release notes from the git log. +\end{itemize} + +These workflows treat AI agents as first-class CI participants: they are triggered by GitHub Actions events, receive a clean environment, and produce reviewable pull-request diffs. This integration reflects the view that AI-assisted development is increasingly part of the standard software engineering toolkit and should be configured consistently across projects. + +% ───────────────────────────────────────────────────────────────────────────── +\section{Operational Experience} +\label{sec:experience} +% ───────────────────────────────────────────────────────────────────────────── + +\subsection{Downstream Adoption} + +Rhiza has been applied to more than twenty Python projects spanning quantitative finance libraries, data-pipeline tools, and developer-tooling packages. Across these projects, the mean time to adopt a new infrastructure improvement---for example, upgrading to a new Python version or enabling a new \texttt{ruff} lint rule---has fallen from several weeks to typically one or two days: the time needed for Renovate to open a PR against the Rhiza template repository and for each downstream project to rebase against the new tag. + +\subsection{Conflict Resolution in Practice} + +The most common source of friction in living-template systems is merge conflicts when a project has customised a file that the template also manages. Rhiza mitigates this in several ways: + +\begin{enumerate}[leftmargin=*] + \item \textbf{Bundle granularity.} By grouping files into coarse bundles, Rhiza reduces the frequency with which any given downstream file is touched by an upstream update. + \item \textbf{Makefile hooks.} Downstream projects that need to extend a Makefile target can do so by adding a \texttt{::} target in \texttt{local.mk} rather than modifying the managed file. The managed file can therefore be updated freely. + \item \textbf{Configuration delegation.} Where possible, tool configuration resides in \texttt{pyproject.toml} (owned by the downstream project) rather than in separate managed files. The template provides a commented scaffold section; the downstream project fills it in. +\end{enumerate} + +\subsection{Architecture Decision Records} + +All non-trivial design decisions in Rhiza are recorded as Architecture Decision Records~\cite{nygard2011adr} in \texttt{docs/adr/}. This practice serves two purposes. First, it provides newcomers with the \emph{reasoning} behind choices, not just the choices themselves---critical for understanding why, say, \texttt{uv} was chosen over \texttt{poetry} or why the CLI and template repository are separate packages. Second, it creates a structured audit trail that makes revisiting decisions easier when the context changes. + +Nine ADRs have been recorded to date, covering: the use of ADRs themselves (ADR-0001), adoption of \texttt{uv} (ADR-0002), CI platform choices (ADR-0003), the modular Makefile architecture (ADR-0004), separation of template from CLI (ADR-0005), the bundle system (ADR-0006), dual GitHub/GitLab support (ADR-0007), Marimo integration (ADR-0008), and pre-commit hook strategy (ADR-0009). + +% ───────────────────────────────────────────────────────────────────────────── +\section{Related Work} +\label{sec:related} +% ───────────────────────────────────────────────────────────────────────────── + +Rhiza sits at the intersection of several research and engineering topics: + +\paragraph{Infrastructure as Code.} The IaC movement~\cite{humble2010continuous} established that infrastructure should be version-controlled, peer-reviewed, and automatically applied. Rhiza applies this principle to the layer \emph{below} application infrastructure: the configuration files and tooling that enable developers to write and test application code. + +\paragraph{Convention over configuration.} Ruby on Rails~\cite{rails} popularised the principle that opinionated defaults reduce cognitive overhead. Rhiza's template bundles encode opinionated defaults for Python projects, with the expectation that most projects will use them as-is and only override where genuinely necessary. + +\paragraph{Package managers and dependency pinning.} The reproducibility guarantees that Nix~\cite{dolstra2004nix}, Cargo, and modern Python tools like \texttt{uv} provide for application dependencies are extended by Rhiza to infrastructure configuration: the \texttt{ref} field in \texttt{template.yml} pins the exact version of infrastructure used, making environments reproducible not just in terms of library versions but also in terms of development tooling. + +\paragraph{Polyglot monorepos.} Organisations using monorepos (e.g.,~\cite{potvin2016google}) face a similar challenge: keeping build and CI configuration consistent across many packages. Rhiza offers a lighter-weight alternative for organisations that prefer independent repositories. + +% ───────────────────────────────────────────────────────────────────────────── +\section{Future Directions} +\label{sec:future} +% ───────────────────────────────────────────────────────────────────────────── + +The Rhiza roadmap identifies several directions for future work: + +\paragraph{Finer-grained synchronisation.} The current sync model operates at the bundle level. Future work will explore file-level and section-level synchronisation, enabling, for example, the update of a single GitHub Actions job within a larger workflow file without touching the rest of the file. + +\paragraph{Validation tooling.} A \texttt{rhiza validate} command currently checks that a project's \texttt{template.yml} is consistent with the files on disk. Expanding this to a full compliance report---identifying files that have drifted from the template---would help teams understand the cost of deferred upgrades. + +\paragraph{Multi-language support.} The core abstractions (versioned templates, named bundles, modular task files) are language-agnostic. Extending Rhiza to support TypeScript and Rust projects would broaden its applicability without requiring fundamental architectural changes. + +\paragraph{Community bundle registry.} Currently, all bundles live in the canonical Rhiza repository. A community registry would allow third parties to publish bundles---for example, a bundle for a particular cloud provider's deployment tooling---that can be mixed in alongside the core set. + +% ───────────────────────────────────────────────────────────────────────────── +\section{Conclusion} +\label{sec:conclusion} +% ───────────────────────────────────────────────────────────────────────────── + +Software infrastructure is not a one-time concern. It requires the same ongoing investment as the application code it supports, yet most organisations lack the tooling to make that investment systematically rather than heroically. Rhiza addresses this gap with a living-template system that treats infrastructure configuration as a versioned, upgradeable dependency. Its key contributions are: + +\begin{enumerate}[leftmargin=*] + \item A two-component architecture that separates template content from the sync mechanism, enabling independent evolution of each. + \item A named bundle system that provides coarse-grained, dependency-aware feature selection. + \item A modular Makefile architecture with hook points that accommodate local customisation without preventing template updates. + \item First-class integration with modern Python tooling (\texttt{uv}, \texttt{ruff}, \texttt{pre-commit}, Marimo) and AI-assisted development workflows. + \item A documented, ADR-driven design process that makes the system's rationale transparent and revisable. +\end{enumerate} + +Rhiza is open source under the MIT licence. The template repository is available at \url{https://github.com/jebel-quant/rhiza} and the CLI is installable via \texttt{uvx rhiza}. + +% ───────────────────────────────────────────────────────────────────────────── +\bibliographystyle{plain} +\begin{thebibliography}{99} + +\bibitem{claudecode} +Anthropic, Claude Code: Agentic coding in the terminal. \url{https://claude.ai/code}, 2025--. + +\bibitem{uv} +Astral, uv: An extremely fast Python package and project manager. \url{https://docs.astral.sh/uv}, 2024--. + +\bibitem{cunningham1992wycash} +Cunningham, W., The WyCash portfolio management system. In \textit{Proc. ACM OOPSLA}, 1992. + +\bibitem{dolstra2004nix} +Dolstra, E., de~Jonge, M., and Visser, E., Nix: A safe and policy-free system for software deployment. In \textit{Proc. LISA}, 2004. + +\bibitem{rails} +Hansson, D.~H., Ruby on Rails. \url{https://rubyonrails.org}, 2004--. + +\bibitem{humble2010continuous} +Humble, J. and Farley, D., \textit{Continuous Delivery: Reliable Software Releases through Build, Test, and Deployment Automation}. Addison-Wesley, 2010. + +\bibitem{cruft} +Kothari, S. et al., cruft. \url{https://cruft.github.io/cruft}, 2019--. + +\bibitem{copier} +Macías, J.~M. et al., Copier. \url{https://copier.readthedocs.io}, 2019--. + +\bibitem{fowler2016infrastructure} +Morris, K., \textit{Infrastructure as Code: Managing Servers in the Cloud}. O'Reilly Media, 2016. + +\bibitem{nygard2011adr} +Nygard, M.~T., Documenting architecture decisions. \url{https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions}, 2011. + +\bibitem{potvin2016google} +Potvin, R. and Levenberg, J., Why Google stores billions of lines of code in a single repository. \textit{Communications of the ACM}, vol.~59, no.~7, pp.~78--87, 2016. + +\bibitem{cookiecutter} +Roy, A., Greenfeld, D. et al., Cookiecutter. \url{https://cookiecutter.readthedocs.io}, 2013--. + +\end{thebibliography} + +\end{document} diff --git a/docs/presentations/README.md b/docs/presentations/README.md new file mode 100644 index 00000000..b57cd7bc --- /dev/null +++ b/docs/presentations/README.md @@ -0,0 +1,9 @@ +# Presentation + +Presentation slides built with [Marp](https://marp.app/) from `PRESENTATION.md`. + +```bash +make presentation # Generate HTML slides +make presentation-pdf # Generate PDF slides +make presentation-serve # Serve with live reload +``` diff --git a/pyproject.toml b/pyproject.toml index 4f4e620c..7c50e84b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "rhiza" -version = "0.8.17" +version = "0.8.20" description = "Reusable configuration templates for modern Python projects" readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index e6a54943..b143a651 100644 --- a/uv.lock +++ b/uv.lock @@ -565,7 +565,7 @@ wheels = [ [[package]] name = "rhiza" -version = "0.8.17" +version = "0.8.20" source = { virtual = "." } [package.dev-dependencies]