ci.yml

name: CI Pipeline

on:
  push:
    branches: [main, develop, "002-ai-tutorial-stages"]
  pull_request:
    branches: [main, develop]
  workflow_dispatch:

jobs:
  lint-and-test:
    name: Lint & Test
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.11", "3.12"]

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v2
        with:
          version: "latest"

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Create virtual environment
        run: uv venv --python ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          source .venv/bin/activate
          uv pip install -e ".[dev,stage3]"

      - name: Run black (code formatting check)
        run: |
          source .venv/bin/activate
          black --check scripts/ tests/

      - name: Run ruff (linting)
        run: |
          source .venv/bin/activate
          ruff check scripts/ tests/

      - name: Run mypy (type checking)
        run: |
          source .venv/bin/activate
          mypy scripts/ --ignore-missing-imports

      - name: Run pytest (unit tests)
        run: |
          source .venv/bin/activate
          pytest tests/ -v --cov=scripts --cov-report=xml --cov-report=term

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ./coverage.xml
          flags: unittests
          name: codecov-umbrella

  validate-entities:
    name: Validate YAML Entities
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v2

      - name: Set up Python 3.11
        run: uv python install 3.11

      - name: Create virtual environment
        run: uv venv --python 3.11

      - name: Install dependencies
        run: |
          source .venv/bin/activate
          uv pip install -e ".[dev]"

      - name: Validate entity configs
        run: |
          source .venv/bin/activate
          if [ -d "configs/content" ]; then
            python scripts/validation/validate-entities.py --config-dir configs/content
          else
            echo "⚠️ configs/content not found, skipping validation"
          fi

  verify-data:
    name: Verify Data Integrity
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v2

      - name: Set up Python 3.11
        run: uv python install 3.11

      - name: Create virtual environment
        run: uv venv --python 3.11

      - name: Install dependencies
        run: |
          source .venv/bin/activate
          uv pip install -e ".[stage3]"

      - name: Verify data checksums
        run: |
          source .venv/bin/activate
          if [ -f "scripts/data/verify.py" ]; then
            python scripts/data/verify.py --checksums-only
          else
            echo "⚠️ scripts/data/verify.py not found, skipping verification"
          fi

  build-docs:
    name: Build Documentation
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v2

      - name: Set up Python 3.11
        run: uv python install 3.11

      - name: Create virtual environment
        run: uv venv --python 3.11

      - name: Install dependencies
        run: |
          source .venv/bin/activate
          uv pip install -e ".[docs]"

      - name: Build MkDocs site
        run: |
          source .venv/bin/activate
          mkdocs build --strict

      - name: Upload documentation artifacts
        uses: actions/upload-artifact@v3
        with:
          name: docs-site
          path: site/

  test-notebooks:
    name: Test Jupyter Notebooks
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v2

      - name: Set up Python 3.11
        run: uv python install 3.11

      - name: Create virtual environment
        run: uv venv --python 3.11

      - name: Install dependencies
        run: |
          source .venv/bin/activate
          uv pip install -e ".[dev,stage3]"

      - name: Test notebooks with nbval
        run: |
          source .venv/bin/activate
          if [ -d "notebooks/stage3" ] && [ "$(ls -A notebooks/stage3/*.ipynb 2>/dev/null)" ]; then
            pytest --nbval notebooks/stage3/ --ignore=notebooks/stage3/.ipynb_checkpoints
          else
            echo "⚠️ No notebooks found in notebooks/stage3, skipping tests"
          fi

  security-scan:
    name: Security Scan
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v2

      - name: Set up Python 3.11
        run: uv python install 3.11

      - name: Create virtual environment
        run: uv venv --python 3.11

      - name: Install dependencies
        run: |
          source .venv/bin/activate
          uv pip install -e ".[dev]"

      - name: Run bandit security scan
        run: |
          source .venv/bin/activate
          pip install bandit
          bandit -r scripts/ -f json -o bandit-report.json || true

      - name: Run safety check
        run: |
          source .venv/bin/activate
          pip install safety
          safety check --json || true