feat(build): Add benchmarks framework (WIP)

.github/workflows/benchmark-backfill.yml

@@ -0,0 +1,105 @@
+name: Benchmarks (Backfill)
+
+# CodSpeed uses Valgrind to count CPU instructions.
+# Valgrind wraps `hatch run` and traces child processes (hatch → uv → python → pytest).
+# Do NOT skip hatch/uv in Valgrind args. Skipping a parent process causes Valgrind to lose track of ALL children (no data captured).
+# See `benchmarks/README.md` for more information.
+
+on:
+  workflow_dispatch:
+    inputs:
+      from_commit:
+        description: 'Start commit SHA (older)'
+        required: true
+        type: string
+      to_commit:
+        description: 'End commit SHA (newer, default: HEAD)'
+        required: false
+        default: 'HEAD'
+        type: string
+      benchmark_set:
+        description: 'Benchmark set to run'
+        required: false
+        default: 'fast'
+        type: choice
+        options:
+          - fast
+          - full
+      force:
+        description: 'Force recalculation (reserved for future use)'
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      commits: ${{ steps.get-commits.outputs.commits }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get commits between range
+        id: get-commits
+        run: |
+          FROM="${{ inputs.from_commit }}"
+          TO="${{ inputs.to_commit }}"
+
+          # Get list of commits from oldest to newest
+          COMMITS=$(git rev-list --reverse "$FROM^..$TO" | head -20)
+
+          # Convert to JSON array
+          JSON_COMMITS=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "commits=$JSON_COMMITS" >> $GITHUB_OUTPUT
+          echo "Will benchmark these commits:"
+          echo "$COMMITS"
+
+  backfill:
+    needs: prepare
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 1  # Run sequentially to avoid overwhelming CodSpeed
+      matrix:
+        commit: ${{ fromJson(needs.prepare.outputs.commits) }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ matrix.commit }}
+          fetch-depth: 0
+
+      # Copy benchmarks from the workflow's source branch (to run up-to-date benchmarks on older commits)
+      - name: Ensure benchmarks directory exists
+        run: |
+          if [ ! -d "benchmarks" ]; then
+            echo "Benchmarks directory not found in commit ${{ matrix.commit }}"
+            echo "Fetching from workflow source..."
+            git fetch origin ${{ github.ref }}
+            git checkout FETCH_HEAD -- benchmarks/ || echo "No benchmarks to copy"
+          fi
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Run benchmarks with CodSpeed
+        uses: CodSpeedHQ/action@v4
+        with:
+          run: |
+            if [ "${{ inputs.benchmark_set }}" = "fast" ]; then
+              hatch run benchmark:run --codspeed -m "not slow and not integration"
+            else
+              hatch run benchmark:run --codspeed
+            fi

.github/workflows/benchmarks-nightly.yml

@@ -0,0 +1,111 @@
+name: Benchmarks (Nightly)
+
+# CodSpeed uses Valgrind to count CPU instructions.
+# Valgrind wraps `hatch run` and traces child processes (hatch → uv → python → pytest).
+# Do NOT skip hatch/uv in Valgrind args. Skipping a parent process causes Valgrind to lose track of ALL children (no data captured).
+# See `benchmarks/README.md` for more information.
+
+on:
+  schedule:
+    - cron: "0 2 * * *" # 2 AM UTC daily
+  workflow_dispatch: # Allow manual trigger
+  pull_request:
+    types: [labeled]
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      should_run: ${{ steps.check.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Check if should run
+        id: check
+        run: |
+          # Always run for label triggers and manual dispatch
+          if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "should_run=true" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # For scheduled runs, skip if HEAD hasn't changed in 24 hours
+          LAST_COMMIT_TIME=$(git log -1 --format=%ct)
+          NOW=$(date +%s)
+          HOURS_AGO=$(( (NOW - LAST_COMMIT_TIME) / 3600 ))
+
+          if [ "$HOURS_AGO" -gt 24 ]; then
+            echo "No commits in the last 24 hours, skipping nightly benchmark"
+            echo "should_run=false" >> $GITHUB_OUTPUT
+          else
+            echo "should_run=true" >> $GITHUB_OUTPUT
+          fi
+
+  integration-benchmark:
+    needs: check-changes
+    if: |
+      needs.check-changes.outputs.should_run == 'true' &&
+      (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Required for baseline comparison
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Run full benchmarks with CodSpeed
+        uses: CodSpeedHQ/action@v4
+        env:
+          # Skip shell and git in various install locations to reduce noise
+          CODSPEED_VALGRIND_ARGS: "--trace-children=yes --trace-children-skip=*/sh,*/bash,*/git,/bin/sh,/usr/bin/git"
+          UV_NO_PROGRESS: "1"
+        with:
+          run: hatch run benchmark:run --codspeed
+
+      - name: Generate benchmark JSON (fallback)
+        if: always()
+        env:
+          UV_NO_PROGRESS: "1"
+        run: |
+          hatch run benchmark:run \
+            --benchmark-only \
+            --benchmark-json=benchmark-results-full.json || true
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: benchmark-results-full
+          path: benchmark-results-full.json
+          retention-days: 90
+
+      - name: Run memory profiling
+        if: always()
+        env:
+          UV_NO_PROGRESS: "1"
+        run: |
+          hatch run benchmark:memory \
+            --memray-bin-path=memray-results || true
+
+      - name: Upload memory results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: memory-results
+          path: memray-results/
+          retention-days: 90

.github/workflows/benchmarks.yml

@@ -0,0 +1,54 @@
+name: Benchmarks
+
+# CodSpeed uses Valgrind to count CPU instructions.
+# Valgrind wraps `hatch run` and traces child processes (hatch → uv → python → pytest).
+# Do NOT skip hatch/uv in Valgrind args. Skipping a parent process causes Valgrind to lose track of ALL children (no data captured).
+# See `benchmarks/README.md` for more information.
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  benchmark-fast:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Required for baseline comparison
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Run fast benchmarks with CodSpeed
+        uses: CodSpeedHQ/action@v4
+        with:
+          run: hatch run benchmark:run --codspeed -m "not slow and not integration"
+
+      - name: Generate benchmark JSON (fallback)
+        if: always()
+        run: |
+          hatch run benchmark:run \
+            --benchmark-only \
+            --benchmark-json=benchmark-results.json \
+            -m "not slow and not integration" || true
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: benchmark-results
+          path: benchmark-results.json
+          retention-days: 30