runtime-regression.yml

name: Runtime Regression

on:
  push:
    branches:
      - main
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  runtime-regression:
    name: Runtime regression
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
      HEAD_SHA: ${{ github.sha }}
      OMP_NUM_THREADS: "1"
      MKL_NUM_THREADS: "1"
      OPENBLAS_NUM_THREADS: "1"
      NUMEXPR_NUM_THREADS: "1"
      PYTHONHASHSEED: "0"
    steps:
      - name: Checkout branch
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
      - name: Ensure base commit is available
        shell: bash
        run: |
          set -euo pipefail
          # Push events for the first commit can have an all-zero "before" SHA.
          if [[ "$BASE_SHA" =~ ^0+$ ]]; then
            BASE_SHA="$(git rev-parse "$HEAD_SHA^")"
            echo "BASE_SHA=$BASE_SHA" >> "$GITHUB_ENV"
          fi
          if ! git fetch --no-tags --prune origin "$BASE_SHA"; then
            echo "Failed to fetch BASE_SHA=$BASE_SHA from origin." >&2
          fi
          if ! git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then
            echo "BASE_SHA $BASE_SHA is not available locally after fetch." >&2
            exit 1
          fi
      - name: Set up pixi
        uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4
        with:
          environments: benchmark
          cache: true
      - name: Run benchmarks (base vs head) and compare
        shell: bash
        run: |
          set -euo pipefail

          ROOT="$PWD"
          mkdir -p "$ROOT/glum_benchmarks/results"
          BASE_WT="$(mktemp -d /tmp/glum-base-XXXXXX)"
          HEAD_WT="$(mktemp -d /tmp/glum-head-XXXXXX)"

          cleanup() {
            git worktree remove --force "$BASE_WT" >/dev/null 2>&1 || true
            git worktree remove --force "$HEAD_WT" >/dev/null 2>&1 || true
            rm -rf "$BASE_WT" "$HEAD_WT" >/dev/null 2>&1 || true
          }
          trap cleanup EXIT

          git worktree add --detach "$BASE_WT" "$BASE_SHA"
          git worktree add --detach "$HEAD_WT" "$HEAD_SHA"

          CACHE_DIR="$ROOT/glum_benchmarks/results/cache-ci"

          run_ref () {
            local wt="$1"
            local run_name="$2"
            local do_warmup="${3:-false}"
            cd "$wt"
            pixi run -e benchmark python -m pip install \
              --no-build-isolation --no-deps --disable-pip-version-check \
              -e .

            if [ "$do_warmup" = "true" ]; then
              local warmup_run_name="warmup-${run_name}"
              GLM_BENCHMARKS_CACHE="$CACHE_DIR" PYTHONPATH="$ROOT" pixi run -e benchmark python "$ROOT/glum_benchmarks/run_benchmarks.py" \
                --config "$ROOT/glum_benchmarks/config_ci.yaml" \
                --run-name "$warmup_run_name"
              rm -rf "$ROOT/glum_benchmarks/results/$warmup_run_name"
            fi

            # Measured pass
            GLM_BENCHMARKS_CACHE="$CACHE_DIR" PYTHONPATH="$ROOT" pixi run -e benchmark python "$ROOT/glum_benchmarks/run_benchmarks.py" \
              --config "$ROOT/glum_benchmarks/config_ci.yaml" \
              --run-name "$run_name"
          }

          run_ref "$BASE_WT" "ci-base" true
          run_ref "$HEAD_WT" "ci-head"

          cd "$ROOT"
          PYTHONPATH="$ROOT" pixi run -e benchmark python glum_benchmarks/compare_results.py \
            --base glum_benchmarks/results/ci-base/results.csv \
            --head glum_benchmarks/results/ci-head/results.csv \
            --config glum_benchmarks/config_ci.yaml \
            --summary-out glum_benchmarks/results/ci_runtime_summary.md
      - name: Publish summary
        if: always()
        run: |
          echo "## Runtime regression summary" >> "$GITHUB_STEP_SUMMARY"
          if [ -f glum_benchmarks/results/ci_runtime_summary.md ]; then
            cat glum_benchmarks/results/ci_runtime_summary.md >> "$GITHUB_STEP_SUMMARY"
          else
            echo "_No summary produced (benchmark step failed early)_" >> "$GITHUB_STEP_SUMMARY"
          fi
      - name: Upload runtime artifacts
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: ci-runtime-regression
          if-no-files-found: warn
          path: |
            glum_benchmarks/results/ci_runtime_summary.md
            glum_benchmarks/results/ci-base/**/*
            glum_benchmarks/results/ci-head/**/*