Switch benchmarks to comment-triggered invocation

.github/workflows/benchmark-help.yml

@@ -0,0 +1,35 @@
+name: Benchmark Help
+
+on:
+  pull_request:
+    types: [opened]
+    paths:
+      - 'moonfish/**'
+      - 'opening_book/**'
+      - 'scripts/**'
+      - 'pyproject.toml'
+      - 'requirements.txt'
+
+permissions:
+  pull-requests: write
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Post benchmark instructions
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        gh pr comment ${{ github.event.pull_request.number }} \
+          --repo ${{ github.repository }} \
+          --body '### Benchmarks
+
+        The following benchmarks are available for this PR:
+
+        | Command | Description |
+        |---------|-------------|
+        | `/run-nps-benchmark` | NPS speed benchmark (depth 5, 48 positions) |
+        | `/run-stockfish-benchmark` | Stockfish strength benchmark (300 games) |
+
+        Post a comment with the command to trigger a benchmark run.'

.github/workflows/benchmark.yml

@@ -1,14 +1,8 @@
 name: Stockfish Benchmark
 
 on:
-  pull_request:
-    paths:
-      # Only run benchmarks when engine code changes
-      - 'moonfish/**'
-      - 'opening_book/**'
-      - 'scripts/**'
-      - 'pyproject.toml'
-      - 'requirements.txt'
+  issue_comment:
+    types: [created]
 
 permissions:
   contents: read
@@ -19,20 +13,22 @@ env:
   MOONFISH_OPENING_BOOK: ${{ github.workspace }}/opening_book/cerebellum.bin
 
 jobs:
-  react-start:
+  react:
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
+    if: >-
+      github.event.issue.pull_request &&
+      contains(github.event.comment.body, '/run-stockfish-benchmark')
     steps:
-    - name: Add eyes reaction to PR
+    - name: React to comment
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/reactions \
+        gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
           -f content='eyes' --silent || true
 
   benchmark:
     runs-on: ubuntu-latest
-    needs: react-start
+    needs: react
     strategy:
       fail-fast: false
       matrix:
@@ -47,6 +43,11 @@ jobs:
         lfs: false
         fetch-depth: 0
 
+    - name: Checkout PR branch
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: gh pr checkout ${{ github.event.issue.number }}
+
     - name: Ensure opening book
       run: |
         set -euo pipefail
@@ -308,24 +309,16 @@ jobs:
         cat pr-comment.md >> $GITHUB_STEP_SUMMARY
 
     - name: Comment on PR
-      if: github.event_name == 'pull_request'
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        gh pr comment ${{ github.event.pull_request.number }} --body-file pr-comment.md
+        gh pr comment ${{ github.event.issue.number }} --body-file pr-comment.md
 
-    - name: Update PR reaction (eyes -> thumbs up)
-      if: github.event_name == 'pull_request'
+    - name: React with thumbs up on completion
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        # Remove eyes reaction
-        REACTIONS=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/reactions --jq '.[] | select(.content == "eyes") | .id' || true)
-        for ID in $REACTIONS; do
-          gh api -X DELETE repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/reactions/$ID --silent || true
-        done
-        # Add thumbs up
-        gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/reactions \
+        gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
           -f content='+1' --silent || true
 
     - name: Upload aggregated results

.github/workflows/nps-benchmark.yml

@@ -0,0 +1,103 @@
+name: NPS Benchmark
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+env:
+  UV_SYSTEM_PYTHON: 1
+
+jobs:
+  nps-benchmark:
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.issue.pull_request &&
+      contains(github.event.comment.body, '/run-nps-benchmark')
+
+    steps:
+    - name: React to comment
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
+          -f content='eyes' --silent || true
+
+    - uses: actions/checkout@v4
+
+    - name: Checkout PR branch
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: gh pr checkout ${{ github.event.issue.number }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        cache-dependency-glob: "requirements.txt"
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: make install
+
+    - name: Run NPS benchmark
+      run: |
+        python -m moonfish.main --mode bench --depth 5 2>&1 | tee bench-output.txt
+
+    - name: Parse results and comment on PR
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        OUTPUT="bench-output.txt"
+
+        TOTAL_TIME=$(grep "^Total time" "$OUTPUT" | awk '{print $NF}')
+        TOTAL_NODES=$(grep "^Nodes searched" "$OUTPUT" | awk '{print $NF}')
+        NPS=$(grep "^Nodes/second" "$OUTPUT" | awk '{print $NF}')
+        NUM_POSITIONS=$(grep -c "^Position" "$OUTPUT")
+
+        # Format numbers with commas
+        TOTAL_NODES_FMT=$(printf "%'d" "$TOTAL_NODES")
+        NPS_FMT=$(printf "%'d" "$NPS")
+
+        # Build per-position breakdown
+        PER_POS=$(grep "^Position" "$OUTPUT")
+
+        cat > pr-comment.md << EOF
+        ## ⚡ NPS Benchmark Results
+
+        | Metric | Value |
+        |--------|-------|
+        | Depth | 5 |
+        | Positions | $NUM_POSITIONS |
+        | Total nodes | $TOTAL_NODES_FMT |
+        | Total time | ${TOTAL_TIME}s |
+        | Nodes/second | $NPS_FMT |
+
+        > **Node count is the primary signal** — it's deterministic and catches search behavior changes. If the node count changes, the PR changed search behavior. NPS is informational only (CI runner performance varies).
+
+        <details><summary>Per-position breakdown</summary>
+
+        \`\`\`
+        $PER_POS
+        \`\`\`
+
+        </details>
+        EOF
+
+        # Remove leading whitespace from heredoc
+        sed -i 's/^        //' pr-comment.md
+
+        cat pr-comment.md >> $GITHUB_STEP_SUMMARY
+
+        gh pr comment ${{ github.event.issue.number }} --body-file pr-comment.md
+
+        # Add thumbs up reaction to signal completion
+        gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
+          -f content='+1' --silent || true