diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..690f219 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,72 @@ +name: Benchmark CI + +on: + pull_request: + branches: [main] + +permissions: + issues: write + pull-requests: write + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - name: Checkout PR code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Install critcmp + run: cargo install critcmp + + - name: Run comparison script + run: | + ./scripts/ci_bench.sh + + - name: Post or update benchmark comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const marker = ''; + const body = marker + '\n' + fs.readFileSync('benchmark-report.md', 'utf8'); + + // Get all comments on the PR + const { data: comments } = await github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + }); + + // Look for a comment containing our marker + const existing = comments.find(comment => + comment.body.includes(marker) + ); + + if (existing) { + // Update the existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + console.log('Updated existing benchmark comment'); + } else { + // Create a new comment + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body, + }); + console.log('Created new benchmark comment'); + } diff --git a/scripts/ci_bench.sh b/scripts/ci_bench.sh new file mode 100755 index 0000000..4da01ad --- /dev/null +++ b/scripts/ci_bench.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -euo pipefail + +# Configuration +BENCH_NAME="serde_bench" +WORKTREE_DIR="/tmp/bench-main" +REPORT_FILE="benchmark-report.md" + +# Cleanup trap +cleanup() { + echo "๐Ÿงน Cleaning up..." + git worktree remove --force "$WORKTREE_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +# Fetch and prepare main branch in a clean worktree +echo "๐Ÿ“ฆ Checking out 'main' into temporary worktree..." +git fetch origin main +git worktree add --force "$WORKTREE_DIR" origin/main + +# Run benchmarks on main branch +echo "๐Ÿš€ Running benchmarks on 'main'..." +( + cd "$WORKTREE_DIR" + cargo bench --bench "$BENCH_NAME" -- --save-baseline main +) +mkdir -p ./target/criterion +cp -r "$WORKTREE_DIR/target/criterion" ./target/criterion + +# Run benchmarks on current branch +echo "๐Ÿš€ Running benchmarks on PR branch..." +cargo bench --bench "$BENCH_NAME" -- --save-baseline pr + +# Compare with critcmp +echo "๐Ÿ“Š Comparing benchmarks..." + +cat < "$REPORT_FILE" +## ๐Ÿ“Š Benchmark Comparison Report + +This pull request includes Criterion benchmarks comparing performance to the \`main\` branch. +This comment will automatically update as the benchmarks are re-run on each commit. + +The table below shows **relative ratios** and **timing stats** for each benchmark group: + +\`\`\` +$(critcmp main pr) +\`\`\` + +โœ… Benchmarks completed successfully. + +๐Ÿง  **Notes**: +- These benchmarks are not a pass/fail gate and are informative only. +- Use this as a signal to review performance-sensitive changes. +- Results may be unreliable due to GHA runner hardware variance. +- If results indicate a significant performance regression, run the benchmarks locally to confirm. + +_Reported by the benchmark CI bot_ +EOF + +echo "โœ… Benchmark comparison saved to $REPORT_FILE"