benchmark.yml

name: Benchmark Ledger Backtest
on:
  workflow_call:
    inputs:
      machine:
        type: string
        default: linux_gcc_zen2
      extras:
        type: string
        default: "handholding"
      github-token:
        type: string
        required: true
        description: 'GitHub token for posting comments'
  workflow_dispatch:

permissions:
  contents: read
  issues: write
  pull-requests: write

jobs:
  benchmark:
    runs-on:
      labels: [self-hosted, 512G]
      group: fd-public-repo
    timeout-minutes: 15
    env:
      CC: gcc
      MACHINE: ${{ inputs.machine }}
      EXTRAS: ${{ inputs.extras }}

    steps:
    - name: Checkout current commit
      uses: actions/checkout@v6

    - uses: ./.github/actions/deps
      with:
        extras: +dev

    - uses: ./.github/actions/hugepages
      with:
        count_gigantic: 398
        count_huge: 535
    - uses: actions/setup-node@v6
      with:
        node-version: '14'
    - run: make -j
    - run: echo OBJDIR=$(make help | grep OBJDIR | awk '{print $4}') >> $GITHUB_ENV

    - name: Checkout main for baseline
      uses: actions/checkout@v6
      with:
        ref: main
        clean: false

    - run: make -j

    - name: Run baseline benchmarks
      run: |
        sudo $OBJDIR/bin/firedancer-dev configure init keys
        rm -rf baseline.log
        DUMP_DIR=../dump ./src/flamenco/runtime/tests/run_ledger_backtest.sh \
          -l mainnet-368528500-perf -y 5 -m 2000000 -e 368529500 \
          --log baseline.log --skip-checksum
        SPS_BASELINE=$(grep "Backtest playback done." baseline.log | awk -F 'sec/slot: ' '{print $2}')
        TOTAL_ELAPSED_BASELINE=$(grep "Backtest playback done." baseline.log | awk -F 'elapsed: ' '{print $2}' | awk '{print $1}')
        SNAPSHOT_LOAD_BASELINE=$(grep "accounts from snapshot in" baseline.log | awk -F 'accounts from snapshot in ' '{print $2}' | awk '{print $1}')
        echo "Baseline SEC_SLOT=$SPS_BASELINE"
        echo "Baseline TOTAL_ELAPSED=$TOTAL_ELAPSED_BASELINE"
        echo SPS_BASELINE=$SPS_BASELINE >> $GITHUB_ENV
        echo TOTAL_ELAPSED_BASELINE=$TOTAL_ELAPSED_BASELINE >> $GITHUB_ENV
        echo "Baseline SNAPSHOT_LOAD=$SNAPSHOT_LOAD_BASELINE"
        echo SNAPSHOT_LOAD_BASELINE=$SNAPSHOT_LOAD_BASELINE >> $GITHUB_ENV

        DEFAULT_MEM_BASELINE=$($OBJDIR/bin/firedancer-dev mem --mainnet | grep "Total Memory Locked" | awk -F ':' '/Total Memory Locked/ {gsub(/ bytes.*/,"",$2); printf "%.2f\n", $2/1024/1024/1024}')
        echo "Baseline DEFAULT_MEM=$DEFAULT_MEM_BASELINE"
        echo DEFAULT_MEM_BASELINE=$DEFAULT_MEM_BASELINE >> $GITHUB_ENV

    - name: Checkout new changes
      uses: actions/checkout@v6
      with:
        clean: false

    - run: make -j

    - name: Run new benchmarks
      run: |
        rm -rf new.log
        sudo $OBJDIR/bin/firedancer-dev configure init keys
        DUMP_DIR=../dump ./src/flamenco/runtime/tests/run_ledger_backtest.sh \
          -l mainnet-368528500-perf -y 5 -m 2000000 -e 368529500 \
          --log new.log --skip-checksum
        SPS_NEWVAL=$(grep "Backtest playback done." new.log | awk -F 'sec/slot: ' '{print $2}')
        TOTAL_ELAPSED_NEWVAL=$(grep "Backtest playback done." new.log | awk -F 'elapsed: ' '{print $2}' | awk '{print $1}')
        SNAPSHOT_LOAD_NEWVAL=$(grep "accounts from snapshot in" new.log | awk -F 'accounts from snapshot in ' '{print $2}' | awk '{print $1}')
        echo TOTAL_ELAPSED_NEWVAL=$TOTAL_ELAPSED_NEWVAL >> $GITHUB_ENV
        echo "New SEC_SLOT=$SPS_NEWVAL"
        echo SPS_NEWVAL=$SPS_NEWVAL >> $GITHUB_ENV
        echo "New TOTAL_ELAPSED=$TOTAL_ELAPSED_NEWVAL"
        echo "New SNAPSHOT_LOAD=$SNAPSHOT_LOAD_NEWVAL"
        echo SNAPSHOT_LOAD_NEWVAL=$SNAPSHOT_LOAD_NEWVAL >> $GITHUB_ENV

        DEFAULT_MEM_NEWVAL=$($OBJDIR/bin/firedancer-dev mem --mainnet | grep "Total Memory Locked" | awk -F ':' '/Total Memory Locked/ {gsub(/ bytes.*/,"",$2); printf "%.2f\n", $2/1024/1024/1024}')
        echo "New DEFAULT_MEM=$DEFAULT_MEM_NEWVAL"
        echo DEFAULT_MEM_NEWVAL=$DEFAULT_MEM_NEWVAL >> $GITHUB_ENV

    - name: Compare benchmarks
      run: |
        sps_ratio=$(echo "${SPS_NEWVAL} / ${SPS_BASELINE}" | bc -l)
        echo "Ratio: $sps_ratio"
        echo "SPS_RATIO=$sps_ratio" >> $GITHUB_ENV
        total_elapsed_ratio=$(echo "${TOTAL_ELAPSED_NEWVAL} / ${TOTAL_ELAPSED_BASELINE}" | bc -l)
        echo "Total elapsed ratio: $total_elapsed_ratio"
        echo "TOTAL_ELAPSED_RATIO=$total_elapsed_ratio" >> $GITHUB_ENV
        max_sps_ratio=$(echo "$sps_ratio > 1.10" | bc -l)
        max_total_elapsed_ratio=$(echo "$total_elapsed_ratio > 1.10" | bc -l)
        max_snapshot_load_ratio=$(echo "$SNAPSHOT_LOAD_NEWVAL / $SNAPSHOT_LOAD_BASELINE" | bc -l)
        echo "Snapshot load ratio: $max_snapshot_load_ratio"
        echo "SNAPSHOT_LOAD_RATIO=$max_snapshot_load_ratio" >> $GITHUB_ENV
        default_mem_ratio=$(echo "${DEFAULT_MEM_NEWVAL} / ${DEFAULT_MEM_BASELINE}" | bc -l)
        echo "Default mem ratio: $default_mem_ratio"
        echo "DEFAULT_MEM_RATIO=$default_mem_ratio" >> $GITHUB_ENV


    - name: Generate benchmark.json
      run: |
        sps_ratio=$(echo "${SPS_NEWVAL} / ${SPS_BASELINE}" | bc -l)
        sps_pct_change=$(echo "(${SPS_NEWVAL} - ${SPS_BASELINE}) * 100 / ${SPS_BASELINE}" | bc -l)
        # Force proper numeric formatting with leading zero and limited decimal places
        sps_ratio_fmt=$(printf "%.6f" "$sps_ratio")
        sps_pct_change_fmt=$(printf "%.6f" "$sps_pct_change")
        baseline_sps_fmt=$(printf "%.6f" "$SPS_BASELINE")
        newval_sps_fmt=$(printf "%.6f" "$SPS_NEWVAL")
        # Export for later steps
        echo "SPS_RATIO=$sps_ratio_fmt" >> $GITHUB_ENV
        echo "SPS_PCT_CHANGE=$sps_pct_change_fmt" >> $GITHUB_ENV
        cat <<EOF > benchmark.json
        [
          {"name": "Ledger SEC/slot (Baseline)", "unit": "sec/slot", "value": ${baseline_sps_fmt}},
          {"name": "Ledger SEC/slot (New)", "unit": "sec/slot", "value": ${newval_sps_fmt}},
          {"name": "Performance ratio (x)", "unit": "x", "value": ${sps_ratio_fmt}},
          {"name": "Performance change (%)", "unit": "%", "value": ${sps_pct_change_fmt}}
        ]
        EOF

    - name: Fetch gh-pages branch
      run: |
        git fetch origin gh-pages:gh-pages

    - name: Publish benchmark results
      uses: benchmark-action/github-action-benchmark@v1
      with:
        name: "Ledger Backtest Benchmark"
        tool: "customSmallerIsBetter"
        output-file-path: benchmark.json
        github-token: ${{ inputs.github-token || github.token }}
        skip-fetch-gh-pages: true
        auto-push: false
        alert-threshold: "0%"
        fail-on-alert: false
        comment-always: false

    - name: Post PR Comment with baseline vs new (Markdown table)
      if: github.event_name == 'pull_request' || github.event_name == 'issue_comment'
      uses: actions/github-script@v8
      with:
        github-token: ${{ inputs.github-token || github.token }}
        script: |
          const baseline_sps = parseFloat(process.env.SPS_BASELINE);
          const newval_sps = parseFloat(process.env.SPS_NEWVAL);
          const sps_ratio = parseFloat(process.env.SPS_RATIO);
          const sps_pct_change = ((newval_sps - baseline_sps) * 100 / baseline_sps).toFixed(3);

          const baseline_total_elapsed = parseFloat(process.env.TOTAL_ELAPSED_BASELINE);
          const newval_total_elapsed = parseFloat(process.env.TOTAL_ELAPSED_NEWVAL);
          const total_elapsed_ratio = parseFloat(process.env.TOTAL_ELAPSED_RATIO);
          const total_elapsed_pct_change = ((newval_total_elapsed - baseline_total_elapsed) * 100 / baseline_total_elapsed).toFixed(3);

          const baseline_snapshot_load = parseFloat(process.env.SNAPSHOT_LOAD_BASELINE);
          const newval_snapshot_load = parseFloat(process.env.SNAPSHOT_LOAD_NEWVAL);
          const snapshot_load_ratio = parseFloat(process.env.SNAPSHOT_LOAD_RATIO);
          const snapshot_load_pct_change = ((newval_snapshot_load - baseline_snapshot_load) * 100 / baseline_snapshot_load).toFixed(3);

          const baseline_default_mem = parseFloat(process.env.DEFAULT_MEM_BASELINE);
          const newval_default_mem = parseFloat(process.env.DEFAULT_MEM_NEWVAL);
          const default_mem_ratio = parseFloat(process.env.DEFAULT_MEM_RATIO);
          const default_mem_pct_change = ((newval_default_mem - baseline_default_mem) * 100 / baseline_default_mem).toFixed(3);

          // Helper for status icon
          const changeIcon = (pct, max) => {
            const num = parseFloat(pct);
            return num > max ? "⚠️" : "✅";
          };

          // Format numbers with units and backticks
          const fmt = (val, unit) => `\`${val} ${unit}\``;

          const table = `
          ## Performance Measurements ⏳
          | Suite  | Baseline | New   | Change |
          |--------|----------|-------|--------|
          | backtest \`mainnet-368528500-perf\` per slot | ${fmt(baseline_sps, "s")} | ${fmt(newval_sps, "s")} | \`${sps_pct_change}%\` ${changeIcon(sps_pct_change, 5.0)} |
          | backtest \`mainnet-368528500-perf\` snapshot load | ${fmt(baseline_snapshot_load, "s")} | ${fmt(newval_snapshot_load, "s")} | \`${snapshot_load_pct_change}%\` ${changeIcon(snapshot_load_pct_change, 5.0)} |
          | backtest \`mainnet-368528500-perf\` total elapsed | ${fmt(baseline_total_elapsed, "s")} | ${fmt(newval_total_elapsed, "s")} | \`${total_elapsed_pct_change}%\` ${changeIcon(total_elapsed_pct_change, 5.0)} |
          | \`firedancer mem\` usage with \`mainnet.toml\` | ${fmt(baseline_default_mem, "GiB")} | ${fmt(newval_default_mem, "GiB")} | \`${default_mem_pct_change}%\` ${changeIcon(default_mem_pct_change, 5.0)} |
          `;

          const body = table;

          github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body
          });