benches.yml

name: Benchmark Suite

on:
  pull_request:
    paths:
      - ".github/workflows/benches.yml"
      - ".cargo/**"
      - "benches/**"
      - "lib/**"
      - "proto/**"
      - "src/**"
      - "tests/**"
      - "build.rs"
      - "Cargo.lock"
      - "Cargo.toml"
      - "rust-toolchain"
  push:
    branches:
      - master
    paths:
      - ".github/workflows/benches.yml"
      - ".cargo/**"
      - "benches/**"
      - "lib/**"
      - "proto/**"
      - "src/**"
      - "tests/**"
      - "build.rs"
      - "Cargo.lock"
      - "Cargo.toml"
      - "rust-toolchain"
  workflow_dispatch:

env:
  VERBOSE: true
  AWS_ACCESS_KEY_ID: "dummy"
  AWS_SECRET_ACCESS_KEY: "dummy"
  CI: true
  AWS_REGION: us-east-1

jobs:
  cancel-previous:
    runs-on: ubuntu-20.04
    timeout-minutes: 3
    if: github.ref != 'refs/heads/master'
    steps:
      - uses: styfle/cancel-workflow-action@0.9.1
        with:
          access_token: ${{ secrets.GITHUB_TOKEN }}
          all_but_latest: true # can cancel workflows scheduled later

  bench:
    name: Bench - Linux (${{ matrix.target }})
    runs-on: [self-hosted, linux, x64, benchmarks]
    strategy:
      matrix:
        target: [bench, bench-remap-functions, bench-remap, bench-languages, bench-metrics, bench-dnstap, bench-transform]
    steps:
      - uses: actions/checkout@v2.3.5
      - uses: actions/cache@v2.1.6
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
      - run: bash scripts/environment/prepare.sh
      - run: echo "::add-matcher::.github/matchers/rust.json"
      # We always make sure to stop any previous sccache process before starting it fresh, that
      # way we can be sure we're using the right credentials for storage, etc.
      - name: Start sccache
        env:
          SCCACHE_REDIS: ${{ secrets.SCCACHE_REDIS }}
          SCCACHE_IDLE_TIMEOUT: 0
        run: |
          sccache --stop-server || true
          sccache --start-server
          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
      - run: make slim-builds
      # First, we run the benchmarks against master to establish our baseline numbers. Our benchmark
      # runners are configured to isolate CPU 0 from OS scheduling so that benchmarks aren't subject
      # to scheduling noise.  We make sure to utilize all CPUs for compiling the benchmark binaries,
      # but only utilize CPU 0 when actually running them.
      - name: Checkout master
        run: |
          git fetch --depth 1 origin master
          git checkout --force origin/master
      - name: Build master benchmarks
        run: taskset -c "0-$(nproc)" make ${{ matrix.target }} CARGO_BENCH_FLAGS="--no-run"
      - name: Run master benchmarks
        run: setarch $(uname -m) -R taskset -c "$(cat /sys/devices/system/cpu/isolated)" make ${{ matrix.target }}
      - name: Checkout PR branch
        run: git clean --force && git checkout --force $GITHUB_SHA
      - name: Build PR benchmarks
        run: taskset -c "0-$(nproc)" make ${{ matrix.target }} CARGO_BENCH_FLAGS="--no-run"
      - name: Run PR benchmarks
        run: |
          mkdir -p target/criterion
          setarch $(uname -m) -R taskset -c "$(cat /sys/devices/system/cpu/isolated)" make ${{ matrix.target }} | tee target/criterion/out
      # We keep the Criterion results no matter what, but where they're uploaded depends on whether
      # this is a PR run or a master run.
      - uses: actions/upload-artifact@v2
        with:
          name: "criterion"
          path: "./target/criterion"
      - name: Upload Criterion data to S3
        run: scripts/upload-benchmarks-s3.sh ${{ matrix.target }}
        if: github.ref == 'refs/heads/master'
        env:
          AWS_ACCESS_KEY_ID: "${{ secrets.CI_AWS_ACCESS_KEY_ID }}"
          AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}"
      # Finally, run the comparison to see if we've hit any regressions.
      - name: Compare benchmarks
        run: |
          echo "Comparing $(git rev-parse HEAD) with $(git rev-parse origin/master)"
          cat target/criterion/out | scripts/check-criterion-output.sh
      - name: Stop sccache
        run: sccache --stop-server

  master-failure:
    name: master-failure
    if: failure() && github.ref == 'refs/heads/master'
    needs:
      - cancel-previous
      - bench
    runs-on: ubuntu-20.04
    steps:
    - name: Discord notification
      env:
        DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
      uses: Ilshidur/action-discord@0.3.2
      with:
        args: "Master benchmarks failed: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}>"