benchmark.yml

# Benchmark libgit2 against the git reference implementation.
name: Benchmark

on:
  workflow_dispatch:
    inputs:
      suite:
        description: Benchmark suite to run
      debug:
        type: boolean
        description: Debugging output
      deploy:
        type: boolean
        description: Deploy the benchmark site
  schedule:
  - cron: '15 4 * * *'

permissions:
  contents: read

jobs:
  # Run our benchmarks. We build a matrix with the various build
  # targets and their details. Unlike our CI builds, we run these
  # directly on the VM instead of in containers since we do not
  # need the breadth of platform diversity.
  build:
    # Only run scheduled workflows on the main repository; prevents people
    # from using build minutes on their forks.
    if: github.repository == 'libgit2/libgit2'

    strategy:
      matrix:
        platform:
        - name: "Linux (clang, OpenSSL)"
          id: linux
          os: ubuntu-latest
          setup-script: ubuntu
          env:
            CC: clang
            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
        - name: "macOS"
          id: macos
          os: macos-latest
          setup-script: osx
          env:
            CC: clang
            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
            PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
        - name: "Windows (amd64, Visual Studio)"
          id: windows
          os: windows-2022
          setup-script: win32
          env:
            ARCH: amd64
            CMAKE_GENERATOR: Visual Studio 17 2022
            CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
            CMAKE_BUILD_OPTIONS: --config RelWithDebInfo
      fail-fast: false
    name: "Benchmark ${{ matrix.platform.name }}"
    env: ${{ matrix.platform.env }}
    runs-on: ${{ matrix.platform.os }}
    steps:
    - name: Check out repository
      uses: actions/checkout@v4
      with:
        path: source
        fetch-depth: 0
    - name: Set up benchmark environment
      run: source/ci/setup-${{ matrix.platform.setup-script }}-benchmark.sh
      shell: bash
      if: matrix.platform.setup-script != ''
    - name: Clone resource repositories
      run: |
        # TODO:
        # we need a superior way to package the benchmark resources; lfs
        # is too expensive
        # git lfs install
        # git clone https://github.com/libgit2/benchmark-resources resources

        git clone --bare https://github.com/git/git resources/git

        # TODO:
        # avoid linux temporarily; the linux blame benchmarks are simply
        # too slow to use
        # git clone --bare https://github.com/torvalds/linux resources/linux
    - name: Build
      run: |
        mkdir build && cd build
        ../source/ci/build.sh
      shell: bash
    - name: Benchmark
      run: |
        # TODO:
        # avoid benchmark resource path currently
        #export BENCHMARK_RESOURCES_PATH="$(pwd)/resources"
        export BENCHMARK_GIT_PATH="$(pwd)/resources/git"
        # avoid linux temporarily; the linux blame benchmarks are simply
        # too slow to use
        # export BENCHMARK_LINUX_PATH="$(pwd)/resources/linux"

        if [[ "$(uname -s)" == MINGW* ]]; then
          GIT2_CLI="$(cygpath -w $(pwd))\\build\\RelWithDebInfo\\git2"
        else
          GIT2_CLI="$(pwd)/build/git2"
        fi

        if [ "${{ github.event.inputs.suite }}" != "" ]; then
          SUITE_FLAG="--suite ${{ github.event.inputs.suite }}"
        fi

        if [ "${{ github.event.inputs.debug }}" = "true" ]; then
          DEBUG_FLAG="--debug"
        fi

        mkdir benchmark && cd benchmark
        ../source/tests/benchmarks/benchmark.sh \
            ${SUITE_FLAG} ${DEBUG_FLAG} \
            --admin \
            --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \
            --json benchmarks.json --flamegraph --zip benchmarks.zip
      shell: bash
    - name: Upload results
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-${{ matrix.platform.id }}
        path: benchmark
      if: always()

  # Publish the results
  publish:
    name: Publish results
    needs: [ build ]
    if: always() && github.repository == 'libgit2/libgit2'
    runs-on: ubuntu-latest
    steps:
    - name: Check out benchmark repository
      uses: actions/checkout@v4
      with:
        repository: libgit2/benchmarks
        path: site
        fetch-depth: 0
        ssh-key: ${{ secrets.BENCHMARKS_PUBLISH_KEY }}
    - name: Download test results
      uses: actions/download-artifact@v4
    - name: Generate API
      run: |
        # Move today's benchmark run into the right place
        for platform in linux macos windows; do
          TIMESTAMP=$(jq .time.start < "benchmark-${platform}/benchmarks.json")
          TIMESTAMP_LEN=$(echo -n ${TIMESTAMP} | wc -c | xargs)
          DENOMINATOR=1
          if [ "${TIMESTAMP_LEN}" = "19" ]; then
            DENOMINATOR="1000000000"
          elif [ "${TIMESTAMP_LEN}" = "13" ]; then
            DENOMINATOR="1000"
          else
            echo "unknown timestamp"
            exit 1
          fi

          if [[ "$(uname -s)" == "Darwin" ]]; then
            DATE=$(date -R -r $(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d")
          else
            DATE=$(date -d @$(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d")
          fi

          # move the complete results in
          mkdir -p "site/public/api/runs/${DATE}"
          cp "benchmark-${platform}/benchmarks.json" "site/public/api/runs/${DATE}/${platform}.json"

          # unzip the individual results
          PLATFORM_TEMP=$(mktemp -d)
          unzip "benchmark-${platform}/benchmarks.zip" -d "${PLATFORM_TEMP}"

          mkdir -p "site/public/api/runs/${DATE}/${platform}"
          find "${PLATFORM_TEMP}" -name \*\.svg -exec cp {} "site/public/api/runs/${DATE}/${platform}" \;
        done

        (cd site && node scripts/aggregate.js)
      shell: bash

      # in debug mode, don't deploy the site; only create a zip file and
      # upload it for debugging
    - name: Upload site
      uses: actions/upload-artifact@v4
      with:
        name: site
        path: site
      if: github.event_name == 'workflow_dispatch'
    - name: Publish API
      run: |
        git config user.name 'Benchmark Site Generation' &&
        git config user.email 'libgit2@users.noreply.github.com' &&
        git add . &&
        git commit --allow-empty -m"benchmark update ${DATE}" &&
        git push origin main
      shell: bash
      working-directory: site
      if: github.event_name == 'schedule' || github.event.inputs.deploy == 'true'