From b696ce93e8c50831a2ac1a313031c4bfdfc66a0a Mon Sep 17 00:00:00 2001 From: Allison Piper Date: Sat, 30 Aug 2025 19:23:40 +0000 Subject: [PATCH 1/3] Add targeted build/test + bisect scripts and workflow - build_and_test_targets.sh: reproducible single-target configure/build/test (ninja, ctest, lit) - git_bisect.sh: wrapper to automate regression testing with minimal targets - Workflow/Bisect: GitHub Actions job to run bisect remotely and generate Markdown summaries --- .github/workflows/git-bisect.yml | 324 ++++++++++++++++++ ci/util/build_and_test_targets.sh | 166 +++++++++ ci/util/git_bisect.sh | 277 +++++++++++++++ .../development/build_and_bisect_tools.rst | 136 ++++++++ docs/cccl/development/index.rst | 2 + 5 files changed, 905 insertions(+) create mode 100644 .github/workflows/git-bisect.yml create mode 100755 ci/util/build_and_test_targets.sh create mode 100755 ci/util/git_bisect.sh create mode 100644 docs/cccl/development/build_and_bisect_tools.rst diff --git a/.github/workflows/git-bisect.yml b/.github/workflows/git-bisect.yml new file mode 100644 index 00000000000..7b4d5e72c70 --- /dev/null +++ b/.github/workflows/git-bisect.yml @@ -0,0 +1,324 @@ +name: "Git Bisect" + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +on: + workflow_dispatch: + inputs: + runner: + description: "Runner label. Ex: 'linux-amd64-cpu16', 'linux-amd64-gpu-rtxa6000-latest-1'" + required: true + default: linux-amd64-cpu16 + type: string + launch_args: + description: "'.devcontainer/launch.sh -d' add'l args. Ex: '--ctk 12.9 --host gcc13'" + required: false + default: "" + type: string + preset: + description: "CMake preset. Ex: cub-cpp20" + required: true + default: "" + type: string + cmake_options: + description: "Additional options passed to CMake preset configure (e.g. -DVAR=ON)" + required: false + default: "" + type: string + build_targets: + description: "Space separated ninja build targets to build." + required: false + default: "" + type: string + ctest_targets: + description: "Space separated CTest -R regexes to run." + required: false + default: "" + type: string + lit_precompile_tests: + description: "Space-separated libcudacxx lit test paths to compile without execution" + required: false + default: "" + type: string + lit_tests: + description: "Space-separated libcudacxx lit test paths to compile and execute" + required: false + default: "" + type: string + good_ref: + description: "Good ref/sha/tag/branch. Defaults to latest release tag. '-Nd' means 'N days ago on main'." + required: false + type: string + bad_ref: + description: "Bad ref/sha/tag/branch. Defaults to main. '-Nd' means 'N days ago on main'." + required: false + type: string + workflow_call: + inputs: + runner: + description: "Runner label. Ex: 'linux-amd64-cpu16', 'linux-amd64-gpu-rtxa6000-latest-1'" + required: true + default: linux-amd64-cpu16 + type: string + launch_args: + description: "'.devcontainer/launch.sh -d' add'l args. Ex: '--ctk 12.9 --host gcc13'" + required: false + default: "" + type: string + preset: + description: "CMake preset. Ex: cub-cpp20" + required: true + default: "" + type: string + cmake_options: + description: "Additional options passed to CMake preset configure (e.g. -DVAR=ON)" + required: false + default: "" + type: string + build_targets: + description: "Space separated ninja build targets to build." + required: false + default: "" + type: string + ctest_targets: + description: "Space separated CTest -R regexes to run." + required: false + default: "" + type: string + lit_precompile_tests: + description: "Space-separated libcudacxx lit test paths to compile without execution" + required: false + default: "" + type: string + lit_tests: + description: "Space-separated libcudacxx lit test paths to compile and execute" + required: false + default: "" + type: string + good_ref: + description: "Good ref/sha/tag/branch. Defaults to latest release tag. '-Nd' means 'N days ago on main'." + required: false + type: string + bad_ref: + description: "Bad ref/sha/tag/branch. Defaults to main. '-Nd' means 'N days ago on main'." + required: false + type: string + +jobs: + # Hash all of the inputs and generate a descriptive, but unique, name for the bisect job. + # This is necessary because GitHub Actions does not provide an easy way to get the numeric + # job id from within a running job, unless the name is unique across the entire run. + generate-job-name: + name: Generate Unique Job Name + runs-on: ubuntu-latest + outputs: + job-name: ${{ steps.set-name.outputs.unique-name }} + steps: + - name: Set unique name + id: set-name + run: | + # Hash the inputs to create a unique identifier + input_string=$(cat < "begin_d.end" + delim="_" + keep_chars=5 + compressed_str=${input:0:keep_chars}$delim${input: -keep_chars} + compressed_length=${#compressed_str} + + if [[ $input_length -gt $compressed_length ]]; then + echo "$compressed_str" + else + echo "$input" + fi + } + + preset=$(sanitize "${{ inputs.preset }}") + good_ref=$(sanitize "${{ inputs.good_ref }}") + bad_ref=$(sanitize "${{ inputs.bad_ref }}") + build_targets=$(sanitize "${{ inputs.build_targets }}") + ctest_targets=$(sanitize "${{ inputs.ctest_targets }}") + lit_precompile_tests=$(sanitize "${{ inputs.lit_precompile_tests }}") + lit_tests=$(sanitize "${{ inputs.lit_tests }}") + + build_targets_part=$(compress "$build_targets") + ctest_targets_part=$(compress "$ctest_targets") + lit_precompile_tests_part=$(compress "$lit_precompile_tests") + lit_tests_part=$(compress "$lit_tests") + + # Parse "--cuda XX.Y" / "-c XX.Y" and "--host " / "-H " + launch_args="${{ inputs.launch_args }}" + cuda_version="ctk$(grep -oP '(?:--cuda|-c)\s+\K\S+' <<< "$launch_args" || true)" + host_name=$(grep -oP '(?:--host|-H)\s+\K\S+' <<< "$launch_args" || true) + + # Parse `amd64/arm64` and -gpu- from runner name. + # Ex: 'linux-amd64-gpu-rtxa6000-latest-1' + runner=${{ inputs.runner }} + cpu_arch=$(echo "${runner}" | grep -oP '(?:^|-)\K(?:amd64|arm64)(?=-)' || true) + gpu_name=$(echo "${runner}" | grep -oP '(?:-gpu-)\K[^-]+' || true) + + # Vars to include in unique name: + declare -a vars=( + "cuda_version" + "host_name" + "preset" + "build_targets_part" + "ctest_targets_part" + "lit_precompile_tests_part" + "lit_tests_part" + "gpu_name" + "cpu_arch" + "short_hash" + ) + unique_name="bisect" + for var in "${vars[@]}"; do + val=${!var} + if [[ -n "$val" ]]; then + unique_name+="-$val" + fi + done + + echo "Unique job name: $unique_name" + echo "unique-name=$unique_name" >> "$GITHUB_OUTPUT" + + bisect: + needs: [generate-job-name] + name: ${{ needs.generate-job-name.outputs.job-name }} + runs-on: ${{ inputs.runner }} + permissions: + id-token: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 # FULL history + fetch-tags: true # Ensure tags are present + + - name: Get AWS credentials for sccache bucket + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA + aws-region: us-east-2 + role-duration-seconds: 43200 # 12 hours + + - name: Prepare AWS config for devcontainer + run: | + # The devcontainer will mount this path to the home directory + aws_dir="${{ github.workspace }}/.aws" + mkdir -p "${aws_dir}" + cat > "${aws_dir}/config" < "${aws_dir}/credentials" <> "$GITHUB_STEP_SUMMARY" || : + fi + + echo -e "\e[1;34mGHA Log URL: $GHA_LOG_URL\e[0m" + echo -e "\e[1;34mBisection Results: $STEP_SUMMARY_URL\e[0m" + + exit $rc diff --git a/ci/util/build_and_test_targets.sh b/ci/util/build_and_test_targets.sh new file mode 100755 index 00000000000..0a783faaeea --- /dev/null +++ b/ci/util/build_and_test_targets.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This must be run from the cccl repo root, but the script may be relocated by git_bisect.sh. +# Check that the current directory looks like the repo root: +if [[ ! -f "./cccl-version.json" ]]; then + echo "This script must be run from the cccl repo root." + exit 1 +fi + +usage() { + cat <&2; usage; exit 2 ;; + esac +done + +if [[ -z "${PRESET}" && -z "${CONFIGURE_OVERRIDE}" ]]; then + echo "::error:: --preset or --configure-override is required" >&2 + usage + exit 2 +fi + +if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then + if [[ -n "${PRESET}" ]]; then + echo "::warning:: --preset ignored due to --configure-override" >&2 + fi + if [[ -n "${CMAKE_OPTIONS}" ]]; then + echo "::warning:: --cmake-options ignored due to --configure-override" >&2 + fi +fi + +echo "::group::โš™๏ธ Testing $(git log --oneline | head -n1)" + +# Configure and parse the build directory from CMake output +BUILD_DIR="" +cmlog_file="$(mktemp /tmp/cmake-config-XXXXXX.log)" +if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then + if ! (set -x; eval "${CONFIGURE_OVERRIDE}") 2>&1 | tee "${cmlog_file}"; then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ“ Configuration override failed ($(elapsed_time)):\n\t${CONFIGURE_OVERRIDE}" + exit 1 + fi +else + read -r -a _cmake_opts <<< "${CMAKE_OPTIONS}" + if ! (set -x; cmake --preset "${PRESET}" "${_cmake_opts[@]}") 2>&1 | tee "${cmlog_file}"; then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ“ CMake configure failed for preset ${PRESET} ($(elapsed_time))" + exit 1 + fi +fi +BUILD_DIR=$(awk -F': ' '/-- Build files have been written to:/ {print $2}' "${cmlog_file}" | tail -n1) +if [[ -z "${BUILD_DIR}" ]]; then + echo "::endgroup::" + echo "๐Ÿ”ดโ€ผ๏ธ Unable to determine build directory ($(elapsed_time))" + exit 1 +fi + +if [[ -n "${BUILD_TARGETS}" ]]; then + if ! (set -x; ninja -C "${BUILD_DIR}" ${BUILD_TARGETS}); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ› ๏ธ Ninja build failed for targets ($(elapsed_time)): ${BUILD_TARGETS}" + exit 1 + fi +fi + +if [[ -n "${CTEST_TARGETS}" ]]; then + for t in ${CTEST_TARGETS}; do + if ! (set -x; ctest --test-dir "${BUILD_DIR}" -R "$t" -V --output-on-failure); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ”Ž CTest failed for target $t ($(elapsed_time))" + exit 1 + fi + done +fi + +if [[ -n "${LIT_PRECOMPILE_TESTS}" || -n "${LIT_TESTS}" ]]; then + lit_site_cfg="${BUILD_DIR}/libcudacxx/test/libcudacxx/lit.site.cfg" + if [[ ! -f "${lit_site_cfg}" ]]; then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช LIT site config not found ($(elapsed_time)): ${lit_site_cfg}" + exit 1 + fi +fi + +if [[ -n "${LIT_PRECOMPILE_TESTS}" ]]; then + for t in ${LIT_PRECOMPILE_TESTS}; do + t_path="libcudacxx/test/libcudacxx/${t}" + if ! (set -x; LIBCUDACXX_SITE_CONFIG="${lit_site_cfg}" lit -v "-Dexecutor=NoopExecutor()" "${t_path}"); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช LIT precompile failed ($(elapsed_time)): ${t}" + exit 1 + fi + done +fi + +if [[ -n "${LIT_TESTS}" ]]; then + for t in ${LIT_TESTS}; do + t_path="libcudacxx/test/libcudacxx/${t}" + if ! (set -x; LIBCUDACXX_SITE_CONFIG="${lit_site_cfg}" lit -v "${t_path}"); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช LIT test failed ($(elapsed_time)): ${t}" + exit 1 + fi + done +fi + +if [[ -n "${CUSTOM_TEST_CMD}" ]]; then + if ! (set -x; eval "${CUSTOM_TEST_CMD}"); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช Custom test command failed ($(elapsed_time)): ${CUSTOM_TEST_CMD}" + exit 1 + fi +fi + +echo "::endgroup::" +echo "๐ŸŸขโœ… Passed ($(elapsed_time))" +exit 0 diff --git a/ci/util/git_bisect.sh b/ci/util/git_bisect.sh new file mode 100755 index 00000000000..efdd568eaa8 --- /dev/null +++ b/ci/util/git_bisect.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +set -euo pipefail + +ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ci_dir/.." + +usage() { + cat <&2; usage; exit 2 ;; + esac +done + +if [[ -z "${PRESET}" && -z "${CONFIGURE_OVERRIDE}" ]]; then + echo "::error:: --preset or --configure-override is required" >&2 + usage + exit 2 +fi + +if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then + if [[ -n "${PRESET}" ]]; then + echo "::warning:: --preset ignored due to --configure-override" >&2 + fi + if [[ -n "${CMAKE_OPTIONS}" ]]; then + echo "::warning:: --cmake-options ignored due to --configure-override" >&2 + fi +fi + +# Resolve good and bad refs +good_ref="${GOOD_REF}" +bad_ref="${BAD_REF}" + +# Helper to resolve '-Nd' (N days ago on origin/main) to a SHA +_resolve_days_ago() { + local spec="$1" + local base_branch="origin/main" + local n="${spec#-}" + n="${n%d}" + if [[ -z "$n" || ! "$n" =~ ^[0-9]+$ ]]; then + return 1 + fi + local when + when=$(date -u -d "$n days ago" '+%Y-%m-%d %H:%M:%S %z') + git rev-list -n 1 --before="$when" "$base_branch" +} + +# Resolve good_ref +if [[ -z "$good_ref" ]]; then + good_ref=$(git tag --list 'v*' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n 1 || :) + echo "Good ref defaulted to last release: $good_ref" +fi +if [[ "$good_ref" =~ ^-[0-9]+d$ ]]; then + good_sha=$(_resolve_days_ago "$good_ref") + if [[ -z "$good_sha" ]]; then + echo "::error::Unable to resolve good_ref '$good_ref' to a commit on origin/main" >&2 + exit 1 + fi + echo "Resolved good_ref '$good_ref' to origin/main @ $good_sha" +else + if [[ -z "$good_ref" ]]; then + echo "::error::Unable to determine good ref" >&2 + exit 1 + fi + good_sha=$(git rev-parse "$good_ref") +fi + +# Resolve bad_ref +if [[ -z "$bad_ref" ]]; then + bad_ref="origin/main" + echo "Bad ref defaulted to origin/main: $bad_ref" +fi +if [[ "$bad_ref" =~ ^-[0-9]+d$ ]]; then + bad_sha=$(_resolve_days_ago "$bad_ref") + if [[ -z "$bad_sha" ]]; then + echo "::error::Unable to resolve bad_ref '$bad_ref' to a commit on origin/main" >&2 + exit 1 + fi + echo "Resolved bad_ref '$bad_ref' to origin/main @ $bad_sha" +else + bad_sha=$(git rev-parse "$bad_ref") +fi + +# Copy the build-and-test runner to a temp file so it remains available as HEAD changes: +tmp_runner="$(mktemp /tmp/build-and-test-XXXXXX.sh)" +cp "${ci_dir}/util/build_and_test_targets.sh" "${tmp_runner}" +chmod +x "${tmp_runner}" + +# Writer that always prints to stdout and tees to file if provided +write_summary() { + if [[ -n "${SUMMARY_FILE}" ]]; then + tee -a "${SUMMARY_FILE}" + else + cat + fi +} + +echo "Starting bisect with:" +echo " BAD_SHA: $bad_sha" +echo " GOOD_SHA: $good_sha" + +echo "::group::โš™๏ธ Starting git bisect" +(set -x; git bisect start "$bad_sha" "$good_sha") +echo "::endgroup::" + +bisect_log="$(mktemp /tmp/git-bisect-run-XXXXXX.log)" +( + set -x + git bisect run "${tmp_runner}" \ + --preset "${PRESET}" \ + --build-targets "${BUILD_TARGETS}" \ + --ctest-targets "${CTEST_TARGETS}" \ + --lit-precompile-tests "${LIT_PRECOMPILE_TESTS}" \ + --lit-tests "${LIT_TESTS}" \ + --cmake-options "${CMAKE_OPTIONS}" \ + --configure-override "${CONFIGURE_OVERRIDE}" \ + --custom-test-cmd "${CUSTOM_TEST_CMD}" \ + | tee "${bisect_log}" || : + git bisect reset || : +) + +if grep -q " is the first bad commit" "${bisect_log}"; then + bad_commit=$(awk '/ is the first bad commit/ {print $1}' "${bisect_log}") + echo -e "\e[1;32mFound bad commit in $(elapsed_time): $bad_commit\e[0m" + found=true +else + echo -e "\e[1;31mNo bad commit found ($(elapsed_time)).\e[0m" + found=false +fi + +function print_repro { + echo "### โ™ป๏ธ Reproduction Steps" + echo + echo '```bash' + if [[ -n "${LAUNCH_ARGS:-}" ]]; then + echo " .devcontainer/launch.sh \\" + echo " ${LAUNCH_ARGS} \\" + echo " -- \\" + fi + echo " ./ci/util/build_and_test_targets.sh \\" + + declare -a build_test_vars=( + PRESET + CMAKE_OPTIONS + CONFIGURE_OVERRIDE + BUILD_TARGETS + CTEST_TARGETS + LIT_PRECOMPILE_TESTS + LIT_TESTS + CUSTOM_TEST_CMD + ) + + # only print the above vars if they're non-empty. + first=true + for var in "${build_test_vars[@]}"; do + if [[ -n "${!var:-}" ]]; then + flag="--${var,,}" + flag=${flag//_/-} # replace _ with - + if ! $first; then + echo " \\" # Trailing "\" to escape newlines + fi + echo -n " ${flag} \"${!var}\"" + first=false + fi + done + echo # Final newline for last argument + echo '```' +} + +if [[ "${found}" == "true" ]]; then + commit_info=$(git log "$bad_commit" -1 --pretty=format:'%h %s') + pr_ref=$(echo "$commit_info" | grep -oE '#[0-9]+' | head -n1 || :) + ( + echo "## ๐Ÿ”Ž Bisect Result" + echo + echo "- Culprit Commit: $commit_info" + if [[ -n "$pr_ref" ]]; then + pr_num=${pr_ref#\#} + echo "- Culprit PR: https://github.com/NVIDIA/cccl/pull/$pr_num" + fi + echo "- Commit SHA: $bad_commit" + echo "- Commit URL: https://github.com/NVIDIA/cccl/commit/${bad_commit}" + if [[ -n "${GHA_LOG_URL:-}" ]]; then + echo "- Bisection Logs: [GHA Job](${GHA_LOG_URL})" + echo "- Bisection Summary: [GHA Report](${STEP_SUMMARY_URL})" + fi + echo + print_repro + echo + echo "### โ„น๏ธ Commit Details" + echo + echo '```' + git show "$bad_commit" --stat + echo '```' + + ) | write_summary +else + ( + echo "## โ€ผ๏ธ Bisect Failed" + echo + echo "git bisect did not resolve to a single commit." + echo + if [[ -n "${GHA_LOG_URL:-}" ]]; then + echo "- Bisection Logs: [GHA Job](${GHA_LOG_URL})" + echo "- Bisection Summary: [GHA Report](${STEP_SUMMARY_URL})" + fi + echo + print_repro + ) | write_summary + exit 1 +fi diff --git a/docs/cccl/development/build_and_bisect_tools.rst b/docs/cccl/development/build_and_bisect_tools.rst new file mode 100644 index 00000000000..3d5f31e6b27 --- /dev/null +++ b/docs/cccl/development/build_and_bisect_tools.rst @@ -0,0 +1,136 @@ +.. _build-and-bisect-tools: + +Build and Bisect Utilities +========================== + +``build_and_test_targets.sh`` +----------------------------- + +:file:`ci/util/build_and_test_targets.sh` configures, builds, and tests selected +CMake targets. + +Options +~~~~~~~ +- ``--preset `` - choose a CMake preset. +- ``--cmake-options `` - extra arguments for the preset configuration. +- ``--configure-override `` - run a custom configuration command instead of +a preset. When used, ``--preset`` and ``--cmake-options`` are ignored. +- ``--build-targets `` - space separated Ninja targets. If omitted, + nothing builds. +- ``--ctest-targets `` - space separated CTest ``-R`` patterns. If + omitted, nothing runs. +- ``--lit-precompile-tests `` - space separated libcudacxx lit test paths + to precompile (no run). Paths are relative to ``libcudacxx/test/libcudacxx/``. +- ``--lit-tests `` - space separated libcudacxx lit test paths to execute. + Paths are relative to ``libcudacxx/test/libcudacxx/``. +- ``--custom-test-cmd `` - arbitrary command executed after build/tests. + +Combine with ``.devcontainer/launch.sh -d`` to reproduce CI commands inside a +container and choose a CUDA toolkit and host compiler: +``.devcontainer/launch.sh -d [--cuda ] [--host ] [--gpus all] --