diff --git a/.devcontainer/launch.sh b/.devcontainer/launch.sh index 865bf7e2410..b70d9692dc2 100755 --- a/.devcontainer/launch.sh +++ b/.devcontainer/launch.sh @@ -134,6 +134,9 @@ launch_docker() { # Update run arguments and container environment variables ### + # Always clean up docker containers run via this script. + RUN_ARGS+=("--rm") + # Only pass `-it` if the shell is a tty if ! ${CI:-'false'} && tty >/dev/null 2>&1 && (exec "begin_d.end" + delim="_" + keep_chars=5 + compressed_str=${input:0:keep_chars}$delim${input: -keep_chars} + compressed_length=${#compressed_str} + + if [[ $input_length -gt $compressed_length ]]; then + echo "$compressed_str" + else + echo "$input" + fi + } + + preset=$(sanitize "${{ inputs.preset }}") + good_ref=$(sanitize "${{ inputs.good_ref }}") + bad_ref=$(sanitize "${{ inputs.bad_ref }}") + build_targets=$(sanitize "${{ inputs.build_targets }}") + ctest_targets=$(sanitize "${{ inputs.ctest_targets }}") + lit_precompile_tests=$(sanitize "${{ inputs.lit_precompile_tests }}") + lit_tests=$(sanitize "${{ inputs.lit_tests }}") + + build_targets_part=$(compress "$build_targets") + ctest_targets_part=$(compress "$ctest_targets") + lit_precompile_tests_part=$(compress "$lit_precompile_tests") + lit_tests_part=$(compress "$lit_tests") + + # Parse "--cuda XX.Y" / "-c XX.Y" and "--host " / "-H " + launch_args="${{ inputs.launch_args }}" + cuda_version="ctk$(grep -oP '(?:--cuda|-c)\s+\K\S+' <<< "$launch_args" || true)" + host_name=$(grep -oP '(?:--host|-H)\s+\K\S+' <<< "$launch_args" || true) + + # Parse `amd64/arm64` and -gpu- from runner name. + # Ex: 'linux-amd64-gpu-rtxa6000-latest-1' + runner=${{ inputs.runner }} + cpu_arch=$(echo "${runner}" | grep -oP '(?:^|-)\K(?:amd64|arm64)(?=-)' || true) + gpu_name=$(echo "${runner}" | grep -oP '(?:-gpu-)\K[^-]+' || true) + + # Vars to include in unique name: + declare -a vars=( + "cuda_version" + "host_name" + "preset" + "build_targets_part" + "ctest_targets_part" + "lit_precompile_tests_part" + "lit_tests_part" + "gpu_name" + "cpu_arch" + "short_hash" + ) + unique_name="bisect" + for var in "${vars[@]}"; do + val=${!var} + if [[ -n "$val" ]]; then + unique_name+="-$val" + fi + done + + echo "Unique job name: $unique_name" + echo "unique-name=$unique_name" >> "$GITHUB_OUTPUT" + + bisect: + needs: [generate-job-name] + name: ${{ needs.generate-job-name.outputs.job-name }} + runs-on: ${{ github.repository == 'NVIDIA/cccl' && inputs.runner || 'ubuntu-latest' }} + permissions: + id-token: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 # FULL history + fetch-tags: true # Ensure tags are present + + - name: Get AWS credentials for sccache bucket + if: ${{ github.repository == 'NVIDIA/cccl' }} + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA + aws-region: us-east-2 + role-duration-seconds: 43200 # 12 hours + + - name: Prepare AWS config for devcontainer + run: | + # The devcontainer will mount this path to the home directory + aws_dir="${{ github.workspace }}/.aws" + mkdir -p "${aws_dir}" + cat > "${aws_dir}/config" < "${aws_dir}/credentials" <> "$GITHUB_STEP_SUMMARY" || : + fi + + echo -e "\e[1;34mGHA Log URL: $GHA_LOG_URL\e[0m" + echo -e "\e[1;34mBisection Results: $STEP_SUMMARY_URL\e[0m" + + exit $rc diff --git a/ci/util/build_and_test_targets.sh b/ci/util/build_and_test_targets.sh new file mode 100755 index 00000000000..0a783faaeea --- /dev/null +++ b/ci/util/build_and_test_targets.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This must be run from the cccl repo root, but the script may be relocated by git_bisect.sh. +# Check that the current directory looks like the repo root: +if [[ ! -f "./cccl-version.json" ]]; then + echo "This script must be run from the cccl repo root." + exit 1 +fi + +usage() { + cat <&2; usage; exit 2 ;; + esac +done + +if [[ -z "${PRESET}" && -z "${CONFIGURE_OVERRIDE}" ]]; then + echo "::error:: --preset or --configure-override is required" >&2 + usage + exit 2 +fi + +if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then + if [[ -n "${PRESET}" ]]; then + echo "::warning:: --preset ignored due to --configure-override" >&2 + fi + if [[ -n "${CMAKE_OPTIONS}" ]]; then + echo "::warning:: --cmake-options ignored due to --configure-override" >&2 + fi +fi + +echo "::group::โš™๏ธ Testing $(git log --oneline | head -n1)" + +# Configure and parse the build directory from CMake output +BUILD_DIR="" +cmlog_file="$(mktemp /tmp/cmake-config-XXXXXX.log)" +if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then + if ! (set -x; eval "${CONFIGURE_OVERRIDE}") 2>&1 | tee "${cmlog_file}"; then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ“ Configuration override failed ($(elapsed_time)):\n\t${CONFIGURE_OVERRIDE}" + exit 1 + fi +else + read -r -a _cmake_opts <<< "${CMAKE_OPTIONS}" + if ! (set -x; cmake --preset "${PRESET}" "${_cmake_opts[@]}") 2>&1 | tee "${cmlog_file}"; then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ“ CMake configure failed for preset ${PRESET} ($(elapsed_time))" + exit 1 + fi +fi +BUILD_DIR=$(awk -F': ' '/-- Build files have been written to:/ {print $2}' "${cmlog_file}" | tail -n1) +if [[ -z "${BUILD_DIR}" ]]; then + echo "::endgroup::" + echo "๐Ÿ”ดโ€ผ๏ธ Unable to determine build directory ($(elapsed_time))" + exit 1 +fi + +if [[ -n "${BUILD_TARGETS}" ]]; then + if ! (set -x; ninja -C "${BUILD_DIR}" ${BUILD_TARGETS}); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ› ๏ธ Ninja build failed for targets ($(elapsed_time)): ${BUILD_TARGETS}" + exit 1 + fi +fi + +if [[ -n "${CTEST_TARGETS}" ]]; then + for t in ${CTEST_TARGETS}; do + if ! (set -x; ctest --test-dir "${BUILD_DIR}" -R "$t" -V --output-on-failure); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿ”Ž CTest failed for target $t ($(elapsed_time))" + exit 1 + fi + done +fi + +if [[ -n "${LIT_PRECOMPILE_TESTS}" || -n "${LIT_TESTS}" ]]; then + lit_site_cfg="${BUILD_DIR}/libcudacxx/test/libcudacxx/lit.site.cfg" + if [[ ! -f "${lit_site_cfg}" ]]; then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช LIT site config not found ($(elapsed_time)): ${lit_site_cfg}" + exit 1 + fi +fi + +if [[ -n "${LIT_PRECOMPILE_TESTS}" ]]; then + for t in ${LIT_PRECOMPILE_TESTS}; do + t_path="libcudacxx/test/libcudacxx/${t}" + if ! (set -x; LIBCUDACXX_SITE_CONFIG="${lit_site_cfg}" lit -v "-Dexecutor=NoopExecutor()" "${t_path}"); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช LIT precompile failed ($(elapsed_time)): ${t}" + exit 1 + fi + done +fi + +if [[ -n "${LIT_TESTS}" ]]; then + for t in ${LIT_TESTS}; do + t_path="libcudacxx/test/libcudacxx/${t}" + if ! (set -x; LIBCUDACXX_SITE_CONFIG="${lit_site_cfg}" lit -v "${t_path}"); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช LIT test failed ($(elapsed_time)): ${t}" + exit 1 + fi + done +fi + +if [[ -n "${CUSTOM_TEST_CMD}" ]]; then + if ! (set -x; eval "${CUSTOM_TEST_CMD}"); then + echo "::endgroup::" + echo "๐Ÿ”ด๐Ÿงช Custom test command failed ($(elapsed_time)): ${CUSTOM_TEST_CMD}" + exit 1 + fi +fi + +echo "::endgroup::" +echo "๐ŸŸขโœ… Passed ($(elapsed_time))" +exit 0 diff --git a/ci/util/git_bisect.sh b/ci/util/git_bisect.sh new file mode 100755 index 00000000000..efdd568eaa8 --- /dev/null +++ b/ci/util/git_bisect.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +set -euo pipefail + +ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ci_dir/.." + +usage() { + cat <&2; usage; exit 2 ;; + esac +done + +if [[ -z "${PRESET}" && -z "${CONFIGURE_OVERRIDE}" ]]; then + echo "::error:: --preset or --configure-override is required" >&2 + usage + exit 2 +fi + +if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then + if [[ -n "${PRESET}" ]]; then + echo "::warning:: --preset ignored due to --configure-override" >&2 + fi + if [[ -n "${CMAKE_OPTIONS}" ]]; then + echo "::warning:: --cmake-options ignored due to --configure-override" >&2 + fi +fi + +# Resolve good and bad refs +good_ref="${GOOD_REF}" +bad_ref="${BAD_REF}" + +# Helper to resolve '-Nd' (N days ago on origin/main) to a SHA +_resolve_days_ago() { + local spec="$1" + local base_branch="origin/main" + local n="${spec#-}" + n="${n%d}" + if [[ -z "$n" || ! "$n" =~ ^[0-9]+$ ]]; then + return 1 + fi + local when + when=$(date -u -d "$n days ago" '+%Y-%m-%d %H:%M:%S %z') + git rev-list -n 1 --before="$when" "$base_branch" +} + +# Resolve good_ref +if [[ -z "$good_ref" ]]; then + good_ref=$(git tag --list 'v*' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n 1 || :) + echo "Good ref defaulted to last release: $good_ref" +fi +if [[ "$good_ref" =~ ^-[0-9]+d$ ]]; then + good_sha=$(_resolve_days_ago "$good_ref") + if [[ -z "$good_sha" ]]; then + echo "::error::Unable to resolve good_ref '$good_ref' to a commit on origin/main" >&2 + exit 1 + fi + echo "Resolved good_ref '$good_ref' to origin/main @ $good_sha" +else + if [[ -z "$good_ref" ]]; then + echo "::error::Unable to determine good ref" >&2 + exit 1 + fi + good_sha=$(git rev-parse "$good_ref") +fi + +# Resolve bad_ref +if [[ -z "$bad_ref" ]]; then + bad_ref="origin/main" + echo "Bad ref defaulted to origin/main: $bad_ref" +fi +if [[ "$bad_ref" =~ ^-[0-9]+d$ ]]; then + bad_sha=$(_resolve_days_ago "$bad_ref") + if [[ -z "$bad_sha" ]]; then + echo "::error::Unable to resolve bad_ref '$bad_ref' to a commit on origin/main" >&2 + exit 1 + fi + echo "Resolved bad_ref '$bad_ref' to origin/main @ $bad_sha" +else + bad_sha=$(git rev-parse "$bad_ref") +fi + +# Copy the build-and-test runner to a temp file so it remains available as HEAD changes: +tmp_runner="$(mktemp /tmp/build-and-test-XXXXXX.sh)" +cp "${ci_dir}/util/build_and_test_targets.sh" "${tmp_runner}" +chmod +x "${tmp_runner}" + +# Writer that always prints to stdout and tees to file if provided +write_summary() { + if [[ -n "${SUMMARY_FILE}" ]]; then + tee -a "${SUMMARY_FILE}" + else + cat + fi +} + +echo "Starting bisect with:" +echo " BAD_SHA: $bad_sha" +echo " GOOD_SHA: $good_sha" + +echo "::group::โš™๏ธ Starting git bisect" +(set -x; git bisect start "$bad_sha" "$good_sha") +echo "::endgroup::" + +bisect_log="$(mktemp /tmp/git-bisect-run-XXXXXX.log)" +( + set -x + git bisect run "${tmp_runner}" \ + --preset "${PRESET}" \ + --build-targets "${BUILD_TARGETS}" \ + --ctest-targets "${CTEST_TARGETS}" \ + --lit-precompile-tests "${LIT_PRECOMPILE_TESTS}" \ + --lit-tests "${LIT_TESTS}" \ + --cmake-options "${CMAKE_OPTIONS}" \ + --configure-override "${CONFIGURE_OVERRIDE}" \ + --custom-test-cmd "${CUSTOM_TEST_CMD}" \ + | tee "${bisect_log}" || : + git bisect reset || : +) + +if grep -q " is the first bad commit" "${bisect_log}"; then + bad_commit=$(awk '/ is the first bad commit/ {print $1}' "${bisect_log}") + echo -e "\e[1;32mFound bad commit in $(elapsed_time): $bad_commit\e[0m" + found=true +else + echo -e "\e[1;31mNo bad commit found ($(elapsed_time)).\e[0m" + found=false +fi + +function print_repro { + echo "### โ™ป๏ธ Reproduction Steps" + echo + echo '```bash' + if [[ -n "${LAUNCH_ARGS:-}" ]]; then + echo " .devcontainer/launch.sh \\" + echo " ${LAUNCH_ARGS} \\" + echo " -- \\" + fi + echo " ./ci/util/build_and_test_targets.sh \\" + + declare -a build_test_vars=( + PRESET + CMAKE_OPTIONS + CONFIGURE_OVERRIDE + BUILD_TARGETS + CTEST_TARGETS + LIT_PRECOMPILE_TESTS + LIT_TESTS + CUSTOM_TEST_CMD + ) + + # only print the above vars if they're non-empty. + first=true + for var in "${build_test_vars[@]}"; do + if [[ -n "${!var:-}" ]]; then + flag="--${var,,}" + flag=${flag//_/-} # replace _ with - + if ! $first; then + echo " \\" # Trailing "\" to escape newlines + fi + echo -n " ${flag} \"${!var}\"" + first=false + fi + done + echo # Final newline for last argument + echo '```' +} + +if [[ "${found}" == "true" ]]; then + commit_info=$(git log "$bad_commit" -1 --pretty=format:'%h %s') + pr_ref=$(echo "$commit_info" | grep -oE '#[0-9]+' | head -n1 || :) + ( + echo "## ๐Ÿ”Ž Bisect Result" + echo + echo "- Culprit Commit: $commit_info" + if [[ -n "$pr_ref" ]]; then + pr_num=${pr_ref#\#} + echo "- Culprit PR: https://github.com/NVIDIA/cccl/pull/$pr_num" + fi + echo "- Commit SHA: $bad_commit" + echo "- Commit URL: https://github.com/NVIDIA/cccl/commit/${bad_commit}" + if [[ -n "${GHA_LOG_URL:-}" ]]; then + echo "- Bisection Logs: [GHA Job](${GHA_LOG_URL})" + echo "- Bisection Summary: [GHA Report](${STEP_SUMMARY_URL})" + fi + echo + print_repro + echo + echo "### โ„น๏ธ Commit Details" + echo + echo '```' + git show "$bad_commit" --stat + echo '```' + + ) | write_summary +else + ( + echo "## โ€ผ๏ธ Bisect Failed" + echo + echo "git bisect did not resolve to a single commit." + echo + if [[ -n "${GHA_LOG_URL:-}" ]]; then + echo "- Bisection Logs: [GHA Job](${GHA_LOG_URL})" + echo "- Bisection Summary: [GHA Report](${STEP_SUMMARY_URL})" + fi + echo + print_repro + ) | write_summary + exit 1 +fi diff --git a/docs/cccl/development/build_and_bisect_tools.rst b/docs/cccl/development/build_and_bisect_tools.rst new file mode 100644 index 00000000000..3d5f31e6b27 --- /dev/null +++ b/docs/cccl/development/build_and_bisect_tools.rst @@ -0,0 +1,136 @@ +.. _build-and-bisect-tools: + +Build and Bisect Utilities +========================== + +``build_and_test_targets.sh`` +----------------------------- + +:file:`ci/util/build_and_test_targets.sh` configures, builds, and tests selected +CMake targets. + +Options +~~~~~~~ +- ``--preset `` - choose a CMake preset. +- ``--cmake-options `` - extra arguments for the preset configuration. +- ``--configure-override `` - run a custom configuration command instead of +a preset. When used, ``--preset`` and ``--cmake-options`` are ignored. +- ``--build-targets `` - space separated Ninja targets. If omitted, + nothing builds. +- ``--ctest-targets `` - space separated CTest ``-R`` patterns. If + omitted, nothing runs. +- ``--lit-precompile-tests `` - space separated libcudacxx lit test paths + to precompile (no run). Paths are relative to ``libcudacxx/test/libcudacxx/``. +- ``--lit-tests `` - space separated libcudacxx lit test paths to execute. + Paths are relative to ``libcudacxx/test/libcudacxx/``. +- ``--custom-test-cmd `` - arbitrary command executed after build/tests. + +Combine with ``.devcontainer/launch.sh -d`` to reproduce CI commands inside a +container and choose a CUDA toolkit and host compiler: +``.devcontainer/launch.sh -d [--cuda ] [--host ] [--gpus all] --