From 333e4b438f4b4c3129b381c2e1956d9f992c8fdb Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 10 Dec 2025 17:29:12 -0600 Subject: [PATCH 1/5] Use CCCL from cudf --- cpp/velox/CMakeLists.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 9547cde0e254..3fbc7de99c79 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -417,6 +417,11 @@ if(BUILD_EXAMPLES) endif() if(ENABLE_GPU) + # Find cudf to get CCCL and other dependencies from its fetched dependencies + # This ensures we use CCCL from cudf's build rather than /usr/local/cuda/include + set(cudf_DIR "${VELOX_BUILD_PATH}/_deps/cudf-build") + find_package(cudf REQUIRED CONFIG) + import_library( facebook::velox::velox_cudf_expression ${VELOX_BUILD_PATH}/velox/experimental/cudf/expression/libvelox_cudf_expression.a @@ -438,9 +443,7 @@ if(ENABLE_GPU) ${VELOX_BUILD_PATH}/_deps/kvikio-src/cpp/include ${VELOX_BUILD_PATH}/_deps/nvtx3-src/c/include ${VELOX_BUILD_PATH}/_deps/nvcomp_proprietary_binary-src/include - ${VELOX_BUILD_PATH}/_deps/rapids_logger-src/include - /usr/local/cuda/include/cccl - /usr/local/cuda/include) + ${VELOX_BUILD_PATH}/_deps/rapids_logger-src/include) target_compile_definitions( velox PRIVATE LIBCUDACXX_ENABLE_EXPERIMENTAL_MEMORY_RESOURCE) @@ -450,6 +453,8 @@ if(ENABLE_GPU) PUBLIC facebook::velox::velox_cudf_exec facebook::velox::velox_cudf_vector facebook::velox::velox_cudf_hive_connector facebook::velox::velox_cudf_expression) + # Link CCCL from cudf's dependencies (provides thrust, cub, libcudacxx headers) + target_link_libraries(velox PRIVATE CCCL::CCCL) target_link_libraries( velox PRIVATE From 6842b18aa8c6e1025f955b8e0387fd6b7f666695 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 11 Dec 2025 08:07:20 +0000 Subject: [PATCH 2/5] Fix CMakeLists to find_package(cudf) --- cpp/velox/CMakeLists.txt | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 3fbc7de99c79..b8c3896216ba 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -436,14 +436,6 @@ if(ENABLE_GPU) facebook::velox::velox_cudf_hive_connector ${VELOX_BUILD_PATH}/velox/experimental/cudf/connectors/hive/libvelox_cudf_hive_connector.a ) - target_include_directories( - velox - PRIVATE ${VELOX_BUILD_PATH}/_deps/cudf-src/cpp/include - ${VELOX_BUILD_PATH}/_deps/rmm-src/cpp/include - ${VELOX_BUILD_PATH}/_deps/kvikio-src/cpp/include - ${VELOX_BUILD_PATH}/_deps/nvtx3-src/c/include - ${VELOX_BUILD_PATH}/_deps/nvcomp_proprietary_binary-src/include - ${VELOX_BUILD_PATH}/_deps/rapids_logger-src/include) target_compile_definitions( velox PRIVATE LIBCUDACXX_ENABLE_EXPERIMENTAL_MEMORY_RESOURCE) @@ -453,18 +445,9 @@ if(ENABLE_GPU) PUBLIC facebook::velox::velox_cudf_exec facebook::velox::velox_cudf_vector facebook::velox::velox_cudf_hive_connector facebook::velox::velox_cudf_expression) - # Link CCCL from cudf's dependencies (provides thrust, cub, libcudacxx headers) - target_link_libraries(velox PRIVATE CCCL::CCCL) - target_link_libraries( - velox - PRIVATE - ${VELOX_BUILD_PATH}/_deps/cudf-build/libcudf.so - ${VELOX_BUILD_PATH}/_deps/rmm-build/librmm.so - ${VELOX_BUILD_PATH}/_deps/kvikio-build/libkvikio.so - ${VELOX_BUILD_PATH}/_deps/nvcomp_proprietary_binary-src/lib64/libnvcomp.so - ${VELOX_BUILD_PATH}/_deps/nvcomp_proprietary_binary-src/lib64/libnvcomp_cpu.so - ${VELOX_BUILD_PATH}/_deps/rapids_logger-build/librapids_logger.so - /usr/local/cuda/lib64/libcudart.so) + # Link cuDF and its dependencies (provides cudf, rmm, kvikio, nvcomp, rapids_logger) + # CCCL provides thrust, cub, libcudacxx headers + target_link_libraries(velox PRIVATE cudf::cudf CCCL::CCCL) endif() add_custom_command( From 0518453b998ae6b0716ed19fd1dc06c557bfb63b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 11 Dec 2025 08:07:31 +0000 Subject: [PATCH 3/5] Remove workarounds to enabled GCC 14 --- .github/workflows/velox_backend_cache.yml | 5 +++-- .github/workflows/velox_backend_x86.yml | 10 +++++----- dev/ci-velox-buildstatic-centos-9.sh | 2 +- dev/docker/cudf/Dockerfile | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/velox_backend_cache.yml b/.github/workflows/velox_backend_cache.yml index 8d0a3da0c91e..77b331b5277c 100644 --- a/.github/workflows/velox_backend_cache.yml +++ b/.github/workflows/velox_backend_cache.yml @@ -167,14 +167,15 @@ jobs: dnf autoremove -y && dnf clean all dnf remove -y cuda-toolkit-12* && dnf install -y cuda-toolkit-13-1 ls -l /usr/local/ - source /opt/rh/gcc-toolset-12/enable + + source /opt/rh/gcc-toolset-14/enable export CMAKE_BUILD_PARALLEL_LEVEL=4 export NUM_THREADS=4 export CCACHE_MAXSIZE=1G export CCACHE_DIR=/work/.ccache mkdir -p /work/.ccache - + cd /work bash dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON --build_benchmarks=ON --enable_gpu=ON # TODO: re-enable tests with more disk space rm -rf ep/build-velox/build/velox_ep diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 12bc7acabe9a..ee1748f1acbb 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -301,7 +301,7 @@ jobs: --extra-conf=spark.gluten.ras.enabled=true \ && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --extra-conf=spark.gluten.ras.enabled=true + --extra-conf=spark.gluten.ras.enabled=true tpc-test-centos7: needs: build-native-lib-centos-7 @@ -356,7 +356,7 @@ jobs: --extra-conf=spark.gluten.ras.enabled=true \ && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --extra-conf=spark.gluten.ras.enabled=true + --extra-conf=spark.gluten.ras.enabled=true " tpc-test-ubuntu-oom: @@ -399,7 +399,7 @@ jobs: echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | - cd $GITHUB_WORKSPACE/ + cd $GITHUB_WORKSPACE/ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn clean install -P${{ matrix.spark }} @@ -514,7 +514,7 @@ jobs: echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | - cd $GITHUB_WORKSPACE/ + cd $GITHUB_WORKSPACE/ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/build/mvn clean install -P${{ matrix.spark }} @@ -1376,7 +1376,7 @@ jobs: dnf autoremove -y && dnf clean all dnf remove -y cuda-toolkit-12* && dnf install -y cuda-toolkit-13-1 ls -l /usr/local/ - source /opt/rh/gcc-toolset-12/enable + source /opt/rh/gcc-toolset-14/enable export CMAKE_BUILD_PARALLEL_LEVEL=4 export NUM_THREADS=4 diff --git a/dev/ci-velox-buildstatic-centos-9.sh b/dev/ci-velox-buildstatic-centos-9.sh index fd99283ebd21..317238b60ffa 100644 --- a/dev/ci-velox-buildstatic-centos-9.sh +++ b/dev/ci-velox-buildstatic-centos-9.sh @@ -17,7 +17,7 @@ set -e -source /opt/rh/gcc-toolset-12/enable +source /opt/rh/gcc-toolset-14/enable if [ "$(uname -m)" = "aarch64" ]; then export CPU_TARGET="aarch64"; export VCPKG_FORCE_SYSTEM_BINARIES=1; diff --git a/dev/docker/cudf/Dockerfile b/dev/docker/cudf/Dockerfile index c900c632ea1f..9569ee609ab7 100644 --- a/dev/docker/cudf/Dockerfile +++ b/dev/docker/cudf/Dockerfile @@ -31,7 +31,7 @@ WORKDIR /opt/gluten RUN rm -rf /opt/rh/gcc-toolset-12 && ln -s /opt/rh/gcc-toolset-14 /opt/rh/gcc-toolset-12; \ dnf remove -y cuda-toolkit-12* && dnf install -y cuda-toolkit-13-1; \ dnf autoremove -y && dnf clean all; \ - source /opt/rh/gcc-toolset-12/enable; \ + source /opt/rh/gcc-toolset-14/enable; \ bash ./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=ON --spark_version=3.4 --build_tests=ON --build_benchmarks=ON --enable_gpu=ON && rm -rf /opt/gluten # You can try the data in folder backends-velox/src/test/resources/tpch-data-parquet From 1f638a9f3394e90c72b975f9677d52fbbc8f6dee Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 11 Dec 2025 08:37:12 +0000 Subject: [PATCH 4/5] Fix CMake --- cpp/velox/CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index b8c3896216ba..9507cf383dcc 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -417,8 +417,7 @@ if(BUILD_EXAMPLES) endif() if(ENABLE_GPU) - # Find cudf to get CCCL and other dependencies from its fetched dependencies - # This ensures we use CCCL from cudf's build rather than /usr/local/cuda/include + # Find cuDF package which provides cudf::cudf target with all transitive dependencies set(cudf_DIR "${VELOX_BUILD_PATH}/_deps/cudf-build") find_package(cudf REQUIRED CONFIG) @@ -445,9 +444,9 @@ if(ENABLE_GPU) PUBLIC facebook::velox::velox_cudf_exec facebook::velox::velox_cudf_vector facebook::velox::velox_cudf_hive_connector facebook::velox::velox_cudf_expression) - # Link cuDF and its dependencies (provides cudf, rmm, kvikio, nvcomp, rapids_logger) - # CCCL provides thrust, cub, libcudacxx headers - target_link_libraries(velox PRIVATE cudf::cudf CCCL::CCCL) + # Link cuDF which transitively provides all dependencies + # (rmm, kvikio, nvcomp, rapids_logger, CCCL for CUDA headers) + target_link_libraries(velox PRIVATE cudf::cudf) endif() add_custom_command( From 14b44284acc5c8c357c2947299c2da0d6af91f77 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 11 Dec 2025 08:37:35 +0000 Subject: [PATCH 5/5] Use CC/CXX if set --- ep/build-velox/src/build-velox.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ep/build-velox/src/build-velox.sh b/ep/build-velox/src/build-velox.sh index ac62f8fc27f3..87eabbd1b41c 100755 --- a/ep/build-velox/src/build-velox.sh +++ b/ep/build-velox/src/build-velox.sh @@ -104,7 +104,16 @@ function compile { CXX_FLAGS='-Wno-error=stringop-overflow -Wno-error=cpp -Wno-missing-field-initializers \ -Wno-error=uninitialized -Wno-unknown-warning-option' - COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=OFF \ + # Explicitly set compilers for CMake if CC and CXX are set + COMPILER_OPTS="" + if [ -n "${CC:-}" ]; then + COMPILER_OPTS="$COMPILER_OPTS -DCMAKE_C_COMPILER=$CC" + fi + if [ -n "${CXX:-}" ]; then + COMPILER_OPTS="$COMPILER_OPTS -DCMAKE_CXX_COMPILER=$CXX" + fi + + COMPILE_OPTION="$COMPILER_OPTS -DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=OFF \ -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF -DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON \ -DVELOX_ENABLE_GEO=ON" if [ $BUILD_TEST_UTILS == "ON" ]; then