From 58b60d18d47eb41066e69b4f783eb2c4bf3a4016 Mon Sep 17 00:00:00 2001 From: glutenperfbot Date: Tue, 7 Oct 2025 12:17:00 +0000 Subject: [PATCH 1/4] [GLUTEN-6887][VL] Daily Update Velox Version (2025_10_07) Upstream Velox's New Commits: 80be1866a by Masha Basmanova, feat: Add Variant::toJson(Type) API that doesn't require shared_ptr (#15070) 5c1dac468 by Xiaoxuan Meng, fix: Fix the test task cancellation condition (#15065) aa406cbd6 by Deepak Majeti, feat(cudf): Remove gflags in favor of CudfConfig (#14963) 78159fa46 by Xiaoxuan Meng, refactor: Remove legacy task spill code after update prestissimo (#15053) a31539e70 by Pedro Eugenio Rocha Pedreira, docs: Add new component and maintainer for LibcuDF bindings (#15057) 5a31ae324 by lingbin, fix(ssdcache): Write rate calculation (#15028) 207c1b679 by Xiaoxuan Meng, fix: Fix sxtream failed test by supporting task cancellation in test (#15052) Signed-off-by: glutenperfbot --- ep/build-velox/src/get_velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 019c38fd7697..6553623c4dc2 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -18,11 +18,11 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2025_10_06 +VELOX_BRANCH=2025_10_07 VELOX_HOME="" RUN_SETUP_SCRIPT=ON VELOX_ENHANCED_REPO=https://github.com/IBM/velox.git -VELOX_ENHANCED_BRANCH=ibm-2025_10_06 +VELOX_ENHANCED_BRANCH=ibm-2025_10_07 ENABLE_ENHANCED_FEATURES=OFF # Developer use only for testing Velox PR. From 11fc27bd19aaee47d83dc3f998c4d0cc37672e07 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 8 Oct 2025 10:13:56 +0800 Subject: [PATCH 2/4] fix --- cpp/core/config/GlutenConfig.h | 4 ++-- cpp/velox/compute/VeloxBackend.cc | 13 ++++++++----- cpp/velox/config/VeloxConfig.h | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index 2f5992b9c0df..110c741a4b4b 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -95,9 +95,9 @@ const std::string kSparkJsonIgnoreNullFields = "spark.sql.jsonGenerator.ignoreNu // cudf const std::string kCudfEnabled = "spark.gluten.sql.columnar.cudf"; -const bool kCudfEnabledDefault = "true"; +constexpr bool kCudfEnabledDefault = true; const std::string kDebugCudf = "spark.gluten.sql.debug.cudf"; -const bool kDebugCudfDefault = "false"; +const std::string kDebugCudfDefault = "false"; std::unordered_map parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength); diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index 54cb08bf572a..d366a7a18b88 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -166,11 +166,14 @@ void VeloxBackend::init( #ifdef GLUTEN_ENABLE_GPU if (backendConf_->get(kCudfEnabled, kCudfEnabledDefault)) { - FLAGS_velox_cudf_debug = backendConf_->get(kDebugCudf, kDebugCudfDefault); - FLAGS_velox_cudf_memory_resource = backendConf_->get(kCudfMemoryResource, kCudfMemoryResourceDefault); - auto& options = velox::cudf_velox::CudfOptions::getInstance(); - options.memoryPercent = backendConf_->get(kCudfMemoryPercent, kCudfMemoryPercentDefault); - velox::cudf_velox::registerCudf(options); + std::unordered_map options = { + {CudfConfig::kCudfEnabled, "true"}, + {CudfConfig::kCudfDebugEnabled, backendConf_->get(kDebugCudf, kDebugCudfDefault)}, + {CudfConfig::kCudfMemoryResource, backendConf_->get(kCudfMemoryResource, kCudfMemoryResourceDefault)}, + {CudfConfig::kCudfMemoryPercent, backendConf_->get(kCudfMemoryPercent, kCudfMemoryPercentDefault)}}; + auto& cudfConfig = velox::cudf_velox::CudfConfig::getInstance(); + cudfConfig.initialize(std::move(options)); + velox::cudf_velox::registerCudf(); } #endif diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h index 690fbd59f15e..4406887978b4 100644 --- a/cpp/velox/config/VeloxConfig.h +++ b/cpp/velox/config/VeloxConfig.h @@ -179,7 +179,7 @@ const std::string kCudfMemoryResourceDefault = // Initial percent of GPU memory to allocate for memory resource for one thread const std::string kCudfMemoryPercent = "spark.gluten.sql.columnar.backend.velox.cudf.memoryPercent"; -const int32_t kCudfMemoryPercentDefault = 50; +const std::string kCudfMemoryPercentDefault = "50"; /// Preferred size of batches in bytes to be returned by operators. const std::string kVeloxPreferredBatchBytes = "spark.gluten.sql.columnar.backend.velox.preferredBatchBytes"; From 30b62f34975050798376f1d03d115dfb93da8eb2 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 8 Oct 2025 12:28:54 +0800 Subject: [PATCH 3/4] fix up --- cpp/velox/compute/VeloxBackend.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index d366a7a18b88..4e5e0c438cfc 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -28,6 +28,7 @@ #include "utils/qat/QatCodec.h" #endif #ifdef GLUTEN_ENABLE_GPU +#include "velox/experimental/cudf/CudfConfig.h" #include "velox/experimental/cudf/connectors/hive/CudfHiveConnector.h" #include "velox/experimental/cudf/exec/ToCudf.h" #endif From 24a0d9e56cad78c16ea4a8bdc7f2cbd53dc60665 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 8 Oct 2025 13:58:25 +0800 Subject: [PATCH 4/4] fix up --- cpp/velox/compute/VeloxBackend.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index 4e5e0c438cfc..fedb5aa1712c 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -168,10 +168,12 @@ void VeloxBackend::init( #ifdef GLUTEN_ENABLE_GPU if (backendConf_->get(kCudfEnabled, kCudfEnabledDefault)) { std::unordered_map options = { - {CudfConfig::kCudfEnabled, "true"}, - {CudfConfig::kCudfDebugEnabled, backendConf_->get(kDebugCudf, kDebugCudfDefault)}, - {CudfConfig::kCudfMemoryResource, backendConf_->get(kCudfMemoryResource, kCudfMemoryResourceDefault)}, - {CudfConfig::kCudfMemoryPercent, backendConf_->get(kCudfMemoryPercent, kCudfMemoryPercentDefault)}}; + {velox::cudf_velox::CudfConfig::kCudfEnabled, "true"}, + {velox::cudf_velox::CudfConfig::kCudfDebugEnabled, backendConf_->get(kDebugCudf, kDebugCudfDefault)}, + {velox::cudf_velox::CudfConfig::kCudfMemoryResource, + backendConf_->get(kCudfMemoryResource, kCudfMemoryResourceDefault)}, + {velox::cudf_velox::CudfConfig::kCudfMemoryPercent, + backendConf_->get(kCudfMemoryPercent, kCudfMemoryPercentDefault)}}; auto& cudfConfig = velox::cudf_velox::CudfConfig::getInstance(); cudfConfig.initialize(std::move(options)); velox::cudf_velox::registerCudf();