From d95c7bd55e4d4ac1805e8df8965588c4cb6b4ff2 Mon Sep 17 00:00:00 2001 From: glutenperfbot Date: Tue, 3 Mar 2026 10:24:52 +0000 Subject: [PATCH 1/9] [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_03_03) Upstream Velox's New Commits: f7906fbbc by Kent Yao, feat(spark): Add assert_not_null function (#16562) 7d3072c85 by Anass Al-Wohoush, Revert D94871321: feat: Add cache lookup and zero-copy APIs to BufferedInput 54eed172a by Xiaoxuan Meng, feat: Add cache lookup and zero-copy APIs to BufferedInput (#16587) 283626a64 by Simon Eves, feat: Support Arrow Decimal64 type via Bridge (#16474) 2d8cec59c by Karthikeyan, fix(cudf): Replace default memory resource and stream with explicit output MR and data stream in Cudf operators (#16454) 9a8e535ee by Wei He, feat(aggregate): Support variadic arguments in simple aggregation function interface (#16475) 236424117 by Krishna Pai, feat: Update to Opus 4.6 and make model configurable (#16594) f90730ce5 by Jimmy Lu, feat: Add table_scan_output_batch_rows_override (#16571) 0d8ca0ce8 by Shang Ma, build: Explicitly enable folly coroutines when compiler supports them (#16581) 11b64f0ca by Xiao Du, feat: Impl memory compact in Aggregate and GroupingSet (#16476) b38f9dc6f by Masha Basmanova, feat: Add kindName() API to IExpr (#16576) 68b2f34ce by Ke Wang, feat: Add option to collect decompression time for drwf reader (#16523) 7b2e58464 by Raymond Lin, fix: Handle empty keyNodes in SelectiveFlatMapAsStructReader (#16570) 76d10eb4c by Ke Wang, feat: Add totalRemainingFilterCpuNanos metric for table scan (#16564) 421c6fc0e by Devavret Makkar, fix(cuDF): Fix stream ordering issue in CudfHashAggregation Distinct (#16525) 70e9b784d by Krishna Pai, feat(ci): Add /claude-query command for asking questions about PRs (#16567) c3f4ec311 by Rui Mo, feat: Add Spark TIMESTAMP_NTZ type (#16344) e6358b677 by Pramod Satya, refactor(cuDF): Consolidate cuDF configurations into single config file (#16528) 085e6f9c5 by Rui Mo, fix(fuzzer): Disable file handle cache in Aggregate fuzzer test (#16548) bf2a4107a by Shrinidhi Joshi, docs: Add hash-table-caching documentation (#16554) f760d54f9 by Shakyan Kushwaha, feat: Add localtimestamp Presto function (#15605) ecd5a60cc by Allen Shen, refactor: Extract apply lambda utility function (#16315) 6de3b6ddb by Xiaoxuan Meng, feat: Add FileLayout API for optimized reader IO and writer state tracking (#16561) 600b0ba8a by Eric Jia, refactor: Optimize HiveConnectorUtil: deduplicate and endWith functions (#16508) 01c114921 by Jimmy Lu, perf: Optimize deduplicated readers when all nested rows are selected (#16550) 493517d19 by Xiaoxuan Meng, fix: Clear barrier flag when task terminates (#16539) 090d01368 by Krishna Pai, feat(ci): Use Opus 4.1 model with stricter review guidelines (#16552) 54b03a289 by Simon Eves, misc(cudf): Disable test AggregationTest.countStarGlobal pending a fix for global COUNT(*) (#16493) c14f6e380 by Charlie Arnold, Add Rust code generation for RemoteFunction.thrift (#16458) 4e5fb428f by Krishna Pai, fix(ci): Fix comment author field and PR permissions in claude-review (#16540) caad670d0 by Natasha Sehgal, feat: Use emplace_back instead of push_back in Parquet writer Schema (#16392) 9046fe2b3 by Krishna Pai, fix: Update review bot to use results response (#16536) 53dcda01e by Xiaoxuan Meng, fix: Clear AsyncDataCache in test SetUp to fix flakiness (#16533) 3591d31dc by Krishna Pai, fix(ci): Enable posting of claude reviews (#16534) cfc3353a2 by Abhinav Mukherjee, Add TRANSFORM_WITH_INDEX UDF (#15978) b6e5220f9 by Abhinav Mukherjee, Fix PartitionedOutput crash after flush (#16496) 8693d331e by Abhinav Mukherjee, Add map_trim_values UDF (#15973) 15c37e98a by Ankita Victor-Levi, perf: Fix redundant filter evaluation in null-aware joins (#16460) 335ac2faa by Bradley Dice, fix(cuda): Scope -G debug flag to Velox targets only (#16470) 360c2d8c7 by Joy Haldar, feat: Skip iceberg positional delete files by position bounds (#16268) Signed-off-by: glutenperfbot --- ep/build-velox/src/get-velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index e0ee3a49198f..8a30dc01d1b5 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_02_24 -VELOX_ENHANCED_BRANCH=ibm-2026_02_24 +VELOX_BRANCH=dft-2026_03_03 +VELOX_ENHANCED_BRANCH=ibm-2026_03_03 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF From 9007abfb4c472c9a862b7e7e4ea8c0c1e951327c Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 3 Mar 2026 13:22:46 +0000 Subject: [PATCH 2/9] fix Signed-off-by: Yuan --- cpp/velox/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index eb0e11e04b59..858ed42a1a5a 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -303,9 +303,6 @@ if(BUILD_TESTS) import_library( facebook::velox::file_test_utils ${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a) - import_library( - facebook::velox::temp_path - ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_temp_path.a) import_library( facebook::velox::exec_test_lib ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_exec_test_lib.a) From 9c676c01b3a64bc247b4f9e369e823dc02ca070b Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 3 Mar 2026 13:32:58 +0000 Subject: [PATCH 3/9] fix Signed-off-by: Yuan --- cpp/velox/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 858ed42a1a5a..f1f87d23884d 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -310,7 +310,7 @@ if(BUILD_TESTS) facebook::velox::exec_test_lib INTERFACE facebook::velox::vector_test_lib facebook::velox::dwio_common_test - facebook::velox::file_test_utils facebook::velox::temp_path) + facebook::velox::file_test_utils) target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib) endif() From a29a9be4f1159bcdd9659871c16596611e501d62 Mon Sep 17 00:00:00 2001 From: Rui Mo Date: Thu, 19 Feb 2026 15:03:35 +0000 Subject: [PATCH 4/9] Enable iceberg tests --- .github/workflows/velox_backend_x86.yml | 2 +- .../scala/org/apache/gluten/execution/VeloxIcebergSuite.scala | 3 --- .../org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 3278e9550661..f6e0daac071b 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -821,7 +821,7 @@ jobs: java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ -DargLine="-Dspark.test.home=$SPARK_HOME" - name: Upload test report diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala index de9eaba59791..edb30dac61fa 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala @@ -16,7 +16,4 @@ */ package org.apache.gluten.execution -import org.apache.gluten.tags.SkipTest - -@SkipTest class VeloxIcebergSuite extends IcebergSuite diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala index 42820c04dcaf..5456e0ba89df 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala @@ -16,8 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.gluten.tags.SkipTest - import org.apache.spark.SparkConf import org.apache.spark.sql.functions.col @@ -25,7 +23,6 @@ import org.apache.iceberg.spark.SparkWriteOptions import java.io.File -@SkipTest class VeloxTPCHIcebergSuite extends VeloxTPCHSuite { protected val tpchBasePath: String = getClass.getResource("/").getPath + "../../../src/test/resources" From bd5113948cf5221d2c4c6c1e4ed3281e7113fc4c Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 3 Mar 2026 15:30:45 +0000 Subject: [PATCH 5/9] enable iceberg connector Signed-off-by: Yuan --- ep/build-velox/src/get-velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 8a30dc01d1b5..dbfe9f2d5b12 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,7 +18,7 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_03_03 +VELOX_BRANCH=dft-2026_03_03-iceberg VELOX_ENHANCED_BRANCH=ibm-2026_03_03 VELOX_HOME="" RUN_SETUP_SCRIPT=ON From 32e8b8fa51e5ef3388fea7fe4304c3f8fdfb058f Mon Sep 17 00:00:00 2001 From: Chengcheng Jin Date: Wed, 4 Mar 2026 11:23:31 +0000 Subject: [PATCH 6/9] fix GPU API --- .../operators/serializer/VeloxGpuColumnarBatchSerializer.cc | 2 +- cpp/velox/utils/GpuBufferBatchResizer.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc index b9993df0e778..a71f81e89b36 100644 --- a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc +++ b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc @@ -24,8 +24,8 @@ #include "velox/common/memory/Memory.h" #include "velox/vector/FlatVector.h" #include "velox/vector/arrow/Bridge.h" +#include "velox/experimental/cudf/exec/GpuResources.h" #include "velox/experimental/cudf/exec/VeloxCudfInterop.h" -#include "velox/experimental/cudf/exec/Utilities.h" #include "velox/experimental/cudf/vector/CudfVector.h" #include diff --git a/cpp/velox/utils/GpuBufferBatchResizer.cc b/cpp/velox/utils/GpuBufferBatchResizer.cc index 518f280fc3eb..921f05c63b70 100644 --- a/cpp/velox/utils/GpuBufferBatchResizer.cc +++ b/cpp/velox/utils/GpuBufferBatchResizer.cc @@ -19,7 +19,7 @@ #include "cudf/GpuLock.h" #include "memory/GpuBufferColumnarBatch.h" #include "utils/Timer.h" -#include "velox/experimental/cudf/exec/Utilities.h" +#include "velox/experimental/cudf/exec/GpuResources.h" #include "velox/experimental/cudf/exec/VeloxCudfInterop.h" #include "velox/experimental/cudf/vector/CudfVector.h" #include "velox/vector/FlatVector.h" From 1d9b54e4082dd69987c1a3a3348b656f86f85652 Mon Sep 17 00:00:00 2001 From: Chengcheng Jin Date: Wed, 4 Mar 2026 15:26:30 +0000 Subject: [PATCH 7/9] fix gpu build --- .../operators/serializer/VeloxGpuColumnarBatchSerializer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc index a71f81e89b36..ac50f356cbe0 100644 --- a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc +++ b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc @@ -44,7 +44,8 @@ VeloxGpuColumnarBatchSerializer::VeloxGpuColumnarBatchSerializer( std::shared_ptr VeloxGpuColumnarBatchSerializer::deserialize(uint8_t* data, int32_t size) { auto vb = VeloxColumnarBatchSerializer::deserialize(data, size); auto stream = cudf_velox::cudfGlobalStreamPool().get_stream(); - auto table = cudf_velox::with_arrow::toCudfTable(dynamic_pointer_cast(vb)->getRowVector(), veloxPool_.get(), stream); + auto table = cudf_velox::with_arrow::toCudfTable( + dynamic_pointer_cast(vb)->getRowVector(), veloxPool_.get(), stream, , get_output_mr()); stream.synchronize(); auto vector = std::make_shared( veloxPool_.get(), rowType_, size, std::move(table), stream); From e591948a49541a8806dc8c9ce711ed873e1e1c31 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 4 Mar 2026 19:51:08 +0000 Subject: [PATCH 8/9] fix Signed-off-by: Yuan --- .../operators/serializer/VeloxGpuColumnarBatchSerializer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc index ac50f356cbe0..0a859bc9b43f 100644 --- a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc +++ b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc @@ -45,7 +45,7 @@ std::shared_ptr VeloxGpuColumnarBatchSerializer::deserialize(uint auto vb = VeloxColumnarBatchSerializer::deserialize(data, size); auto stream = cudf_velox::cudfGlobalStreamPool().get_stream(); auto table = cudf_velox::with_arrow::toCudfTable( - dynamic_pointer_cast(vb)->getRowVector(), veloxPool_.get(), stream, , get_output_mr()); + dynamic_pointer_cast(vb)->getRowVector(), veloxPool_.get(), stream, cudf_velox::get_output_mr()); stream.synchronize(); auto vector = std::make_shared( veloxPool_.get(), rowType_, size, std::move(table), stream); From 366b4326734f63637323a25a830343e9e66312b1 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 4 Mar 2026 21:36:51 +0000 Subject: [PATCH 9/9] fix test Signed-off-by: Yuan --- cpp/velox/tests/VeloxGpuShuffleWriterTest.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc b/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc index 364d31e180cc..99c560029d55 100644 --- a/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc +++ b/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc @@ -29,6 +29,7 @@ #include "memory/GpuBufferColumnarBatch.h" #include "utils/GpuBufferBatchResizer.h" +#include "velox/experimental/cudf/CudfNoDefaults.h" #include "velox/experimental/cudf/exec/VeloxCudfInterop.h" #include "velox/experimental/cudf/vector/CudfVector.h" #include "velox/vector/tests/utils/VectorTestBase.h" @@ -126,7 +127,7 @@ RowVectorPtr mergeBufferColumnarBatches(std::vectorgetLeafMemoryPool().get(), "", vector->stream()); + tableView, getDefaultMemoryManager()->getLeafMemoryPool().get(), "", vector->stream(), cudf_velox::get_temp_mr()); } std::vector getTestParams() {