diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 3278e9550661..f6e0daac071b 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -821,7 +821,7 @@ jobs: java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ -DargLine="-Dspark.test.home=$SPARK_HOME" - name: Upload test report diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala index de9eaba59791..edb30dac61fa 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala @@ -16,7 +16,4 @@ */ package org.apache.gluten.execution -import org.apache.gluten.tags.SkipTest - -@SkipTest class VeloxIcebergSuite extends IcebergSuite diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala index 42820c04dcaf..5456e0ba89df 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala @@ -16,8 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.gluten.tags.SkipTest - import org.apache.spark.SparkConf import org.apache.spark.sql.functions.col @@ -25,7 +23,6 @@ import org.apache.iceberg.spark.SparkWriteOptions import java.io.File -@SkipTest class VeloxTPCHIcebergSuite extends VeloxTPCHSuite { protected val tpchBasePath: String = getClass.getResource("/").getPath + "../../../src/test/resources" diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 421765fdea57..be31f18206b3 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -303,9 +303,6 @@ if(BUILD_TESTS) import_library( facebook::velox::file_test_utils ${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a) - import_library( - facebook::velox::temp_path - ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_temp_path.a) import_library( facebook::velox::exec_test_lib ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_exec_test_lib.a) @@ -313,7 +310,7 @@ if(BUILD_TESTS) facebook::velox::exec_test_lib INTERFACE facebook::velox::vector_test_lib facebook::velox::dwio_common_test - facebook::velox::file_test_utils facebook::velox::temp_path) + facebook::velox::file_test_utils) target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib) endif() diff --git a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc index b9993df0e778..0a859bc9b43f 100644 --- a/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc +++ b/cpp/velox/operators/serializer/VeloxGpuColumnarBatchSerializer.cc @@ -24,8 +24,8 @@ #include "velox/common/memory/Memory.h" #include "velox/vector/FlatVector.h" #include "velox/vector/arrow/Bridge.h" +#include "velox/experimental/cudf/exec/GpuResources.h" #include "velox/experimental/cudf/exec/VeloxCudfInterop.h" -#include "velox/experimental/cudf/exec/Utilities.h" #include "velox/experimental/cudf/vector/CudfVector.h" #include @@ -44,7 +44,8 @@ VeloxGpuColumnarBatchSerializer::VeloxGpuColumnarBatchSerializer( std::shared_ptr VeloxGpuColumnarBatchSerializer::deserialize(uint8_t* data, int32_t size) { auto vb = VeloxColumnarBatchSerializer::deserialize(data, size); auto stream = cudf_velox::cudfGlobalStreamPool().get_stream(); - auto table = cudf_velox::with_arrow::toCudfTable(dynamic_pointer_cast(vb)->getRowVector(), veloxPool_.get(), stream); + auto table = cudf_velox::with_arrow::toCudfTable( + dynamic_pointer_cast(vb)->getRowVector(), veloxPool_.get(), stream, cudf_velox::get_output_mr()); stream.synchronize(); auto vector = std::make_shared( veloxPool_.get(), rowType_, size, std::move(table), stream); diff --git a/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc b/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc index 364d31e180cc..99c560029d55 100644 --- a/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc +++ b/cpp/velox/tests/VeloxGpuShuffleWriterTest.cc @@ -29,6 +29,7 @@ #include "memory/GpuBufferColumnarBatch.h" #include "utils/GpuBufferBatchResizer.h" +#include "velox/experimental/cudf/CudfNoDefaults.h" #include "velox/experimental/cudf/exec/VeloxCudfInterop.h" #include "velox/experimental/cudf/vector/CudfVector.h" #include "velox/vector/tests/utils/VectorTestBase.h" @@ -126,7 +127,7 @@ RowVectorPtr mergeBufferColumnarBatches(std::vectorgetLeafMemoryPool().get(), "", vector->stream()); + tableView, getDefaultMemoryManager()->getLeafMemoryPool().get(), "", vector->stream(), cudf_velox::get_temp_mr()); } std::vector getTestParams() { diff --git a/cpp/velox/utils/GpuBufferBatchResizer.cc b/cpp/velox/utils/GpuBufferBatchResizer.cc index 518f280fc3eb..921f05c63b70 100644 --- a/cpp/velox/utils/GpuBufferBatchResizer.cc +++ b/cpp/velox/utils/GpuBufferBatchResizer.cc @@ -19,7 +19,7 @@ #include "cudf/GpuLock.h" #include "memory/GpuBufferColumnarBatch.h" #include "utils/Timer.h" -#include "velox/experimental/cudf/exec/Utilities.h" +#include "velox/experimental/cudf/exec/GpuResources.h" #include "velox/experimental/cudf/exec/VeloxCudfInterop.h" #include "velox/experimental/cudf/vector/CudfVector.h" #include "velox/vector/FlatVector.h" diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index e0ee3a49198f..dbfe9f2d5b12 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_02_24 -VELOX_ENHANCED_BRANCH=ibm-2026_02_24 +VELOX_BRANCH=dft-2026_03_03-iceberg +VELOX_ENHANCED_BRANCH=ibm-2026_03_03 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF