From 75ddda965272f47ed8dc26c6a45cb62b045ec135 Mon Sep 17 00:00:00 2001 From: glutenperfbot Date: Wed, 11 Mar 2026 09:01:27 +0000 Subject: [PATCH 01/22] [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_03_11) Upstream Velox's New Commits: 4d936989e by Abhinav Mukherjee, Add the dot_product UDF (#15971) 10389727a by Pratik Pugalia, Fix Expression Fuzzer crash: block TIME type in isConstantExprSupported (#16688) 5f4a61fdc by Zhenyuan Zhao, fix(parquet): Adjust include for parquet reader (#16706) ea90ba5c3 by Zoltan Arnold Nagy, feat(cudf): Add BigintMultiRange and MultiRange filter support to AST conversion (#16628) 017699636 by Hongze Zhang, perf: Skip hash cache in `VectorHasher::lookupValueIds` when not beneficial (#16677) 60d099d3c by lingbin, refactor: Make StreamArena::allocationQuantum_ a static constexpr (#16698) ca2e35b42 by Natasha Sehgal, feat: Add messageTemplate field to VeloxException for error categorization (#16617) 813fddf91 by lingbin, fix: Update hash table doc to reflect load factor change from 7/8 to 0.7 (#16699) 8a4cf83c4 by Krishna Pai, build: Add gRPC, gh, and jq to ubuntu-22.04 container image (#16701) 7ad5eaa16 by Xiaoxuan Meng, docs: Add Meng Duan as maintainer for Core Library, Memory Management, and Fuzzers (#16702) fc857368d by Zhichen Xu, feat(rpc): Add core RPC types, client interface, and mock client [1/8] (OSS) (#16645) 91b2e4585 by Masha Basmanova, feat: Implement ParquetReader::columnStatistics() (#16700) 2435a61de by Jimmy Lu, docs: Allow multiple chains in column extraction pushdown (#16683) 56992709b by Masha Basmanova, refactor: Add format-agnostic StatisticsBuilder to dwio/common (#16693) d3cad576a by Patrick Sullivan, refactor(geo): Reduce duplicated code in geo tests (#16369) 279f7c80e by Hongze Zhang, feat: Add string dynamic filter pushdown from hash probe (#16472) 4bea9bad3 by Xiaoxuan Meng, fix: Fix TSAN data race in SortBuffer output vector reuse (#16697) f58324cd8 by kpai, fix: Remove Ubuntu Benchmark CI workflow (#16690) f01d62259 by Shruti Shivakumar, fix(cuDF): Fix cudf test failures (#16686) 344087e4f by Krishna Pai, feat: Add support for additional context for claude reviews (#16684) 2cfaa1cc2 by jiangtian, fix: Destroy the previously created accumulator for AggregateWindow to fix memory leak (#15679) 15879ed5c by Yedidya Feldblum, migrate rename to folly::available_concurrency in velox (#16678) Signed-off-by: glutenperfbot --- ep/build-velox/src/get-velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index c3e3cf0f427a..b9779bc94e3f 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_03_10-iceberg -VELOX_ENHANCED_BRANCH=ibm-2026_03_10 +VELOX_BRANCH=dft-2026_03_11 +VELOX_ENHANCED_BRANCH=ibm-2026_03_11 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF From 50e09a138e917f647e93e613c951b457e3b96e47 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 09:16:08 +0000 Subject: [PATCH 02/22] fix folly api Signed-off-by: Yuan --- dev/vcpkg/ports/folly/adding-api.patch | 23 +++++++++++++++++++++++ dev/vcpkg/ports/folly/portfile.cmake | 1 + 2 files changed, 24 insertions(+) create mode 100644 dev/vcpkg/ports/folly/adding-api.patch diff --git a/dev/vcpkg/ports/folly/adding-api.patch b/dev/vcpkg/ports/folly/adding-api.patch new file mode 100644 index 000000000000..f2cbbce7a81d --- /dev/null +++ b/dev/vcpkg/ports/folly/adding-api.patch @@ -0,0 +1,23 @@ +diff --git a/folly/system/HardwareConcurrency.cpp b/folly/system/HardwareConcurrency.cpp +index 71213c80a..cd1852034 100644 +--- a/folly/system/HardwareConcurrency.cpp ++++ b/folly/system/HardwareConcurrency.cpp +@@ -36,4 +36,18 @@ unsigned int hardware_concurrency() noexcept { + return std::thread::hardware_concurrency(); + } + ++unsigned int available_concurrency() noexcept { ++#if defined(__linux__) && !defined(__ANDROID__) ++ cpu_set_t cpuset; ++ if (!sched_getaffinity(0, sizeof(cpuset), &cpuset)) { ++ auto count = CPU_COUNT(&cpuset); ++ if (count != 0) { ++ return count; ++ } ++ } ++#endif ++ ++ return std::thread::hardware_concurrency(); ++} ++ + } // namespace folly diff --git a/dev/vcpkg/ports/folly/portfile.cmake b/dev/vcpkg/ports/folly/portfile.cmake index cd352416345a..bc1c4e969342 100644 --- a/dev/vcpkg/ports/folly/portfile.cmake +++ b/dev/vcpkg/ports/folly/portfile.cmake @@ -13,6 +13,7 @@ vcpkg_from_github( disable-uninitialized-resize-on-new-stl.patch fix-unistd-include.patch fix-absolute-dir.patch + adding-api.patch ) file(REMOVE "${SOURCE_PATH}/CMake/FindFastFloat.cmake") file(REMOVE "${SOURCE_PATH}/CMake/FindFmt.cmake") From 593b889faa136315f274cf0357dfa4a8c7f5f827 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 09:31:13 +0000 Subject: [PATCH 03/22] fix Signed-off-by: Yuan --- dev/vcpkg/ports/folly/adding-api.patch | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dev/vcpkg/ports/folly/adding-api.patch b/dev/vcpkg/ports/folly/adding-api.patch index f2cbbce7a81d..67973b93a089 100644 --- a/dev/vcpkg/ports/folly/adding-api.patch +++ b/dev/vcpkg/ports/folly/adding-api.patch @@ -21,3 +21,14 @@ index 71213c80a..cd1852034 100644 +} + } // namespace folly +diff --git a/folly/system/HardwareConcurrency.h b/folly/system/HardwareConcurrency.h +index 7c497634c..71dfb5db1 100644 +--- a/folly/system/HardwareConcurrency.h ++++ b/folly/system/HardwareConcurrency.h +@@ -20,4 +20,6 @@ namespace folly { + + unsigned int hardware_concurrency() noexcept; + ++unsigned int available_concurrency() noexcept; ++ + } // namespace folly From b36ed6f07381954e17c0b9fd41bac7dbbd76ec3f Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 11:22:38 +0000 Subject: [PATCH 04/22] bump folly in centos8 Signed-off-by: Yuan --- dev/ci-velox-buildshared-centos-8.sh | 2 +- ep/build-velox/src/setup-centos8.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index 48f7b4111585..df0d6a49e7e4 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -18,5 +18,5 @@ set -e source /opt/rh/gcc-toolset-11/enable -./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ +./dev/builddeps-veloxbe.sh --run_setup_script=ON --build_arrow=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index e056ce71dd1f..f98037552dec 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -43,8 +43,8 @@ export CC=/opt/rh/gcc-toolset-11/root/bin/gcc export CXX=/opt/rh/gcc-toolset-11/root/bin/g++ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} -FB_OS_VERSION="v2024.07.01.00" -FMT_VERSION="10.1.1" +FB_OS_VERSION="v2026.01.05.00" +FMT_VERSION="11.2.0" BOOST_VERSION="boost-1.84.0" GEOS_VERSION="3.10.7" From e020bef67a8045fcb3cabb672039c21612712251 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 11:36:09 +0000 Subject: [PATCH 05/22] fix Signed-off-by: Yuan --- ep/build-velox/src/get-velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index b9779bc94e3f..f3a01df66e31 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,7 +18,7 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_03_11 +VELOX_BRANCH=dft-2026_03_11-iceberg VELOX_ENHANCED_BRANCH=ibm-2026_03_11 VELOX_HOME="" RUN_SETUP_SCRIPT=ON From d22d2456a536b5fe831735919d5443f394a3ad25 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 13:05:15 +0000 Subject: [PATCH 06/22] Revert "bump folly in centos8" This reverts commit b36ed6f07381954e17c0b9fd41bac7dbbd76ec3f. --- dev/ci-velox-buildshared-centos-8.sh | 2 +- ep/build-velox/src/setup-centos8.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index df0d6a49e7e4..48f7b4111585 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -18,5 +18,5 @@ set -e source /opt/rh/gcc-toolset-11/enable -./dev/builddeps-veloxbe.sh --run_setup_script=ON --build_arrow=OFF --build_tests=ON \ +./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index f98037552dec..e056ce71dd1f 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -43,8 +43,8 @@ export CC=/opt/rh/gcc-toolset-11/root/bin/gcc export CXX=/opt/rh/gcc-toolset-11/root/bin/g++ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} -FB_OS_VERSION="v2026.01.05.00" -FMT_VERSION="11.2.0" +FB_OS_VERSION="v2024.07.01.00" +FMT_VERSION="10.1.1" BOOST_VERSION="boost-1.84.0" GEOS_VERSION="3.10.7" From 5ce4d7e2516667891dfb23cebb3b5e3ebc43236d Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 13:08:17 +0000 Subject: [PATCH 07/22] switch to use centos9 Signed-off-by: Yuan --- .github/workflows/velox_backend_cache.yml | 10 +++++----- .github/workflows/velox_backend_x86.yml | 8 ++++---- dev/ci-velox-buildshared-centos-9.sh | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 9 deletions(-) create mode 100755 dev/ci-velox-buildshared-centos-9.sh diff --git a/.github/workflows/velox_backend_cache.yml b/.github/workflows/velox_backend_cache.yml index ca0dc29bcc13..a13df4a6b373 100644 --- a/.github/workflows/velox_backend_cache.yml +++ b/.github/workflows/velox_backend_cache.yml @@ -89,28 +89,28 @@ jobs: strategy: matrix: os: [ ubuntu-22.04, ubuntu-24.04-arm ] - container: apache/gluten:centos-8-jdk8 + container: apache/gluten:centos-9-jdk8 steps: - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v3 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos8-release-shared-${{runner.arch}} + ccache-centos9-release-shared-${{runner.arch}} - name: Build Gluten shared libraries run: | df -a export CCACHE_MAXSIZE=1G - bash dev/ci-velox-buildshared-centos-8.sh + bash dev/ci-velox-buildshared-centos-9.sh ccache -s - name: Save Ccache uses: actions/cache/save@v3 id: ccache with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} cache-enhanced-native-lib-centos-7: if: ${{ startsWith(github.repository, 'apache/') }} diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index f6e0daac071b..3934ad78d102 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -1234,20 +1234,20 @@ jobs: cpp-test-udf-test: runs-on: ubuntu-22.04 - container: apache/gluten:centos-8-jdk8 + container: apache/gluten:centos-9-jdk8 steps: - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v4 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos8-release-shared-${{runner.arch}} + ccache-centos9-release-shared-${{runner.arch}} - name: Build Gluten native libraries run: | df -a - bash dev/ci-velox-buildshared-centos-8.sh + bash dev/ci-velox-buildshared-centos-9.sh ccache -s - name: Run CPP unit test run: | diff --git a/dev/ci-velox-buildshared-centos-9.sh b/dev/ci-velox-buildshared-centos-9.sh new file mode 100755 index 000000000000..f64bdd47b0cb --- /dev/null +++ b/dev/ci-velox-buildshared-centos-9.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +source /opt/rh/gcc-toolset-12/enable +./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ + --build_examples=ON --build_benchmarks=ON From 1243fbd374f3446f19862311bf1f8b8634366ebe Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 13:17:18 +0000 Subject: [PATCH 08/22] Revert "switch to use centos9" This reverts commit 5ce4d7e2516667891dfb23cebb3b5e3ebc43236d. --- .github/workflows/velox_backend_cache.yml | 10 +++++----- .github/workflows/velox_backend_x86.yml | 8 ++++---- dev/ci-velox-buildshared-centos-9.sh | 22 ---------------------- 3 files changed, 9 insertions(+), 31 deletions(-) delete mode 100755 dev/ci-velox-buildshared-centos-9.sh diff --git a/.github/workflows/velox_backend_cache.yml b/.github/workflows/velox_backend_cache.yml index a13df4a6b373..ca0dc29bcc13 100644 --- a/.github/workflows/velox_backend_cache.yml +++ b/.github/workflows/velox_backend_cache.yml @@ -89,28 +89,28 @@ jobs: strategy: matrix: os: [ ubuntu-22.04, ubuntu-24.04-arm ] - container: apache/gluten:centos-9-jdk8 + container: apache/gluten:centos-8-jdk8 steps: - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v3 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos9-release-shared-${{runner.arch}} + ccache-centos8-release-shared-${{runner.arch}} - name: Build Gluten shared libraries run: | df -a export CCACHE_MAXSIZE=1G - bash dev/ci-velox-buildshared-centos-9.sh + bash dev/ci-velox-buildshared-centos-8.sh ccache -s - name: Save Ccache uses: actions/cache/save@v3 id: ccache with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} cache-enhanced-native-lib-centos-7: if: ${{ startsWith(github.repository, 'apache/') }} diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 3934ad78d102..f6e0daac071b 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -1234,20 +1234,20 @@ jobs: cpp-test-udf-test: runs-on: ubuntu-22.04 - container: apache/gluten:centos-9-jdk8 + container: apache/gluten:centos-8-jdk8 steps: - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v4 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos9-release-shared-${{runner.arch}} + ccache-centos8-release-shared-${{runner.arch}} - name: Build Gluten native libraries run: | df -a - bash dev/ci-velox-buildshared-centos-9.sh + bash dev/ci-velox-buildshared-centos-8.sh ccache -s - name: Run CPP unit test run: | diff --git a/dev/ci-velox-buildshared-centos-9.sh b/dev/ci-velox-buildshared-centos-9.sh deleted file mode 100755 index f64bdd47b0cb..000000000000 --- a/dev/ci-velox-buildshared-centos-9.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -source /opt/rh/gcc-toolset-12/enable -./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ - --build_examples=ON --build_benchmarks=ON From 0ed7a93d9b19601093c42aa05eebf0272c69b5f8 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 13:34:01 +0000 Subject: [PATCH 09/22] fix Signed-off-by: Yuan --- dev/HardwareConcurrency.h | 27 +++++++++++++++++++++++++++ dev/ci-velox-buildshared-centos-8.sh | 2 ++ 2 files changed, 29 insertions(+) create mode 100644 dev/HardwareConcurrency.h diff --git a/dev/HardwareConcurrency.h b/dev/HardwareConcurrency.h new file mode 100644 index 000000000000..d2af05b69f3c --- /dev/null +++ b/dev/HardwareConcurrency.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace folly { + +unsigned int hardware_concurrency() noexcept; + +unsigned int available_concurrency() noexcept { + return hardware_concurrency(); +} + +} // namespace folly diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index 48f7b4111585..6c050a816548 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -18,5 +18,7 @@ set -e source /opt/rh/gcc-toolset-11/enable +# Fixme: this is a hack to use new header, should upgrade folly instead. +cp dev/HardwareConcurrency.h /usr/local/include/folly/system/HardwareConcurrency.h ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON From bf4039ee226a833566d48a33f96d3356ee172707 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:09:45 +0000 Subject: [PATCH 10/22] [VL] bump folly on centos8 Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index e056ce71dd1f..c31aa4ef785d 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -43,8 +43,9 @@ export CC=/opt/rh/gcc-toolset-11/root/bin/gcc export CXX=/opt/rh/gcc-toolset-11/root/bin/g++ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} -FB_OS_VERSION="v2024.07.01.00" -FMT_VERSION="10.1.1" +FB_OS_VERSION="v2026.01.05.00" +FMT_VERSION="11.2.0" +FAST_FLOAT_VERSION="v8.0.2" BOOST_VERSION="boost-1.84.0" GEOS_VERSION="3.10.7" @@ -78,6 +79,12 @@ function install_conda { dnf_install conda } +function install_git { + # Remove an older version if present. + dnf remove -y git + wget_and_untar https://github.com/git/git/archive/v2.44.0.tar.gz git + cmake_install_dir git +} function install_gflags { # Remove an older version if present. @@ -136,6 +143,11 @@ function install_fizz { cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF } +function install_fast_float { + wget_and_untar https://github.com/fastfloat/fast_float/archive/refs/tags/"${FAST_FLOAT_VERSION}".tar.gz fast_float + cmake_install_dir fast_float -DBUILD_TESTS=OFF +} + function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly cmake_install_dir folly -DFOLLY_HAVE_INT128_T=ON -DFOLLY_NO_EXCEPTION_TRACER=ON @@ -179,6 +191,7 @@ function install_geos { function install_velox_deps { run_and_time install_velox_deps_from_dnf + run_and_time install_git run_and_time install_conda run_and_time install_gflags run_and_time install_glog @@ -187,6 +200,7 @@ function install_velox_deps { run_and_time install_boost run_and_time install_protobuf run_and_time install_fmt + run_and_time install_fast_float run_and_time install_folly run_and_time install_fizz run_and_time install_wangle From b602c09522eab7c90c3f33697941021f19656dc0 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:15:52 +0000 Subject: [PATCH 11/22] fix Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index c31aa4ef785d..e5c313558e48 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -83,7 +83,10 @@ function install_git { # Remove an older version if present. dnf remove -y git wget_and_untar https://github.com/git/git/archive/v2.44.0.tar.gz git - cmake_install_dir git + cd git + make prefix=/usr/local all -j$(nproc) + make prefix=/usr/local install + cd ../ } function install_gflags { From be3ac0c93dc03e112fcb0afe94bc48c8fda54d23 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:18:44 +0000 Subject: [PATCH 12/22] fix Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index e5c313558e48..089e4bf8f570 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -59,7 +59,7 @@ function install_build_prerequisites { dnf_install epel-release dnf-plugins-core # For ccache, ninja dnf config-manager --set-enabled powertools dnf update -y - dnf_install ninja-build curl ccache gcc-toolset-11 git wget which + dnf_install ninja-build curl ccache gcc-toolset-11 git wget which expat-devel gettext-devel dnf_install yasm dnf_install autoconf automake python39 python39-devel python39-pip libtool pip3.9 install cmake==3.28.3 From 6f293188899215f775891fa50fafaa241499281b Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:23:29 +0000 Subject: [PATCH 13/22] fix Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 089e4bf8f570..8a8587edf62b 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -83,10 +83,11 @@ function install_git { # Remove an older version if present. dnf remove -y git wget_and_untar https://github.com/git/git/archive/v2.44.0.tar.gz git - cd git - make prefix=/usr/local all -j$(nproc) - make prefix=/usr/local install - cd ../ + ( + cd ${DEPENDENCY_DIR}/git + make prefix=/usr/local all -j$(nproc) + make prefix=/usr/local install + ) } function install_gflags { From 8ede7fa02825b23a7a2c17a49d4ebacb04b7a2b8 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:24:42 +0000 Subject: [PATCH 14/22] fix Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 8a8587edf62b..c8fc575793ac 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -82,7 +82,7 @@ function install_conda { function install_git { # Remove an older version if present. dnf remove -y git - wget_and_untar https://github.com/git/git/archive/v2.44.0.tar.gz git + wget_and_untar https://github.com/git/git/archive/v2.52.0.tar.gz git ( cd ${DEPENDENCY_DIR}/git make prefix=/usr/local all -j$(nproc) From 949d69fcb157aa8b37e054c7026502f9a7ba2dbe Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:35:08 +0000 Subject: [PATCH 15/22] fix Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index c8fc575793ac..1e4ba42a773f 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -176,7 +176,7 @@ function install_duckdb { if $BUILD_DUCKDB ; then echo 'Building DuckDB' wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb - cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release + cmake_install_dir duckdb -DGIT_COMMIT_HASH="6536a77" -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release fi } From db928c503844537c6c941a3b2a16f187d1319ba4 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:57:20 +0000 Subject: [PATCH 16/22] fix Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 1e4ba42a773f..857415fae69d 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -90,6 +90,14 @@ function install_git { ) } +function install_xxhash { + wget_and_untar https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz xxhash + ( + cd ${DEPENDENCY_DIR}/xxhash + make && make install + ) +} + function install_gflags { # Remove an older version if present. dnf remove -y gflags From 22e84bc98dbd8e79dc298d2314203e511ae7b67b Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 14:58:14 +0000 Subject: [PATCH 17/22] fix Signed-off-by: Yuan --- ep/build-velox/src/setup-centos8.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 857415fae69d..ff69dc97ddf7 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -213,6 +213,7 @@ function install_velox_deps { run_and_time install_protobuf run_and_time install_fmt run_and_time install_fast_float + run_and_time install_xxhash run_and_time install_folly run_and_time install_fizz run_and_time install_wangle From 00ea70187719f48906bc6ff90a25a0c29dadaa65 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 17:29:35 +0000 Subject: [PATCH 18/22] test Signed-off-by: Yuan --- .github/workflows/docker_image.yml | 1 - dev/docker/Dockerfile.centos8-dynamic-build | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 63ab57c28523..11c28528da94 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -263,7 +263,6 @@ jobs: retention-days: 1 build-centos-8: - if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index 9e49b524e19f..cdaa55a186f9 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -36,7 +36,7 @@ RUN set -ex; \ wget -nv ${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download -O /opt/apache-celeborn-0.6.1-bin.tgz; \ wget -nv ${mirror_host}/uniffle/0.10.0/apache-uniffle-0.10.0-bin.tar.gz?action=download -O /opt/apache-uniffle-0.10.0-bin.tar.gz; \ wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download -O /opt/hadoop-2.8.5.tar.gz; \ - git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ + git clone --depth=1 https://github.com/zhouyuan/gluten /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ ./install-resources.sh 3.3; \ ./install-resources.sh 3.4; \ From 73dd8b5bc1c58cc2a2ba3ff874cd4ef4b6e00492 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 20:43:57 +0000 Subject: [PATCH 19/22] Revert "test" This reverts commit 002715cfd2e4e4fc5df84fa55f5ac18f698b72f9. --- .github/workflows/docker_image.yml | 1 + dev/docker/Dockerfile.centos8-dynamic-build | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 11c28528da94..63ab57c28523 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -263,6 +263,7 @@ jobs: retention-days: 1 build-centos-8: + if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index cdaa55a186f9..9e49b524e19f 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -36,7 +36,7 @@ RUN set -ex; \ wget -nv ${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download -O /opt/apache-celeborn-0.6.1-bin.tgz; \ wget -nv ${mirror_host}/uniffle/0.10.0/apache-uniffle-0.10.0-bin.tar.gz?action=download -O /opt/apache-uniffle-0.10.0-bin.tar.gz; \ wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download -O /opt/hadoop-2.8.5.tar.gz; \ - git clone --depth=1 https://github.com/zhouyuan/gluten /opt/gluten; \ + git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ ./install-resources.sh 3.3; \ ./install-resources.sh 3.4; \ From 15dd142270a1bee88764ebc08ae0a01ee9cc1041 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 11 Mar 2026 17:21:32 +0000 Subject: [PATCH 20/22] [GLUTEN-11316][VL] Followup to fix spark-4.1 nightly package (#11744) * fix nightly package Signed-off-by: Yuan * test Signed-off-by: Yuan * remove paimon Signed-off-by: Yuan * Revert "test" This reverts commit 183f9304dea55584e5cbaed2938bff0b6725735e. --------- Signed-off-by: Yuan --- .github/workflows/velox_nightly.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/velox_nightly.yml b/.github/workflows/velox_nightly.yml index 2a8bf47994a7..035494a6fd7d 100644 --- a/.github/workflows/velox_nightly.yml +++ b/.github/workflows/velox_nightly.yml @@ -171,7 +171,8 @@ jobs: run: | cd $GITHUB_WORKSPACE/ && \ ./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip - ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip + # iceberg/hudi support for spark-4.1 is not avaiable, skip it first + ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21 -Pbackends-velox -Pceleborn -Puniffle -Pdelta -DskipTests -Dmaven.source.skip - name: Upload bundle package uses: actions/upload-artifact@v4 with: @@ -287,8 +288,8 @@ jobs: ./build/mvn clean install -Pspark-3.4 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip - # iceberg support for spark-4.1 with java-17 is not avaiable, skip it first - ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21 -Pbackends-velox -Pceleborn -Puniffle -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip + # iceberg/hudi support for spark-4.1 is not avaiable, skip it first + ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Pdelta -DskipTests -Dmaven.source.skip - name: Upload bundle package uses: actions/upload-artifact@v4 with: @@ -323,8 +324,8 @@ jobs: run: | cd $GITHUB_WORKSPACE/ && \ ./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip - # iceberg support for spark-4.1 with java-21 is not avaiable, skip it first - ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21 -Pbackends-velox -Pceleborn -Puniffle -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip + # iceberg/hudi support for spark-4.1 is not avaiable, skip it first + ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21 -Pbackends-velox -Pceleborn -Puniffle -Pdelta -DskipTests -Dmaven.source.skip - name: Upload bundle package uses: actions/upload-artifact@v4 with: @@ -486,8 +487,8 @@ jobs: ./build/mvn clean install -Pspark-3.4 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip - # iceberg support for spark-4.1 with java-17 is not avaiable, skip it first - ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Phudi -Pdelta -Ppaimon -DskipTests -Dmaven.source.skip + # iceberg/hudi support for spark-4.1 with java-17 is not avaiable, skip it first + ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pceleborn -Puniffle -Pdelta -DskipTests -Dmaven.source.skip - name: Upload bundle package uses: actions/upload-artifact@v4 with: From d2a8f25015cfe823b2a959db210079ba54f4997e Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Thu, 12 Mar 2026 01:45:21 +0800 Subject: [PATCH 21/22] [GLUTEN-11550][UT] Enable GlutenDataFrameSubquerySuite for Spark 4.1 (#11727) Fix struct join key validation to align with Spark's equalsStructurally semantics. Remove struct field name comparison in sameType() since Spark's HashJoin only checks structural type compatibility, not field names. Also catch exceptions during offload attempt in validation pipeline to allow graceful fallback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../scala/org/apache/gluten/execution/JoinExecTransformer.scala | 1 - .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala index b4fa188f44e6..ed98bdc1a56c 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala @@ -118,7 +118,6 @@ trait HashJoinLikeExecTransformer extends BaseJoinExec with TransformSupport { fromFields.length == toFields.length && fromFields.zip(toFields).forall { case (l, r) => - l.name.equalsIgnoreCase(r.name) && sameType(l.dataType, r.dataType) } diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 0dadfa1d0bd8..2d693bcd9e01 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -780,7 +780,7 @@ class VeloxTestSettings extends BackendTestSettings { // Generated suites for org.apache.spark.sql enableSuite[GlutenCacheManagerSuite] enableSuite[GlutenDataFrameShowSuite] - // TODO: 4.x enableSuite[GlutenDataFrameSubquerySuite] // 1 failure + enableSuite[GlutenDataFrameSubquerySuite] enableSuite[GlutenDataFrameTableValuedFunctionsSuite] enableSuite[GlutenDataFrameTransposeSuite] enableSuite[GlutenDeprecatedDatasetAggregatorSuite] From 1501fefbd16c6348428e8bc7c6fc8633f1dabff7 Mon Sep 17 00:00:00 2001 From: infvg Date: Wed, 11 Mar 2026 18:05:13 +0300 Subject: [PATCH 22/22] Temp test Change branch # Conflicts: # ep/build-velox/src/get-velox.sh # Conflicts: # ep/build-velox/src/get-velox.sh --- .../enhanced/VeloxIcebergSuite.scala | 136 ++++++++++++++++++ ep/build-velox/src/get-velox.sh | 2 +- 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala index c3d3c8edc658..53ac42ef2801 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala @@ -16,6 +16,7 @@ */ package org.apache.gluten.execution.enhanced +import org.apache.gluten.config.VeloxConfig import org.apache.gluten.execution._ import org.apache.gluten.tags.EnhancedFeaturesTest @@ -383,4 +384,139 @@ class VeloxIcebergSuite extends IcebergSuite { } } } + test("iceberg max aggregate with nulls enhanced true") { + withSQLConf(VeloxConfig.ENABLE_ENHANCED_FEATURES.key -> "true") { + withTable("store_sales_test_nulls_enhanced_true") { + spark.sql(""" + |create table store_sales_test_nulls_enhanced_true ( + | ss_item_sk int, + | ss_sales_price decimal(7,2) + |) using iceberg + |""".stripMargin) + + spark.sql(""" + |insert into store_sales_test_nulls_enhanced_true values + |(1, 200.00), + |(2, 200.00), + |(3, null), + |(4, 199.98), + |(5, 199.96) + |""".stripMargin) + + val result = spark + .sql("select max(ss_sales_price) from store_sales_test_nulls_enhanced_true") + .collect() + + assert(result.length == 1, "Should return 1 row") + assert(result(0).get(0) != null, "MAX should not return NULL") + assert( + result(0).getDecimal(0).doubleValue() == 200.00, + s"MAX should return 200.00, but got ${result(0).get(0)}" + ) + } + } + } + + test("iceberg max aggregate without nulls enhanced true") { + withSQLConf(VeloxConfig.ENABLE_ENHANCED_FEATURES.key -> "true") { + withTable("store_sales_test_no_nulls_enhanced_true") { + spark.sql(""" + |create table store_sales_test_no_nulls_enhanced_true ( + | ss_item_sk int, + | ss_sales_price decimal(7,2) + |) using iceberg + |""".stripMargin) + + spark.sql(""" + |insert into store_sales_test_no_nulls_enhanced_true values + |(1, 200.00), + |(2, 200.00), + |(3, 200.00), + |(4, 199.98), + |(5, 199.96), + |(6, 199.96), + |(7, 199.92), + |(8, 199.92), + |(9, 199.92), + |(10, 199.90), + |(11, null) + |""".stripMargin) + + val result = spark + .sql("select max(ss_sales_price) from store_sales_test_no_nulls_enhanced_true where ss_sales_price is not null") + .collect() + + assert(result.length == 1, "Should return 1 row") + assert(result(0).get(0) != null, "MAX should not return NULL") + assert( + result(0).getDecimal(0).doubleValue() == 200.00, + s"MAX should return 200.00, but got ${result(0).get(0)}" + ) + } + } + } + + test("iceberg max aggregate with nulls enhanced false") { + withSQLConf(VeloxConfig.ENABLE_ENHANCED_FEATURES.key -> "false") { + withTable("store_sales_test_nulls_enhanced_false") { + spark.sql(""" + |create table store_sales_test_nulls_enhanced_false ( + | ss_item_sk int, + | ss_sales_price decimal(7,2) + |) using iceberg + |""".stripMargin) + + spark.sql(""" + |insert into store_sales_test_nulls_enhanced_false values + |(1, 200.00), + |(2, 199.98), + |(3, null), + |(4, 199.96) + |""".stripMargin) + + val result = spark + .sql("select max(ss_sales_price) from store_sales_test_nulls_enhanced_false") + .collect() + + assert(result.length == 1, "Should return 1 row") + assert(result(0).get(0) != null, "MAX should not return NULL") + assert( + result(0).getDecimal(0).doubleValue() == 200.00, + s"MAX should return 200.00, but got ${result(0).get(0)}" + ) + } + } + } + + test("iceberg max aggregate without nulls enhanced false") { + withSQLConf(VeloxConfig.ENABLE_ENHANCED_FEATURES.key -> "false") { + withTable("store_sales_test_no_nulls_enhanced_false") { + spark.sql(""" + |create table store_sales_test_no_nulls_enhanced_false ( + | ss_item_sk int, + | ss_sales_price decimal(7,2) + |) using iceberg + |""".stripMargin) + + spark.sql(""" + |insert into store_sales_test_no_nulls_enhanced_false values + |(1, 200.00), + |(2, 199.98), + |(3, 199.96), + |(4, null) + |""".stripMargin) + + val result = spark + .sql("select max(ss_sales_price) from store_sales_test_no_nulls_enhanced_false where ss_sales_price is not null") + .collect() + + assert(result.length == 1, "Should return 1 row") + assert(result(0).get(0) != null, "MAX should not return NULL") + assert( + result(0).getDecimal(0).doubleValue() == 200.00, + s"MAX should return 200.00, but got ${result(0).get(0)}" + ) + } + } + } } diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index f3a01df66e31..e39b146f704f 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -19,7 +19,7 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git VELOX_BRANCH=dft-2026_03_11-iceberg -VELOX_ENHANCED_BRANCH=ibm-2026_03_11 +VELOX_ENHANCED_BRANCH=temp-fix VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF