diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 6edd8162ea13..3ac8edc42445 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -821,7 +821,7 @@ jobs: java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ -DargLine="-Dspark.test.home=$SPARK_HOME" - name: Upload test report diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala index edb30dac61fa..de9eaba59791 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala @@ -16,4 +16,7 @@ */ package org.apache.gluten.execution +import org.apache.gluten.tags.SkipTest + +@SkipTest class VeloxIcebergSuite extends IcebergSuite diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala index 5456e0ba89df..42820c04dcaf 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.gluten.execution +import org.apache.gluten.tags.SkipTest + import org.apache.spark.SparkConf import org.apache.spark.sql.functions.col @@ -23,6 +25,7 @@ import org.apache.iceberg.spark.SparkWriteOptions import java.io.File +@SkipTest class VeloxTPCHIcebergSuite extends VeloxTPCHSuite { protected val tpchBasePath: String = getClass.getResource("/").getPath + "../../../src/test/resources" diff --git a/cpp/velox/memory/GlutenBufferedInputBuilder.h b/cpp/velox/memory/GlutenBufferedInputBuilder.h index 86116ff1e88e..25b05f98707e 100644 --- a/cpp/velox/memory/GlutenBufferedInputBuilder.h +++ b/cpp/velox/memory/GlutenBufferedInputBuilder.h @@ -30,8 +30,8 @@ class GlutenBufferedInputBuilder : public facebook::velox::connector::hive::Buff const facebook::velox::FileHandle& fileHandle, const facebook::velox::dwio::common::ReaderOptions& readerOpts, const facebook::velox::connector::ConnectorQueryCtx* connectorQueryCtx, - std::shared_ptr ioStats, - std::shared_ptr fsStats, + std::shared_ptr ioStatistics, + std::shared_ptr ioStats, folly::Executor* executor, const folly::F14FastMap& fileReadOps = {}) override { if (connectorQueryCtx->cache()) { @@ -42,8 +42,8 @@ class GlutenBufferedInputBuilder : public facebook::velox::connector::hive::Buff connectorQueryCtx->cache(), facebook::velox::connector::Connector::getTracker(connectorQueryCtx->scanId(), readerOpts.loadQuantum()), fileHandle.groupId, - ioStats, - std::move(fsStats), + std::move(ioStatistics), + std::move(ioStats), executor, readerOpts, fileReadOps); @@ -54,8 +54,8 @@ class GlutenBufferedInputBuilder : public facebook::velox::connector::hive::Buff fileHandle.uuid, facebook::velox::connector::Connector::getTracker(connectorQueryCtx->scanId(), readerOpts.loadQuantum()), fileHandle.groupId, + std::move(ioStatistics), std::move(ioStats), - std::move(fsStats), executor, readerOpts, fileReadOps); diff --git a/cpp/velox/memory/GlutenDirectBufferedInput.h b/cpp/velox/memory/GlutenDirectBufferedInput.h index edaff5c603d8..3aef323da7bc 100644 --- a/cpp/velox/memory/GlutenDirectBufferedInput.h +++ b/cpp/velox/memory/GlutenDirectBufferedInput.h @@ -29,8 +29,8 @@ class GlutenDirectBufferedInput : public facebook::velox::dwio::common::DirectBu facebook::velox::StringIdLease fileNum, std::shared_ptr tracker, facebook::velox::StringIdLease groupId, - std::shared_ptr ioStats, - std::shared_ptr fsStats, + std::shared_ptr ioStatistics, + std::shared_ptr ioStats, folly::Executor* executor, const facebook::velox::io::ReaderOptions& readerOptions, folly::F14FastMap fileReadOps = {}) @@ -40,8 +40,8 @@ class GlutenDirectBufferedInput : public facebook::velox::dwio::common::DirectBu std::move(fileNum), std::move(tracker), std::move(groupId), + std::move(ioStatistics), std::move(ioStats), - std::move(fsStats), executor, readerOptions, std::move(fileReadOps)) {} diff --git a/cpp/velox/utils/VeloxWriterUtils.cc b/cpp/velox/utils/VeloxWriterUtils.cc index 50e4ca601e16..026418a223c4 100644 --- a/cpp/velox/utils/VeloxWriterUtils.cc +++ b/cpp/velox/utils/VeloxWriterUtils.cc @@ -61,7 +61,7 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_mapsecond; if (parquetGzipWindowSizeStr == kGzipWindowSize4k) { auto codecOptions = std::make_shared(); - codecOptions->window_bits = kGzipWindowBits4k; + codecOptions->windowBits = kGzipWindowBits4k; writeOption->codecOptions = std::move(codecOptions); } } @@ -77,7 +77,7 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_map(); auto it = sparkConfs.find(kParquetZSTDCompressionLevel); auto compressionLevel = it != sparkConfs.end() ? std::stoi(it->second) : kZSTDDefaultCompressionLevel; - codecOptions->compression_level = compressionLevel; + codecOptions->compressionLevel = compressionLevel; writeOption->codecOptions = std::move(codecOptions); } else if (boost::iequals(compressionCodecStr, "uncompressed")) { compressionCodec = CompressionKind::CompressionKind_NONE; diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 1f54431a1458..c395089c7252 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_02_06 -VELOX_ENHANCED_BRANCH=ibm-2026_02_06 +VELOX_BRANCH=dft-2026_02_17 +VELOX_ENHANCED_BRANCH=ibm-2026_02_17 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out index cc20dd33e0b2..92e4a861fa1a 100644 --- a/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out +++ b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out @@ -141,12 +141,12 @@ from -- !query schema struct -- !query output -1 0.5488135024422883 1 -1 0.7151893651681639 2 -2 0.5448831775801376 2 -2 0.6027633705776989 1 -3 0.4236547969336536 1 -3 0.6458941151817286 2 +1 0.5234194256885571 2 +1 0.7604953758285915 1 +2 0.0953472826424725 1 +2 0.3163249920547614 2 +3 0.2710259815484829 2 +3 0.7141011170991605 1 -- !query diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out index b54621f8ec0c..a12e830c1117 100644 --- a/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out @@ -662,5 +662,5 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.5488135024422883 7 +0.7604953758285915 7 1.0 2 diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out index a9d334e7f2b1..b269d40c3566 100644 --- a/gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out +++ b/gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out @@ -7,7 +7,7 @@ SELECT rand(0) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -15,7 +15,7 @@ SELECT rand(cast(3 / 7 AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -23,7 +23,7 @@ SELECT rand(NULL) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -31,7 +31,7 @@ SELECT rand(cast(NULL AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out index cc20dd33e0b2..92e4a861fa1a 100644 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out +++ b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out @@ -141,12 +141,12 @@ from -- !query schema struct -- !query output -1 0.5488135024422883 1 -1 0.7151893651681639 2 -2 0.5448831775801376 2 -2 0.6027633705776989 1 -3 0.4236547969336536 1 -3 0.6458941151817286 2 +1 0.5234194256885571 2 +1 0.7604953758285915 1 +2 0.0953472826424725 1 +2 0.3163249920547614 2 +3 0.2710259815484829 2 +3 0.7141011170991605 1 -- !query diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out index 408b9f9425bf..48b35bf1e0d8 100644 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out @@ -691,7 +691,7 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.5488135024422883 7 +0.7604953758285915 7 1.0 2 diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out index a9d334e7f2b1..b269d40c3566 100644 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out +++ b/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out @@ -7,7 +7,7 @@ SELECT rand(0) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -15,7 +15,7 @@ SELECT rand(cast(3 / 7 AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -23,7 +23,7 @@ SELECT rand(NULL) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -31,7 +31,7 @@ SELECT rand(cast(NULL AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out index 5b8637012e57..bf85afd626af 100644 --- a/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out +++ b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out @@ -208,12 +208,12 @@ from -- !query schema struct -- !query output -1 0.5488135024422883 1 -1 0.7151893651681639 2 -2 0.5448831775801376 2 -2 0.6027633705776989 1 -3 0.4236547969336536 1 -3 0.6458941151817286 2 +1 0.5234194256885571 2 +1 0.7604953758285915 1 +2 0.0953472826424725 1 +2 0.3163249920547614 2 +3 0.2710259815484829 2 +3 0.7141011170991605 1 -- !query select * from data group by a, b, 1 diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out index a4a3f76fa6a7..f56420926050 100644 --- a/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out @@ -907,7 +907,7 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.5488135024422883 7 +0.7604953758285915 7 1.0 2 diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out index f8460c1d437e..dea2c69ba035 100644 --- a/gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out +++ b/gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out @@ -4,7 +4,7 @@ SELECT rand(0) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -12,7 +12,7 @@ SELECT rand(cast(3 / 7 AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -20,7 +20,7 @@ SELECT rand(NULL) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -28,7 +28,7 @@ SELECT rand(cast(NULL AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query diff --git a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by-ordinal.sql.out index b968b4e09fac..0f29c27268c8 100644 --- a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by-ordinal.sql.out +++ b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by-ordinal.sql.out @@ -208,12 +208,12 @@ from -- !query schema struct -- !query output -1 0.5488135024422883 1 -1 0.7151893651681639 2 -2 0.5448831775801376 2 -2 0.6027633705776989 1 -3 0.4236547969336536 1 -3 0.6458941151817286 2 +1 0.5234194256885571 2 +1 0.7604953758285915 1 +2 0.0953472826424725 1 +2 0.3163249920547614 2 +3 0.2710259815484829 2 +3 0.7141011170991605 1 -- !query diff --git a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by.sql.out b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by.sql.out index db0b74cd6ac7..4e3a176ba9c6 100644 --- a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/group-by.sql.out @@ -907,7 +907,7 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.5488135024422883 7 +0.7604953758285915 7 1.0 2 diff --git a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/random.sql.out b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/random.sql.out index 17e6f871b9c5..8a182a0646ca 100644 --- a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/random.sql.out +++ b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/random.sql.out @@ -4,7 +4,7 @@ SELECT rand(0) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -12,7 +12,7 @@ SELECT rand(cast(3 / 7 AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -20,7 +20,7 @@ SELECT rand(NULL) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query @@ -28,7 +28,7 @@ SELECT rand(cast(NULL AS int)) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query diff --git a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/table-valued-functions.sql.out b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/table-valued-functions.sql.out index 0d5675fa6fde..1995e9e87b3a 100644 --- a/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/table-valued-functions.sql.out +++ b/gluten-ut/spark35/src/test/resources/backends-velox/sql-tests/results/table-valued-functions.sql.out @@ -247,7 +247,7 @@ select * from explode(array(rand(0))) -- !query schema struct -- !query output -0.5488135024422883 +0.7604953758285915 -- !query diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala index b3d51e802985..2050237e0d9e 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala @@ -22,7 +22,7 @@ class GlutenGeneratorFunctionSuite extends GeneratorFunctionSuite with GlutenSQL testGluten("SPARK-45171: Handle evaluated nondeterministic expression") { withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") { val df = sql("select explode(array(rand(0)))") - checkAnswer(df, Row(0.5488135024422883)) + checkAnswer(df, Row(0.7604953758285915)) } } } diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out b/gluten-ut/spark40/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out index 45a19ba2c3f1..5eb1f6689ba9 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out +++ b/gluten-ut/spark40/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out @@ -139,11 +139,11 @@ from -- !query schema struct -- !query output -1 0.5234194256885571 1 +1 0.5234194256885571 2 1 0.7604953758285915 1 2 0.0953472826424725 1 -2 0.3163249920547614 1 -3 0.2710259815484829 1 +2 0.3163249920547614 2 +3 0.2710259815484829 2 3 0.7141011170991605 1 diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index f5c9d22db6ac..6e94d4cc0ea4 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -742,8 +742,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLWindowFunctionSuite] .exclude("test with low buffer spill threshold") enableSuite[GlutenTakeOrderedAndProjectSuite] - // The results of rand() differ between vanilla spark and velox. - .exclude("SPARK-47104: Non-deterministic expressions in projection") enableSuite[GlutenSessionExtensionSuite] enableSuite[TestFileSourceScanExecTransformer] enableSuite[GlutenBucketedReadWithoutHiveSupportSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala index b3d51e802985..2050237e0d9e 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala @@ -22,7 +22,7 @@ class GlutenGeneratorFunctionSuite extends GeneratorFunctionSuite with GlutenSQL testGluten("SPARK-45171: Handle evaluated nondeterministic expression") { withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") { val df = sql("select explode(array(rand(0)))") - checkAnswer(df, Row(0.5488135024422883)) + checkAnswer(df, Row(0.7604953758285915)) } } } diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala index 2731e0547111..bc231e52adc3 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala @@ -16,50 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} -import org.apache.spark.sql.catalyst.expressions.{Alias, Literal, Rand} -import org.apache.spark.sql.types.{IntegerType, StructType} +import org.apache.spark.sql.GlutenSQLTestsBaseTrait class GlutenTakeOrderedAndProjectSuite extends TakeOrderedAndProjectSuite - with GlutenSQLTestsBaseTrait { - - private def noOpFilter(plan: SparkPlan): SparkPlan = FilterExec(Literal(true), plan) - - testGluten("SPARK-47104: Non-deterministic expressions in projection") { - val expected = (input: SparkPlan) => { - GlobalLimitExec(limit, LocalLimitExec(limit, SortExec(sortOrder, true, input))) - } - val schema = StructType.fromDDL("a int, b int, c double") - val rdd = sparkContext.parallelize( - Seq( - Row(1, 2, 0.6027633705776989d), - Row(2, 3, 0.7151893651681639d), - Row(3, 4, 0.5488135024422883d)), - 1) - val df = spark.createDataFrame(rdd, schema) - val projection = df.queryExecution.sparkPlan.output.take(2) :+ - Alias(Rand(Literal(0, IntegerType)), "_uuid")() - - // test executeCollect - checkThatPlansAgree( - df, - input => - TakeOrderedAndProjectExec(limit, sortOrder, projection, SortExec(sortOrder, false, input)), - input => expected(input), - sortAnswers = false) - - // test doExecute - checkThatPlansAgree( - df, - input => - noOpFilter( - TakeOrderedAndProjectExec( - limit, - sortOrder, - projection, - SortExec(sortOrder, false, input))), - input => expected(input), - sortAnswers = false) - } -} + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out index 45a19ba2c3f1..5eb1f6689ba9 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out +++ b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/udaf/udaf-group-by-ordinal.sql.out @@ -139,11 +139,11 @@ from -- !query schema struct -- !query output -1 0.5234194256885571 1 +1 0.5234194256885571 2 1 0.7604953758285915 1 2 0.0953472826424725 1 -2 0.3163249920547614 1 -3 0.2710259815484829 1 +2 0.3163249920547614 2 +3 0.2710259815484829 2 3 0.7141011170991605 1 diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e8f8dfa76253..2b9e4555d716 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -708,8 +708,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLWindowFunctionSuite] .exclude("test with low buffer spill threshold") enableSuite[GlutenTakeOrderedAndProjectSuite] - // The results of rand() differ between vanilla spark and velox. - .exclude("SPARK-47104: Non-deterministic expressions in projection") enableSuite[GlutenSessionExtensionSuite] enableSuite[TestFileSourceScanExecTransformer] enableSuite[GlutenBucketedReadWithoutHiveSupportSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala index b3d51e802985..2050237e0d9e 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala @@ -22,7 +22,7 @@ class GlutenGeneratorFunctionSuite extends GeneratorFunctionSuite with GlutenSQL testGluten("SPARK-45171: Handle evaluated nondeterministic expression") { withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") { val df = sql("select explode(array(rand(0)))") - checkAnswer(df, Row(0.5488135024422883)) + checkAnswer(df, Row(0.7604953758285915)) } } } diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala index 2731e0547111..bc231e52adc3 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala @@ -16,50 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} -import org.apache.spark.sql.catalyst.expressions.{Alias, Literal, Rand} -import org.apache.spark.sql.types.{IntegerType, StructType} +import org.apache.spark.sql.GlutenSQLTestsBaseTrait class GlutenTakeOrderedAndProjectSuite extends TakeOrderedAndProjectSuite - with GlutenSQLTestsBaseTrait { - - private def noOpFilter(plan: SparkPlan): SparkPlan = FilterExec(Literal(true), plan) - - testGluten("SPARK-47104: Non-deterministic expressions in projection") { - val expected = (input: SparkPlan) => { - GlobalLimitExec(limit, LocalLimitExec(limit, SortExec(sortOrder, true, input))) - } - val schema = StructType.fromDDL("a int, b int, c double") - val rdd = sparkContext.parallelize( - Seq( - Row(1, 2, 0.6027633705776989d), - Row(2, 3, 0.7151893651681639d), - Row(3, 4, 0.5488135024422883d)), - 1) - val df = spark.createDataFrame(rdd, schema) - val projection = df.queryExecution.sparkPlan.output.take(2) :+ - Alias(Rand(Literal(0, IntegerType)), "_uuid")() - - // test executeCollect - checkThatPlansAgree( - df, - input => - TakeOrderedAndProjectExec(limit, sortOrder, projection, SortExec(sortOrder, false, input)), - input => expected(input), - sortAnswers = false) - - // test doExecute - checkThatPlansAgree( - df, - input => - noOpFilter( - TakeOrderedAndProjectExec( - limit, - sortOrder, - projection, - SortExec(sortOrder, false, input))), - input => expected(input), - sortAnswers = false) - } -} + with GlutenSQLTestsBaseTrait {}