From 75b70f33feff9d913b97ad22b01ccfc35d681e74 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 21 Nov 2025 16:27:23 -0700 Subject: [PATCH 01/10] fix --- .../src/math_funcs/internal/make_decimal.rs | 27 +++++++++------- .../comet/serde/decimalExpressions.scala | 8 +++++ .../apache/comet/CometExpressionSuite.scala | 32 +++++++++++++++++-- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/native/spark-expr/src/math_funcs/internal/make_decimal.rs b/native/spark-expr/src/math_funcs/internal/make_decimal.rs index 8feba54f5a..736351d009 100644 --- a/native/spark-expr/src/math_funcs/internal/make_decimal.rs +++ b/native/spark-expr/src/math_funcs/internal/make_decimal.rs @@ -16,7 +16,7 @@ // under the License. use crate::math_funcs::utils::get_precision_scale; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Int32Type}; use arrow::{ array::{AsArray, Decimal128Builder}, datatypes::{validate_decimal_precision, Int64Type}, @@ -40,18 +40,21 @@ pub fn spark_make_decimal( ))), sv => internal_err!("Expected Int64 but found {sv:?}"), }, - ColumnarValue::Array(a) => { - let arr = a.as_primitive::(); - let mut result = Decimal128Builder::new(); - for v in arr.into_iter() { - result.append_option(long_to_decimal(&v, precision)) - } - let result_type = DataType::Decimal128(precision, scale); + ColumnarValue::Array(a) => match a.data_type() { + DataType::Int64 => { + let arr = a.as_primitive::(); + let mut result = Decimal128Builder::new(); + for v in arr.into_iter() { + result.append_option(long_to_decimal(&v, precision)) + } + let result_type = DataType::Decimal128(precision, scale); - Ok(ColumnarValue::Array(Arc::new( - result.finish().with_data_type(result_type), - ))) - } + Ok(ColumnarValue::Array(Arc::new( + result.finish().with_data_type(result_type), + ))) + } + av => internal_err!("Expected Int64 but found {av:?}"), + }, } } diff --git a/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala b/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala index c606d1ac5b..9a019b9035 100644 --- a/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala +++ b/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala @@ -38,6 +38,14 @@ object CometUnscaledValue extends CometExpressionSerde[UnscaledValue] { } object CometMakeDecimal extends CometExpressionSerde[MakeDecimal] { + + override def getSupportLevel(expr: MakeDecimal): SupportLevel = { + expr.child.dataType match { + case _: LongType => Compatible() + case other => Unsupported(Some(s"Unsupported input data type: $other")) + } + } + override def convert( expr: MakeDecimal, inputs: Seq[Attribute], diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index b0c718a2b6..afd6561b66 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -27,8 +27,8 @@ import org.scalactic.source.Position import org.scalatest.Tag import org.apache.hadoop.fs.Path -import org.apache.spark.sql.{CometTestBase, DataFrame, Row} -import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, FromUnixTime, Literal, TruncDate, TruncTimestamp} +import org.apache.spark.sql.{Column, CometTestBase, DataFrame, Row} +import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, FromUnixTime, Literal, MakeDecimal, TruncDate, TruncTimestamp} import org.apache.spark.sql.catalyst.optimizer.SimplifyExtractValueOps import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometProjectExec} import org.apache.spark.sql.execution.{InputAdapter, ProjectExec, SparkPlan, WholeStageCodegenExec} @@ -3187,4 +3187,32 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { CometConcat.unsupportedReason) } } + + // https://github.com/apache/datafusion-comet/issues/2813 + test("make decimal using DataFrame API") { + + sql("create table t1 using parquet as select 123456 as c1 from range(1)") + + withSQLConf( + CometConf.COMET_EXEC_ENABLED.key -> "true", + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true", + "spark.sql.ansi.enabled" -> "false", + "spark.sql.adaptive.enabled" -> "false", + "spark.comet.expression.Sum.allowIncompatible" -> "true", + CometConf.COMET_NATIVE_SCAN_IMPL.key -> "native_iceberg_compat", + "spark.sql.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { + + val df = sql("select * from t1") + val makeDecimalExpr = MakeDecimal(df.col("c1").expr, 3, 0) + val makeDecimalColumn = new Column(makeDecimalExpr) + val df1 = df.withColumn("result", makeDecimalColumn) + df1.explain("formatted") + + checkSparkAnswerAndOperator(df1) + } + + } + } From 7803ed562c4097d24a3f75c6a81e6f0e5e4fb36f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 21 Nov 2025 16:28:47 -0700 Subject: [PATCH 02/10] remove unused import --- native/spark-expr/src/math_funcs/internal/make_decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/spark-expr/src/math_funcs/internal/make_decimal.rs b/native/spark-expr/src/math_funcs/internal/make_decimal.rs index 736351d009..3383175202 100644 --- a/native/spark-expr/src/math_funcs/internal/make_decimal.rs +++ b/native/spark-expr/src/math_funcs/internal/make_decimal.rs @@ -16,7 +16,7 @@ // under the License. use crate::math_funcs::utils::get_precision_scale; -use arrow::datatypes::{DataType, Int32Type}; +use arrow::datatypes::DataType; use arrow::{ array::{AsArray, Decimal128Builder}, datatypes::{validate_decimal_precision, Int64Type}, From a71832d9abf36271359ca60028cb532ad7523426 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 21 Nov 2025 16:33:12 -0700 Subject: [PATCH 03/10] test --- .../apache/comet/CometExpressionSuite.scala | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index afd6561b66..967ec5eee0 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -20,12 +20,9 @@ package org.apache.comet import java.time.{Duration, Period} - import scala.util.Random - import org.scalactic.source.Position import org.scalatest.Tag - import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Column, CometTestBase, DataFrame, Row} import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, FromUnixTime, Literal, MakeDecimal, TruncDate, TruncTimestamp} @@ -37,10 +34,10 @@ import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.SESSION_LOCAL_TIMEZONE import org.apache.spark.sql.types._ - import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.CometConcat import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} +import org.apache.spark.sql.catalyst.expressions.aggregate.Sum class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ @@ -3191,28 +3188,28 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { // https://github.com/apache/datafusion-comet/issues/2813 test("make decimal using DataFrame API") { - sql("create table t1 using parquet as select 123456 as c1 from range(1)") + withTable("t1") { + sql("create table t1 using parquet as select 123456 as c1 from range(1)") - withSQLConf( - CometConf.COMET_EXEC_ENABLED.key -> "true", - SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", - CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true", - "spark.sql.ansi.enabled" -> "false", - "spark.sql.adaptive.enabled" -> "false", - "spark.comet.expression.Sum.allowIncompatible" -> "true", - CometConf.COMET_NATIVE_SCAN_IMPL.key -> "native_iceberg_compat", - "spark.sql.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { + withSQLConf( + CometConf.COMET_EXEC_ENABLED.key -> "true", + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true", + SQLConf.ANSI_ENABLED.key -> "false", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + CometConf.getExprAllowIncompatConfigKey(classOf[Sum]) -> "true", + CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_ICEBERG_COMPAT, + SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { - val df = sql("select * from t1") - val makeDecimalExpr = MakeDecimal(df.col("c1").expr, 3, 0) - val makeDecimalColumn = new Column(makeDecimalExpr) - val df1 = df.withColumn("result", makeDecimalColumn) - df1.explain("formatted") + val df = sql("select * from t1") + val makeDecimalExpr = MakeDecimal(df.col("c1").expr, 3, 0) + val makeDecimalColumn = new Column(makeDecimalExpr) + val df1 = df.withColumn("result", makeDecimalColumn) - checkSparkAnswerAndOperator(df1) + checkSparkAnswerAndFallbackReason(df1, "Unsupported input data type: IntegerType") + } } - } } From e4df8cb4dd7b800c7145196cabb56818cf0f9658 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 21 Nov 2025 16:34:37 -0700 Subject: [PATCH 04/10] scalastyle --- .../test/scala/org/apache/comet/CometExpressionSuite.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 967ec5eee0..80686941a6 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -20,12 +20,16 @@ package org.apache.comet import java.time.{Duration, Period} + import scala.util.Random + import org.scalactic.source.Position import org.scalatest.Tag + import org.apache.hadoop.fs.Path import org.apache.spark.sql.{Column, CometTestBase, DataFrame, Row} import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, FromUnixTime, Literal, MakeDecimal, TruncDate, TruncTimestamp} +import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.spark.sql.catalyst.optimizer.SimplifyExtractValueOps import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometProjectExec} import org.apache.spark.sql.execution.{InputAdapter, ProjectExec, SparkPlan, WholeStageCodegenExec} @@ -34,10 +38,10 @@ import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.SESSION_LOCAL_TIMEZONE import org.apache.spark.sql.types._ + import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.CometConcat import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} -import org.apache.spark.sql.catalyst.expressions.aggregate.Sum class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ From c8394a32cf71959c1882546dc09a1605bb7f9684 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 21 Nov 2025 17:22:42 -0700 Subject: [PATCH 05/10] shims --- .../test/scala/org/apache/comet/CometExpressionSuite.scala | 4 +--- .../spark-3.4/org/apache/spark/sql/ShimCometTestBase.scala | 6 +++++- .../spark-3.5/org/apache/spark/sql/ShimCometTestBase.scala | 6 +++++- .../spark-4.0/org/apache/spark/sql/ShimCometTestBase.scala | 6 +++++- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 80686941a6..303b380a56 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -3191,7 +3191,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { // https://github.com/apache/datafusion-comet/issues/2813 test("make decimal using DataFrame API") { - withTable("t1") { sql("create table t1 using parquet as select 123456 as c1 from range(1)") @@ -3207,8 +3206,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { val df = sql("select * from t1") - val makeDecimalExpr = MakeDecimal(df.col("c1").expr, 3, 0) - val makeDecimalColumn = new Column(makeDecimalExpr) + val makeDecimalColumn = createMakeDecimalColumn(df.col("c1").expr, 3, 0) val df1 = df.withColumn("result", makeDecimalColumn) checkSparkAnswerAndFallbackReason(df1, "Unsupported input data type: IntegerType") diff --git a/spark/src/test/spark-3.4/org/apache/spark/sql/ShimCometTestBase.scala b/spark/src/test/spark-3.4/org/apache/spark/sql/ShimCometTestBase.scala index b8ecfacb31..a7dfb42645 100644 --- a/spark/src/test/spark-3.4/org/apache/spark/sql/ShimCometTestBase.scala +++ b/spark/src/test/spark-3.4/org/apache/spark/sql/ShimCometTestBase.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.{Expression, MakeDecimal} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan trait ShimCometTestBase { @@ -46,4 +46,8 @@ trait ShimCometTestBase { def extractLogicalPlan(df: DataFrame): LogicalPlan = { df.logicalPlan } + + def createMakeDecimalColumn(child: Expression, precision: Int, scale: Int): Column = { + new Column(MakeDecimal(child, precision, scale)) + } } diff --git a/spark/src/test/spark-3.5/org/apache/spark/sql/ShimCometTestBase.scala b/spark/src/test/spark-3.5/org/apache/spark/sql/ShimCometTestBase.scala index f2b4195565..7f22494ad2 100644 --- a/spark/src/test/spark-3.5/org/apache/spark/sql/ShimCometTestBase.scala +++ b/spark/src/test/spark-3.5/org/apache/spark/sql/ShimCometTestBase.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.{Expression, MakeDecimal} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan trait ShimCometTestBase { @@ -47,4 +47,8 @@ trait ShimCometTestBase { df.logicalPlan } + def createMakeDecimalColumn(child: Expression, precision: Int, scale: Int): Column = { + new Column(MakeDecimal(child, precision, scale)) + } + } diff --git a/spark/src/test/spark-4.0/org/apache/spark/sql/ShimCometTestBase.scala b/spark/src/test/spark-4.0/org/apache/spark/sql/ShimCometTestBase.scala index 8fb2e69705..5ad4543220 100644 --- a/spark/src/test/spark-4.0/org/apache/spark/sql/ShimCometTestBase.scala +++ b/spark/src/test/spark-4.0/org/apache/spark/sql/ShimCometTestBase.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.{Expression, MakeDecimal} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.classic.{Dataset, ExpressionColumnNode, SparkSession} @@ -47,4 +47,8 @@ trait ShimCometTestBase { def extractLogicalPlan(df: DataFrame): LogicalPlan = { df.queryExecution.analyzed } + + def createMakeDecimalColumn(child: Expression, precision: Int, scale: Int): Column = { + new Column(ExpressionColumnNode.apply(MakeDecimal(child, precision, scale, true))) + } } From 316387587303c8feca6e14fc9719869aedc58aa6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 21 Nov 2025 17:26:22 -0700 Subject: [PATCH 06/10] scalastyle --- .../test/scala/org/apache/comet/CometExpressionSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 303b380a56..b8d0864749 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -27,8 +27,8 @@ import org.scalactic.source.Position import org.scalatest.Tag import org.apache.hadoop.fs.Path -import org.apache.spark.sql.{Column, CometTestBase, DataFrame, Row} -import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, FromUnixTime, Literal, MakeDecimal, TruncDate, TruncTimestamp} +import org.apache.spark.sql.{CometTestBase, DataFrame, Row} +import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, FromUnixTime, Literal, TruncDate, TruncTimestamp} import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.spark.sql.catalyst.optimizer.SimplifyExtractValueOps import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometProjectExec} From d41366ff66e6b506dfe201ce598125655d79398d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 4 Dec 2025 14:18:51 -0700 Subject: [PATCH 07/10] Update spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala Co-authored-by: Martin Grigorov --- .../main/scala/org/apache/comet/serde/decimalExpressions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala b/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala index 9a019b9035..880f01742b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala +++ b/spark/src/main/scala/org/apache/comet/serde/decimalExpressions.scala @@ -41,7 +41,7 @@ object CometMakeDecimal extends CometExpressionSerde[MakeDecimal] { override def getSupportLevel(expr: MakeDecimal): SupportLevel = { expr.child.dataType match { - case _: LongType => Compatible() + case LongType => Compatible() case other => Unsupported(Some(s"Unsupported input data type: $other")) } } From e7e67f602e1c6f712d1ecc2e902a956d8576575d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 4 Dec 2025 14:24:51 -0700 Subject: [PATCH 08/10] add test for long and remove some configs --- .../apache/comet/CometExpressionSuite.scala | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 16ef50152a..b55d338161 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -3114,18 +3114,14 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } // https://github.com/apache/datafusion-comet/issues/2813 - test("make decimal using DataFrame API") { + test("make decimal using DataFrame API - integer") { withTable("t1") { sql("create table t1 using parquet as select 123456 as c1 from range(1)") withSQLConf( - CometConf.COMET_EXEC_ENABLED.key -> "true", SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", - CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true", SQLConf.ANSI_ENABLED.key -> "false", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - CometConf.getExprAllowIncompatConfigKey(classOf[Sum]) -> "true", CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_ICEBERG_COMPAT, SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { @@ -3138,4 +3134,24 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("make decimal using DataFrame API - long") { + withTable("t1") { + sql("create table t1 using parquet as select cast(123456 as long) as c1 from range(1)") + + withSQLConf( + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + SQLConf.ANSI_ENABLED.key -> "false", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_ICEBERG_COMPAT, + SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { + + val df = sql("select * from t1") + val makeDecimalColumn = createMakeDecimalColumn(df.col("c1").expr, 3, 0) + val df1 = df.withColumn("result", makeDecimalColumn) + + checkSparkAnswerAndOperator(df1) + } + } + } + } From ccae240b559e04aec0c17e275f5c681c75daac3a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 4 Dec 2025 14:25:39 -0700 Subject: [PATCH 09/10] add test for long and remove some configs --- .../src/test/scala/org/apache/comet/CometExpressionSuite.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index b55d338161..64cfa5568f 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -3122,7 +3122,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", SQLConf.ANSI_ENABLED.key -> "false", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_ICEBERG_COMPAT, SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { val df = sql("select * from t1") @@ -3142,7 +3141,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", SQLConf.ANSI_ENABLED.key -> "false", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_ICEBERG_COMPAT, SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { val df = sql("select * from t1") From 726ca2140317bfb4489fb9df32c71c15c6f7a08b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 4 Dec 2025 15:00:17 -0700 Subject: [PATCH 10/10] scalastyle --- spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 64cfa5568f..c6d505691f 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -29,7 +29,6 @@ import org.scalatest.Tag import org.apache.hadoop.fs.Path import org.apache.spark.sql.{CometTestBase, DataFrame, Row} import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, FromUnixTime, Literal, TruncDate, TruncTimestamp} -import org.apache.spark.sql.catalyst.expressions.aggregate.Sum import org.apache.spark.sql.catalyst.optimizer.SimplifyExtractValueOps import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometProjectExec} import org.apache.spark.sql.execution.{InputAdapter, ProjectExec, SparkPlan, WholeStageCodegenExec}