From 55820f977f1b4040fab045a1a05168a24dccade0 Mon Sep 17 00:00:00 2001 From: Ankita Victor-Levi Date: Sun, 15 Feb 2026 20:24:01 +0530 Subject: [PATCH] Fix decimal partition key serialization --- .../backendsapi/velox/VeloxIteratorApi.scala | 2 +- .../execution/BasicScanExecTransformer.scala | 5 -- .../org/apache/gluten/sql/SQLQuerySuite.scala | 58 ++++++++++++++++++- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 668e60b20542..4c379543085f 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -161,7 +161,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { case _: DateType => DateFormatter.apply().format(pv.asInstanceOf[Integer]) case _: DecimalType => - pv.asInstanceOf[Decimal].toJavaBigInteger.toString + pv.asInstanceOf[Decimal].toJavaBigDecimal.unscaledValue().toString case _: TimestampType => TimestampFormatter .getFractionFormatter(ZoneOffset.UTC) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index f8eb052a9351..3b142fecb952 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -27,7 +27,6 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.spark.Partition import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.types.DecimalType import com.google.protobuf.StringValue import io.substrait.proto.NamedStruct @@ -139,10 +138,6 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource return validationResult } - if (getPartitionSchema.fields.exists(_.dataType.isInstanceOf[DecimalType])) { - return ValidationResult.failed(s"Unsupported decimal partition column in native scan.") - } - val substraitContext = new SubstraitContext val relNode = transform(substraitContext).root diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala index 52a3fe87df60..700d2813e449 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala @@ -58,7 +58,7 @@ class SQLQuerySuite extends WholeStageTransformerSuite { val df = spark.createDataFrame(data).toDF("key", "value") df.createOrReplaceTempView("src") - // decimal + // decimal with fractional truncation sql("create table dynparttest2 (value int) partitioned by (pdec decimal(5, 1))") sql(""" |insert into table dynparttest2 partition(pdec) @@ -68,6 +68,62 @@ class SQLQuerySuite extends WholeStageTransformerSuite { sql("select * from dynparttest2"), Seq(Row(6, new java.math.BigDecimal("100.1")))) } + + // small decimal with scale > 0 + withTable("dynparttest_small") { + sql("create table dynparttest_small (value int) partitioned by (pdec decimal(3, 2))") + sql(""" + |insert into table dynparttest_small partition(pdec) + | select count(*), cast('1.23' as decimal(3, 2)) as pdec from src + """.stripMargin) + checkAnswer( + sql("select * from dynparttest_small"), + Seq(Row(6, new java.math.BigDecimal("1.23")))) + } + + // zero scale with no fractional part + withTable("dynparttest_zero_scale") { + sql("create table dynparttest_zero_scale (value int) partitioned by (pdec decimal(10, 0))") + sql(""" + |insert into table dynparttest_zero_scale partition(pdec) + | select count(*), cast('42' as decimal(10, 0)) as pdec from src + """.stripMargin) + checkAnswer( + sql("select * from dynparttest_zero_scale"), + Seq(Row(6, new java.math.BigDecimal("42")))) + } + + // negative value with scale + withTable("dynparttest_neg") { + sql("create table dynparttest_neg (value int) partitioned by (pdec decimal(5, 2))") + sql(""" + |insert into table dynparttest_neg partition(pdec) + | select count(*), cast('-3.14' as decimal(5, 2)) as pdec from src + """.stripMargin) + checkAnswer( + sql("select * from dynparttest_neg"), + Seq(Row(6, new java.math.BigDecimal("-3.14")))) + } + + // multiple distinct partition values + withTable("dynparttest_multi") { + sql("create table dynparttest_multi (value int) partitioned by (pdec decimal(4, 1))") + sql(""" + |insert into table dynparttest_multi partition(pdec) + | select count(*), cast('10.5' as decimal(4, 1)) as pdec from src + """.stripMargin) + sql(""" + |insert into table dynparttest_multi partition(pdec) + | select count(*), cast('20.3' as decimal(4, 1)) as pdec from src + """.stripMargin) + checkAnswer( + sql("select * from dynparttest_multi order by pdec"), + Seq(Row(6, new java.math.BigDecimal("10.5")), Row(6, new java.math.BigDecimal("20.3")))) + // partition pruning + checkAnswer( + sql("select * from dynparttest_multi where pdec = 10.5"), + Seq(Row(6, new java.math.BigDecimal("10.5")))) + } } }