Skip to content

[GLUTEN-11402][VL] Fix decimal partition key serialization to preserve scale #11618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhouyuan merged 1 commit into from
Mar 5, 2026
Merged

[GLUTEN-11402][VL] Fix decimal partition key serialization to preserve scale #11618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ class VeloxIteratorApi extends IteratorApi with Logging {
case _: DateType =>
DateFormatter.apply().format(pv.asInstanceOf[Integer])
case _: DecimalType =>
pv.asInstanceOf[Decimal].toJavaBigInteger.toString
pv.asInstanceOf[Decimal].toJavaBigDecimal.unscaledValue().toString
Copy link
Contributor

@beliefer beliefer Mar 4, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you explain why decimal partition keys are not supported?

Copy link
Contributor Author

@acvictor acvictor Mar 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not that it's unsupported but rather results in incorrect casting (see #11618). The change is needed because Decimal.toJavaBigInteger truncates the fractional part, producing an incorrect unscaled value. For example, a Decimal("100.1") with scale=1 would serialize as "100" (the truncated BigInteger) instead of "1001" (the correct unscaled representation). This causes Velox reader to misinterpret decimal partition values, returning wrong query results.

case _: TimestampType =>
TimestampFormatter
.getFractionFormatter(ZoneOffset.UTC)
Expand Down
5 changes: 0 additions & 5 deletions gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat

import org.apache.spark.Partition
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.DecimalType

import com.google.protobuf.StringValue
import io.substrait.proto.NamedStruct
Expand Down Expand Up @@ -139,10 +138,6 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource
return validationResult
}

if (getPartitionSchema.fields.exists(_.dataType.isInstanceOf[DecimalType])) {
return ValidationResult.failed(s"Unsupported decimal partition column in native scan.")
}

val substraitContext = new SubstraitContext
val relNode = transform(substraitContext).root

Expand Down
58 changes: 57 additions & 1 deletion gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ class SQLQuerySuite extends WholeStageTransformerSuite {
val df = spark.createDataFrame(data).toDF("key", "value")
df.createOrReplaceTempView("src")

// decimal
// decimal with fractional truncation
sql("create table dynparttest2 (value int) partitioned by (pdec decimal(5, 1))")
sql("""
|insert into table dynparttest2 partition(pdec)
Expand All @@ -68,6 +68,62 @@ class SQLQuerySuite extends WholeStageTransformerSuite {
sql("select * from dynparttest2"),
Seq(Row(6, new java.math.BigDecimal("100.1"))))
}

// small decimal with scale > 0
withTable("dynparttest_small") {
sql("create table dynparttest_small (value int) partitioned by (pdec decimal(3, 2))")
sql("""
|insert into table dynparttest_small partition(pdec)
| select count(*), cast('1.23' as decimal(3, 2)) as pdec from src
""".stripMargin)
checkAnswer(
sql("select * from dynparttest_small"),
Seq(Row(6, new java.math.BigDecimal("1.23"))))
}

// zero scale with no fractional part
withTable("dynparttest_zero_scale") {
sql("create table dynparttest_zero_scale (value int) partitioned by (pdec decimal(10, 0))")
sql("""
|insert into table dynparttest_zero_scale partition(pdec)
| select count(*), cast('42' as decimal(10, 0)) as pdec from src
""".stripMargin)
checkAnswer(
sql("select * from dynparttest_zero_scale"),
Seq(Row(6, new java.math.BigDecimal("42"))))
}

// negative value with scale
withTable("dynparttest_neg") {
sql("create table dynparttest_neg (value int) partitioned by (pdec decimal(5, 2))")
sql("""
|insert into table dynparttest_neg partition(pdec)
| select count(*), cast('-3.14' as decimal(5, 2)) as pdec from src
""".stripMargin)
checkAnswer(
sql("select * from dynparttest_neg"),
Seq(Row(6, new java.math.BigDecimal("-3.14"))))
}

// multiple distinct partition values
withTable("dynparttest_multi") {
sql("create table dynparttest_multi (value int) partitioned by (pdec decimal(4, 1))")
sql("""
|insert into table dynparttest_multi partition(pdec)
| select count(*), cast('10.5' as decimal(4, 1)) as pdec from src
""".stripMargin)
sql("""
|insert into table dynparttest_multi partition(pdec)
| select count(*), cast('20.3' as decimal(4, 1)) as pdec from src
""".stripMargin)
checkAnswer(
sql("select * from dynparttest_multi order by pdec"),
Seq(Row(6, new java.math.BigDecimal("10.5")), Row(6, new java.math.BigDecimal("20.3"))))
// partition pruning
checkAnswer(
sql("select * from dynparttest_multi where pdec = 10.5"),
Seq(Row(6, new java.math.BigDecimal("10.5"))))
}
}
}

Expand Down
Loading