From efe958ed0fa6b0ba6dac010e8753c76ef12c689e Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 25 Sep 2025 19:09:20 +0200 Subject: [PATCH] [VL] Delta: Fix DeleteSQLNameColumnMappingSuite --- .../spark/sql/delta/DeleteSQLSuite.scala | 5 ----- .../extension/DeltaPostTransformRules.scala | 20 +++++++++++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala b/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala index 950c2269d7b7..7bd6a59b7b91 100644 --- a/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala +++ b/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala @@ -20,8 +20,6 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaExcludedTestMixin, DeltaSQLCommandTest} -import org.scalatest.Ignore - // spotless:off class DeleteSQLSuite extends DeleteSuiteBase with DeltaExcludedTestMixin @@ -95,9 +93,6 @@ class DeleteSQLSuite extends DeleteSuiteBase } } -// FIXME: Enable the test. -// Skipping as function input_file_name doesn't get correctly resolved. -@Ignore class DeleteSQLNameColumnMappingSuite extends DeleteSQLSuite with DeltaColumnMappingEnableNameMode { diff --git a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala index 83af341a78e5..156fdf791a8c 100644 --- a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala +++ b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala @@ -31,7 +31,7 @@ import scala.collection.mutable.ListBuffer object DeltaPostTransformRules { def rules: Seq[Rule[SparkPlan]] = - RemoveTransitions :: columnMappingRule :: pushDownInputFileExprRule :: Nil + RemoveTransitions :: pushDownInputFileExprRule :: columnMappingRule :: Nil private val COLUMN_MAPPING_RULE_TAG: TreeNodeTag[String] = TreeNodeTag[String]("org.apache.gluten.delta.column.mapping") @@ -76,6 +76,16 @@ object DeltaPostTransformRules { } } + private def isInputFileRelatedAttribute(attr: Attribute): Boolean = { + attr match { + case AttributeReference(name, _, _, _) => + Seq(InputFileName(), InputFileBlockStart(), InputFileBlockLength()) + .map(_.prettyName) + .contains(name) + case _ => false + } + } + private[gluten] def containsIncrementMetricExpr(expr: Expression): Boolean = { expr match { case e if e.prettyName == "increment_metric" => true @@ -108,12 +118,14 @@ object DeltaPostTransformRules { val originColumnNames = ListBuffer.empty[String] val transformedAttrs = ListBuffer.empty[Attribute] def mapAttribute(attr: Attribute) = { - val newAttr = if (!plan.isMetadataColumn(attr)) { + val newAttr = if (plan.isMetadataColumn(attr)) { + attr + } else if (isInputFileRelatedAttribute(attr)) { + attr + } else { DeltaColumnMapping .createPhysicalAttributes(Seq(attr), fmt.referenceSchema, fmt.columnMappingMode) .head - } else { - attr } if (!originColumnNames.contains(attr.name)) { transformedAttrs += newAttr