From f445e5540045adb0fc3804ac4ee834619b08f665 Mon Sep 17 00:00:00 2001 From: QCLyu Date: Mon, 9 Mar 2026 21:29:34 -0700 Subject: [PATCH 1/6] remove lteSpark32 usage --- .../columnar/transition/Convention.scala | 16 +--------------- .../adaptive/GlutenCostEvaluator.scala | 17 ++++++----------- .../org/apache/spark/util/SparkPlanUtil.scala | 4 ---- .../org/apache/spark/util/SparkTaskUtil.scala | 17 ----------------- 4 files changed, 7 insertions(+), 47 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala index a00e2b510645..2fd378b4a9fa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala @@ -17,7 +17,6 @@ package org.apache.gluten.extension.columnar.transition import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan} -import org.apache.spark.util.SparkVersionUtil import scala.collection.mutable @@ -171,20 +170,7 @@ object Convention { this: SparkPlan => final override def rowType(): RowType = { - if (SparkVersionUtil.lteSpark32) { - // It's known that in Spark 3.2, one Spark plan node is considered either only having - // row-based support or only having columnar support at a time. - // Hence, if the plan supports columnar output, we'd disable its row-based support. - // The same for the opposite. - if (supportsColumnar) { - Convention.RowType.None - } else { - assert(rowType0() != Convention.RowType.None) - rowType0() - } - } else { - rowType0() - } + rowType0() } def rowType0(): RowType diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index f53a80639af0..957230da3903 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenCoreConfig import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.util.{SparkVersionUtil, Utils} +import org.apache.spark.util.Utils /** * This [[CostEvaluator]] is to force use the new physical plan when cost is equal. @@ -30,16 +30,11 @@ import org.apache.spark.util.{SparkVersionUtil, Utils} case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { private val vanillaCostEvaluator: CostEvaluator = { - if (SparkVersionUtil.lteSpark32) { - val clazz = Utils.classForName("org.apache.spark.sql.execution.adaptive.SimpleCostEvaluator$") - clazz.getDeclaredField("MODULE$").get(null).asInstanceOf[CostEvaluator] - } else { - val forceOptimizeSkewedJoin = - conf.getConfString("spark.sql.adaptive.forceOptimizeSkewedJoin").toBoolean - val clazz = Utils.classForName("org.apache.spark.sql.execution.adaptive.SimpleCostEvaluator") - val ctor = clazz.getConstructor(classOf[Boolean]) - ctor.newInstance(forceOptimizeSkewedJoin.asInstanceOf[Object]).asInstanceOf[CostEvaluator] - } + val forceOptimizeSkewedJoin = + conf.getConfString("spark.sql.adaptive.forceOptimizeSkewedJoin").toBoolean + val clazz = Utils.classForName("org.apache.spark.sql.execution.adaptive.SimpleCostEvaluator") + val ctor = clazz.getConstructor(classOf[Boolean]) + ctor.newInstance(forceOptimizeSkewedJoin.asInstanceOf[Object]).asInstanceOf[CostEvaluator] } override def evaluateCost(plan: SparkPlan): Cost = { diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala index 5ce2addafbf4..9d83117c7ebb 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala @@ -23,10 +23,6 @@ import org.apache.spark.sql.internal.SQLConf object SparkPlanUtil { def supportsRowBased(plan: SparkPlan): Boolean = { - if (SparkVersionUtil.lteSpark32) { - return !plan.supportsColumnar - } - val m = classOf[SparkPlan].getMethod("supportsRowBased") m.invoke(plan).asInstanceOf[Boolean] } diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala index afbc03dd1268..a19579e43311 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala @@ -64,23 +64,6 @@ object SparkTaskUtil { ctors.head } - if (SparkVersionUtil.lteSpark32) { - return ctor - .newInstance( - stageId, - stageAttemptNumber, - partitionId, - taskAttemptId, - attemptNumber, - taskMemoryManager, - localProperties, - metricsSystem, - taskMetrics, - resources - ) - .asInstanceOf[TaskContext] - } - if (SparkVersionUtil.eqSpark33) { return ctor .newInstance( From 1df1a3f0b6ec0873ebf0d53effb3e189803aa6a6 Mon Sep 17 00:00:00 2001 From: QCLyu Date: Mon, 9 Mar 2026 21:34:35 -0700 Subject: [PATCH 2/6] Clean up SparkVersionUtil.scala --- .../src/main/scala/org/apache/spark/util/SparkPlanUtil.scala | 2 +- .../src/main/scala/org/apache/spark/util/SparkVersionUtil.scala | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala index 9d83117c7ebb..a7a6cdaf3d21 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkPlanUtil.scala @@ -28,7 +28,7 @@ object SparkPlanUtil { } def isPlannedV1Write(plan: DataWritingCommandExec): Boolean = { - if (SparkVersionUtil.lteSpark33) { + if (SparkVersionUtil.eqSpark33) { return false } diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala index 50114ab7023e..b9089363b53c 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkVersionUtil.scala @@ -17,11 +17,9 @@ package org.apache.spark.util object SparkVersionUtil { - val lteSpark32: Boolean = compareMajorMinorVersion((3, 2)) <= 0 private val comparedWithSpark33 = compareMajorMinorVersion((3, 3)) private val comparedWithSpark35 = compareMajorMinorVersion((3, 5)) val eqSpark33: Boolean = comparedWithSpark33 == 0 - val lteSpark33: Boolean = lteSpark32 || eqSpark33 val gteSpark33: Boolean = comparedWithSpark33 >= 0 val gteSpark35: Boolean = comparedWithSpark35 >= 0 val gteSpark40: Boolean = compareMajorMinorVersion((4, 0)) >= 0 From dda10d6b933661d06939994ce5cd80fdd0444968 Mon Sep 17 00:00:00 2001 From: QCLyu Date: Mon, 9 Mar 2026 21:48:21 -0700 Subject: [PATCH 3/6] Remove Spark 3.2 shim files and test code --- .../gluten/execution/MiscOperatorSuite.scala | 19 +++------ .../parquet/ParquetFooterReaderShim.scala | 42 ------------------- 2 files changed, 5 insertions(+), 56 deletions(-) delete mode 100644 shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index e1a0fd98eeb5..dbc8c5e565dd 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -910,20 +910,11 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } test("Verify parquet field name with special character") { - withTable("t") { - - // https://github.com/apache/spark/pull/35229 Spark remove parquet field name check after 3.2 - if (!SparkShimLoader.getSparkVersion.startsWith("3.2")) { - sql("create table t using parquet as select sum(l_partkey) from lineitem") - runQueryAndCompare("select * from t") { - checkGlutenPlan[FileSourceScanExecTransformer] - } - } else { - val msg = intercept[AnalysisException] { - sql("create table t using parquet as select sum(l_partkey) from lineitem") - }.message - assert(msg.contains("contains invalid character")) - } + withTable("t") { + sql("create table t using parquet as select sum(l_partkey) from lineitem") + runQueryAndCompare("select * from t") { + checkGlutenPlan[FileSourceScanExecTransformer] + } } } diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala deleted file mode 100644 index b1419e5e6233..000000000000 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.parquet.format.converter.ParquetMetadataConverter -import org.apache.parquet.hadoop.metadata.ParquetMetadata - -/** Shim layer for ParquetFooterReader to maintain compatibility across different Spark versions. */ -object ParquetFooterReaderShim { - - /** @since Spark 4.1 */ - def readFooter( - configuration: Configuration, - fileStatus: FileStatus, - filter: ParquetMetadataConverter.MetadataFilter): ParquetMetadata = { - ParquetFooterReader.readFooter(configuration, fileStatus, filter) - } - - /** @since Spark 4.1 */ - def readFooter( - configuration: Configuration, - file: Path, - filter: ParquetMetadataConverter.MetadataFilter): ParquetMetadata = { - ParquetFooterReader.readFooter(configuration, file, filter) - } -} From a33cd483210b7c3b668022ec5523bc51174f08d0 Mon Sep 17 00:00:00 2001 From: QCLyu Date: Mon, 9 Mar 2026 22:04:19 -0700 Subject: [PATCH 4/6] Rename KnownRowTypeForSpark330rLater --- .../src/main/scala/org/apache/gluten/execution/GlutenPlan.scala | 2 +- .../columnar/enumerated/planner/plan/GroupLeafExec.scala | 2 +- .../gluten/extension/columnar/transition/Convention.scala | 2 +- .../spark/sql/execution/ColumnarCollapseTransformStages.scala | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala index 2cd408f67c3f..27448668846d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala @@ -45,7 +45,7 @@ import org.apache.spark.sql.execution.SparkPlan trait GlutenPlan extends SparkPlan with Convention.KnownBatchType - with Convention.KnownRowTypeForSpark33OrLater + with Convention.KnownRowTypeWithDefault with GlutenPlan.SupportsRowBasedCompatible with ConventionReq.KnownChildConvention { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala index 02746c4534f1..fc0432f0688f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicBoolean case class GroupLeafExec(groupId: Int, metadata: GlutenMetadata, convReq: Conv.Req) extends LeafExecNode with Convention.KnownBatchType - with Convention.KnownRowTypeForSpark33OrLater + with Convention.KnownRowTypeWithDefault with GlutenPlan.SupportsRowBasedCompatible { private val frozen = new AtomicBoolean(false) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala index 2fd378b4a9fa..393d9d71db9a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala @@ -166,7 +166,7 @@ object Convention { def rowType(): RowType } - trait KnownRowTypeForSpark33OrLater extends KnownRowType { + trait KnownRowTypeWithDefault extends KnownRowType { this: SparkPlan => final override def rowType(): RowType = { diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala index 134db93e70f9..8e700b9e38b3 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala @@ -188,7 +188,7 @@ case class ColumnarCollapseTransformStages(glutenConf: GlutenConfig) extends Rul case class ColumnarInputAdapter(child: SparkPlan) extends InputAdapterGenerateTreeStringShim with Convention.KnownBatchType - with Convention.KnownRowTypeForSpark33OrLater + with Convention.KnownRowTypeWithDefault with GlutenPlan.SupportsRowBasedCompatible with ConventionReq.KnownChildConvention { override def output: Seq[Attribute] = child.output From 41a012e5d7b8083fb30ebe438b441848a8c34942 Mon Sep 17 00:00:00 2001 From: QCLyu Date: Mon, 9 Mar 2026 22:39:03 -0700 Subject: [PATCH 5/6] Fix scala style violations --- .../org/apache/gluten/execution/MiscOperatorSuite.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index dbc8c5e565dd..8db8295b61ca 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -18,10 +18,9 @@ package org.apache.gluten.execution import org.apache.gluten.config.{GlutenConfig, GlutenCoreConfig, VeloxConfig} import org.apache.gluten.expression.VeloxDummyExpression -import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf -import org.apache.spark.sql.{AnalysisException, DataFrame, Row} +import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQEShuffleReadExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.joins.BaseJoinExec @@ -910,11 +909,11 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } test("Verify parquet field name with special character") { - withTable("t") { + withTable("t") { sql("create table t using parquet as select sum(l_partkey) from lineitem") runQueryAndCompare("select * from t") { checkGlutenPlan[FileSourceScanExecTransformer] - } + } } } From b17772225efc58cc4165fef051bd99816ee381ec Mon Sep 17 00:00:00 2001 From: QCLyu Date: Tue, 10 Mar 2026 21:35:09 -0700 Subject: [PATCH 6/6] simplify KnownRowType --- .../DeletionVectorWriteTransformer.scala | 2 +- .../perf/DeltaOptimizedWriterTransformer.scala | 2 +- .../execution/CHColumnarToCarrierRowExec.scala | 2 +- .../perf/GlutenDeltaOptimizedWriterExec.scala | 2 +- .../stats/GlutenDeltaJobStatsTracker.scala | 2 +- .../ColumnarPartialGenerateExec.scala | 2 +- .../execution/ColumnarPartialProjectExec.scala | 2 +- .../GpuResizeBufferColumnarBatchExec.scala | 2 +- .../VeloxColumnarToCarrierRowExec.scala | 2 +- .../execution/VeloxResizeBatchesExec.scala | 2 +- .../python/ColumnarArrowEvalPythonExec.scala | 2 +- .../sql/execution/BaseArrowScanExec.scala | 2 +- .../GlutenColumnarToColumnarTransition.scala | 2 +- .../apache/gluten/execution/GlutenPlan.scala | 6 +++--- .../planner/plan/GroupLeafExec.scala | 6 +++--- .../columnar/offload/OffloadSingleNode.scala | 2 +- .../columnar/transition/Convention.scala | 12 +----------- .../BasicPhysicalOperatorTransformer.scala | 2 +- .../CartesianProductExecTransformer.scala | 2 +- .../execution/ColumnarCoalesceExec.scala | 2 +- .../ColumnarCollectLimitBaseExec.scala | 2 +- .../ColumnarCollectTailBaseExec.scala | 2 +- .../execution/ColumnarToRowExecBase.scala | 2 +- .../execution/ColumnarV2TableWriteExec.scala | 2 +- .../execution/RowToColumnarExecBase.scala | 2 +- .../TakeOrderedAndProjectExecTransformer.scala | 2 +- .../execution/WholeStageTransformer.scala | 2 +- .../execution/ApplyResourceProfileExec.scala | 2 +- .../ColumnarBroadcastExchangeExec.scala | 2 +- .../ColumnarCollapseTransformStages.scala | 4 ++-- .../ColumnarShuffleExchangeExecBase.scala | 2 +- .../ColumnarSubqueryBroadcastExec.scala | 2 +- .../sql/execution/ColumnarWriteFilesExec.scala | 2 +- .../sql/execution/RDDScanTransformer.scala | 2 +- .../execution/RangeExecBaseTransformer.scala | 2 +- .../transition/TransitionSuiteBase.scala | 18 +++++++++--------- .../execution/FallbackStrategiesSuite.scala | 4 ++-- .../execution/FallbackStrategiesSuite.scala | 4 ++-- .../execution/FallbackStrategiesSuite.scala | 4 ++-- .../execution/FallbackStrategiesSuite.scala | 4 ++-- .../gluten/GlutenFallbackStrategiesSuite.scala | 4 ++-- .../gluten/GlutenFallbackStrategiesSuite.scala | 4 ++-- 42 files changed, 61 insertions(+), 71 deletions(-) diff --git a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala index a66efedc9aac..82dd46df41aa 100644 --- a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala +++ b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/execution/DeletionVectorWriteTransformer.scala @@ -96,7 +96,7 @@ case class DeletionVectorWriteTransformer( override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() } diff --git a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala index 800e4d1766a1..070a56d672e3 100644 --- a/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala +++ b/backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/perf/DeltaOptimizedWriterTransformer.scala @@ -248,7 +248,7 @@ case class DeltaOptimizedWriterTransformer( override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = { child.execute() diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala index 95ac277ec0eb..9d0c209655e8 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHColumnarToCarrierRowExec.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch case class CHColumnarToCarrierRowExec(override val child: SparkPlan) extends ColumnarToCarrierRowExecBase { override protected def fromBatchType(): Convention.BatchType = CHBatchType - override def rowType0(): Convention.RowType = CHCarrierRowType + override def rowType(): Convention.RowType = CHCarrierRowType override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(child = newChild) // Since https://github.com/apache/incubator-gluten/pull/1595. diff --git a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala index 8e83a8af8706..80f8e44e8816 100644 --- a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala +++ b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/perf/GlutenDeltaOptimizedWriterExec.scala @@ -235,7 +235,7 @@ case class GlutenDeltaOptimizedWriterExec( override def batchType(): Convention.BatchType = VeloxBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None } /** diff --git a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala index ae1126394379..680cca108255 100644 --- a/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala +++ b/backends-velox/src-delta33/main/scala/org/apache/spark/sql/delta/stats/GlutenDeltaJobStatsTracker.scala @@ -331,7 +331,7 @@ object GlutenDeltaJobStatsTracker extends Logging { with LeafExecNode { override def output: Seq[Attribute] = keySchema ++ dataSchema override def batchType(): Convention.BatchType = VeloxBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override protected def doExecuteColumnar(): RDD[ColumnarBatch] = throw new UnsupportedOperationException() diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala index 4e447df06435..dcd6380335dc 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialGenerateExec.scala @@ -385,7 +385,7 @@ case class ColumnarPartialGenerateExec(generateExec: GenerateExec, child: SparkP override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None final override def doExecute(): RDD[InternalRow] = { throw new UnsupportedOperationException( diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala index efc2a94cee44..17554262ef73 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala @@ -87,7 +87,7 @@ case class ColumnarPartialProjectExec(projectList: Seq[Expression], child: Spark override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None final override def doExecute(): RDD[InternalRow] = { throw new UnsupportedOperationException( diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala index 84768d71bc19..9de62e5bb859 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/GpuResizeBufferColumnarBatchExec.scala @@ -64,5 +64,5 @@ case class GpuResizeBufferColumnarBatchExec(override val child: SparkPlan, minOu override def batchType(): Convention.BatchType = VeloxBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala index a5007bae7557..65ce184861c7 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToCarrierRowExec.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.execution.SparkPlan case class VeloxColumnarToCarrierRowExec(override val child: SparkPlan) extends ColumnarToCarrierRowExecBase { override protected def fromBatchType(): Convention.BatchType = VeloxBatchType - override def rowType0(): Convention.RowType = VeloxCarrierRowType + override def rowType(): Convention.RowType = VeloxCarrierRowType override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(child = newChild) } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala index 9febad0e94ef..3b2c9490e75f 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala @@ -61,5 +61,5 @@ case class VeloxResizeBatchesExec( override def batchType(): Convention.BatchType = VeloxBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None } diff --git a/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala index 033f8a428470..6bb2f50e1f8b 100644 --- a/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala @@ -226,7 +226,7 @@ case class ColumnarArrowEvalPythonExec( override def batchType(): Convention.BatchType = ArrowJavaBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doValidateInternal(): ValidationResult = { val (_, inputs) = udfs.map(ColumnarArrowEvalPythonExec.collectFunctions).unzip diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala index ebccd6f5bb8c..13d0f5699b5a 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala @@ -25,5 +25,5 @@ trait BaseArrowScanExec extends GlutenPlan { ArrowBatchTypes.ArrowJavaBatchType } - final override def rowType0(): Convention.RowType = Convention.RowType.None + final override def rowType(): Convention.RowType = Convention.RowType.None } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala index d2f62f0bf911..c92224fcdeae 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenColumnarToColumnarTransition.scala @@ -30,7 +30,7 @@ trait GlutenColumnarToColumnarTransition extends ColumnarToColumnarTransition wi override def batchType(): Convention.BatchType = to - override def rowType0(): Convention.RowType = { + override def rowType(): Convention.RowType = { Convention.RowType.None } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala index 27448668846d..91d5ae4e7210 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/GlutenPlan.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.execution.SparkPlan * * Instead, subclasses are expected to implement the following APIs: * - batchType - * - rowType0 + * - rowType * - requiredChildConvention (optional) * * With implementations of the APIs provided, Gluten query planner will be able to find and insert @@ -45,7 +45,7 @@ import org.apache.spark.sql.execution.SparkPlan trait GlutenPlan extends SparkPlan with Convention.KnownBatchType - with Convention.KnownRowTypeWithDefault + with Convention.KnownRowType with GlutenPlan.SupportsRowBasedCompatible with ConventionReq.KnownChildConvention { @@ -59,7 +59,7 @@ trait GlutenPlan override def batchType(): Convention.BatchType - override def rowType0(): Convention.RowType + override def rowType(): Convention.RowType override def requiredChildConvention(): Seq[ConventionReq] = { // In the normal case, children's convention should follow parent node's convention. diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala index fc0432f0688f..ed5f48768a67 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/plan/GroupLeafExec.scala @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicBoolean case class GroupLeafExec(groupId: Int, metadata: GlutenMetadata, convReq: Conv.Req) extends LeafExecNode with Convention.KnownBatchType - with Convention.KnownRowTypeWithDefault + with Convention.KnownRowType with GlutenPlan.SupportsRowBasedCompatible { private val frozen = new AtomicBoolean(false) @@ -62,7 +62,7 @@ case class GroupLeafExec(groupId: Int, metadata: GlutenMetadata, convReq: Conv.R batchType != Convention.BatchType.None } - override val rowType0: Convention.RowType = { + override val rowType: Convention.RowType = { val out = convReq.req.requiredRowType match { case ConventionReq.RowType.Any => Convention.RowType.VanillaRowType case ConventionReq.RowType.Is(r) => r @@ -71,7 +71,7 @@ case class GroupLeafExec(groupId: Int, metadata: GlutenMetadata, convReq: Conv.R } final override val supportsRowBased: Boolean = { - rowType() != Convention.RowType.None + rowType != Convention.RowType.None } private def ensureNotFrozen(): Unit = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala index 4cf3e1d0c95d..db20e9efe268 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNode.scala @@ -99,7 +99,7 @@ object OffloadSingleNode { private lazy val conv: Convention = Convention.get(hiddenPlan) override def batchType(): Convention.BatchType = conv.batchType - override def rowType0(): Convention.RowType = conv.rowType + override def rowType(): Convention.RowType = conv.rowType override def output: Seq[Attribute] = hiddenPlan.output override def outputPartitioning: Partitioning = hiddenPlan.outputPartitioning override def outputOrdering: Seq[SortOrder] = hiddenPlan.outputOrdering diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala index 393d9d71db9a..116a74e53b3f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala @@ -162,17 +162,7 @@ object Convention { def batchType(): BatchType } - sealed trait KnownRowType { + trait KnownRowType { def rowType(): RowType } - - trait KnownRowTypeWithDefault extends KnownRowType { - this: SparkPlan => - - final override def rowType(): RowType = { - rowType0() - } - - def rowType0(): RowType - } } diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala index 57e7373fc3cf..1514aca532b8 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala @@ -248,7 +248,7 @@ case class ColumnarUnionExec(children: Seq[SparkPlan], partitioning: Partitionin override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def outputPartitioning: Partitioning = partitioning diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala index 326bd753292e..868e6f1a9ec7 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala @@ -46,7 +46,7 @@ import java.io.{IOException, ObjectOutputStream} case class ColumnarCartesianProductBridge(child: SparkPlan) extends UnaryExecNode with GlutenPlan { override def output: Seq[Attribute] = child.output override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override protected def doExecuteColumnar(): RDD[ColumnarBatch] = child.executeColumnar() diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala index 107bf544cd6f..ce801d80ec50 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala @@ -39,7 +39,7 @@ case class ColumnarCoalesceExec(numPartitions: Int, child: SparkPlan) override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = { throw new UnsupportedOperationException() diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala index da1999a24cfb..333707f469ff 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectLimitBaseExec.scala @@ -36,7 +36,7 @@ abstract class ColumnarCollectLimitBaseExec( ) extends LimitExec with ValidatablePlan { - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala index 56f9ce69ee6d..4ced96ec5ed1 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarCollectTailBaseExec.scala @@ -58,7 +58,7 @@ abstract class ColumnarCollectTailBaseExec( .genColumnarShuffleExchangeMetrics(sparkContext, shuffleWriterType) ++ readMetrics ++ writeMetrics - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def output: Seq[Attribute] = child.output diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala index 15103bbd84d6..3d3f4445c5bb 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala @@ -42,7 +42,7 @@ abstract class ColumnarToRowExecBase(child: SparkPlan) override def batchType(): Convention.BatchType = Convention.BatchType.None - override def rowType0(): Convention.RowType = Convention.RowType.VanillaRowType + override def rowType(): Convention.RowType = Convention.RowType.VanillaRowType override def requiredChildConvention(): Seq[ConventionReq] = { List( diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala index cb9470765b76..c737284b35a3 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ColumnarV2TableWriteExec.scala @@ -56,7 +56,7 @@ trait ColumnarV2TableWriteExec extends V2TableWriteExec with ValidatablePlan { override def batchType(): Convention.BatchType = Convention.BatchType.None - override def rowType0(): Convention.RowType = RowType.VanillaRowType + override def rowType(): Convention.RowType = RowType.VanillaRowType override def requiredChildConvention(): Seq[ConventionReq] = Seq( ConventionReq.ofBatch( diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala index a500da979036..e8e5ae50ab71 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/RowToColumnarExecBase.scala @@ -47,7 +47,7 @@ abstract class RowToColumnarExecBase(child: SparkPlan) override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def requiredChildConvention(): Seq[ConventionReq] = { Seq(ConventionReq.vanillaRow) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala index 39b37a0d1877..dc47dbba10fb 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/TakeOrderedAndProjectExecTransformer.scala @@ -44,7 +44,7 @@ case class TakeOrderedAndProjectExecTransformer( override def outputPartitioning: Partitioning = SinglePartition override def outputOrdering: Seq[SortOrder] = sortOrder override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def output: Seq[Attribute] = { projectList.map(_.toAttribute) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala index acef5d798ea0..959931b754f8 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala @@ -60,7 +60,7 @@ trait TransformSupport extends ValidatablePlan { BackendsApiManager.getSettings.primaryBatchType } - override def rowType0(): Convention.RowType = { + override def rowType(): Convention.RowType = { Convention.RowType.None } diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala index 17640f461213..b175cb5a05d5 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ApplyResourceProfileExec.scala @@ -42,7 +42,7 @@ case class ApplyResourceProfileExec(child: SparkPlan, resourceProfile: ResourceP Convention.get(child).batchType } - override def rowType0(): Convention.RowType = { + override def rowType(): Convention.RowType = { Convention.get(child).rowType } diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala index 371f9948b730..26a4ef391929 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarBroadcastExchangeExec.scala @@ -128,7 +128,7 @@ case class ColumnarBroadcastExchangeExec(mode: BroadcastMode, child: SparkPlan) override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def doCanonicalize(): SparkPlan = { ColumnarBroadcastExchangeExec(mode.canonicalized, child.canonicalized) diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala index 8e700b9e38b3..c81d380a7817 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala @@ -188,13 +188,13 @@ case class ColumnarCollapseTransformStages(glutenConf: GlutenConfig) extends Rul case class ColumnarInputAdapter(child: SparkPlan) extends InputAdapterGenerateTreeStringShim with Convention.KnownBatchType - with Convention.KnownRowTypeWithDefault + with Convention.KnownRowType with GlutenPlan.SupportsRowBasedCompatible with ConventionReq.KnownChildConvention { override def output: Seq[Attribute] = child.output final override val supportsColumnar: Boolean = true final override val supportsRowBased: Boolean = false - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType override def requiredChildConvention(): Seq[ConventionReq] = Seq( diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala index 17d1ec403814..24db757720be 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExecBase.scala @@ -137,7 +137,7 @@ abstract class ColumnarShuffleExchangeExecBase( override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def doExecute(): RDD[InternalRow] = { throw new UnsupportedOperationException() diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala index 3e91a5c7e49d..611433c51b5b 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala @@ -132,7 +132,7 @@ case class ColumnarSubqueryBroadcastExec( override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = { throw new UnsupportedOperationException( diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala index 998f4f86b50b..2d430bfb779d 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarWriteFilesExec.scala @@ -70,7 +70,7 @@ abstract class ColumnarWriteFilesExec protected ( * Since https://github.com/apache/incubator-gluten/pull/6745. */ override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): RowType = { + override def rowType(): RowType = { RowType.VanillaRowType } diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala index e3fc728477b9..91b719cece90 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RDDScanTransformer.scala @@ -29,7 +29,7 @@ abstract class RDDScanTransformer( override val outputOrdering: Seq[SortOrder] ) extends ValidatablePlan { - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType override def output: Seq[Attribute] = { outputAttributes diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala index 653b93883af5..e28344579f57 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/RangeExecBaseTransformer.scala @@ -44,7 +44,7 @@ abstract class ColumnarRangeBaseExec extends LeafExecNode with ValidatablePlan { s"${output.mkString("[", ", ", "]")}" } - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = { throw new UnsupportedOperationException(s"This operator doesn't support doExecute().") diff --git a/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala b/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala index 827881ebc95c..4fddfcfbd534 100644 --- a/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala +++ b/gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuiteBase.scala @@ -30,7 +30,7 @@ object TransitionSuiteBase { case class BatchLeaf(override val batchType: Convention.BatchType) extends LeafExecNode with GlutenPlan { - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() @@ -40,7 +40,7 @@ object TransitionSuiteBase { case class BatchUnary(override val batchType: Convention.BatchType, override val child: SparkPlan) extends UnaryExecNode with GlutenPlan { - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(child = newChild) @@ -56,7 +56,7 @@ object TransitionSuiteBase { override val right: SparkPlan) extends BinaryExecNode with GlutenPlan { - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def withNewChildrenInternal( newLeft: SparkPlan, @@ -67,7 +67,7 @@ object TransitionSuiteBase { override def output: Seq[Attribute] = left.output ++ right.output } - case class RowLeaf(override val rowType0: Convention.RowType) + case class RowLeaf(override val rowType: Convention.RowType) extends LeafExecNode with GlutenPlan { override def batchType(): Convention.BatchType = Convention.BatchType.None @@ -77,7 +77,7 @@ object TransitionSuiteBase { override def output: Seq[Attribute] = List.empty } - case class RowUnary(override val rowType0: Convention.RowType, override val child: SparkPlan) + case class RowUnary(override val rowType: Convention.RowType, override val child: SparkPlan) extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = Convention.BatchType.None @@ -91,7 +91,7 @@ object TransitionSuiteBase { } case class RowBinary( - override val rowType0: Convention.RowType, + override val rowType: Convention.RowType, override val left: SparkPlan, override val right: SparkPlan) extends BinaryExecNode @@ -114,7 +114,7 @@ object TransitionSuiteBase { extends RowToColumnarTransition with GlutenPlan { override def batchType(): Convention.BatchType = toBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override def requiredChildConvention(): Seq[ConventionReq] = { List(ConventionReq.ofRow(ConventionReq.RowType.Is(fromRowType))) } @@ -133,7 +133,7 @@ object TransitionSuiteBase { extends ColumnarToRowTransition with GlutenPlan { override def batchType(): Convention.BatchType = Convention.BatchType.None - override def rowType0(): Convention.RowType = toRowType + override def rowType(): Convention.RowType = toRowType override def requiredChildConvention(): Seq[ConventionReq] = { List(ConventionReq.ofBatch(ConventionReq.BatchType.Is(fromBatchType))) } @@ -165,7 +165,7 @@ object TransitionSuiteBase { extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = Convention.BatchType.None - override def rowType0(): Convention.RowType = to + override def rowType(): Convention.RowType = to override def requiredChildConvention(): Seq[ConventionReq] = { List(ConventionReq.ofRow(ConventionReq.RowType.Is(from))) } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index c0e3d1a349c1..6c1a073f8709 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -208,7 +208,7 @@ private object FallbackStrategiesSuite { // For replacing LeafOp. case class LeafOpTransformer() extends LeafExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = Seq.empty } @@ -218,7 +218,7 @@ private object FallbackStrategiesSuite { extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = child.output override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1Transformer = diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 6f3894d23b49..6ab2176cf7dd 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -237,7 +237,7 @@ private object FallbackStrategiesSuite { // For replacing LeafOp. case class LeafOpTransformer() extends LeafExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = Seq.empty } @@ -247,7 +247,7 @@ private object FallbackStrategiesSuite { extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = child.output override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1Transformer = diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 6f3894d23b49..6ab2176cf7dd 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -237,7 +237,7 @@ private object FallbackStrategiesSuite { // For replacing LeafOp. case class LeafOpTransformer() extends LeafExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = Seq.empty } @@ -247,7 +247,7 @@ private object FallbackStrategiesSuite { extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = child.output override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1Transformer = diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index de2436a39e33..d7d00bf005a8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -238,7 +238,7 @@ private object FallbackStrategiesSuite { // For replacing LeafOp. case class LeafOpTransformer() extends LeafExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = Seq.empty } @@ -248,7 +248,7 @@ private object FallbackStrategiesSuite { extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = child.output override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1Transformer = diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala index 50beded8defd..57146b4abf5a 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala @@ -239,7 +239,7 @@ private object GlutenFallbackStrategiesSuite { // For replacing LeafOp. case class LeafOpTransformer() extends LeafExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = Seq.empty } @@ -249,7 +249,7 @@ private object GlutenFallbackStrategiesSuite { extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = child.output override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1Transformer = diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala index 50beded8defd..57146b4abf5a 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackStrategiesSuite.scala @@ -239,7 +239,7 @@ private object GlutenFallbackStrategiesSuite { // For replacing LeafOp. case class LeafOpTransformer() extends LeafExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = Seq.empty } @@ -249,7 +249,7 @@ private object GlutenFallbackStrategiesSuite { extends UnaryExecNode with GlutenPlan { override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType0(): Convention.RowType = Convention.RowType.None + override def rowType(): Convention.RowType = Convention.RowType.None override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override def output: Seq[Attribute] = child.output override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1Transformer =