From 73bda44e554a2f8ea38b6f0f24531f212ff7fea5 Mon Sep 17 00:00:00 2001 From: rexan Date: Mon, 13 Jan 2025 21:42:16 -0500 Subject: [PATCH] [VL] Add Spark array_sort support --- .../backendsapi/velox/VeloxSparkPlanExecApi.scala | 8 ++++++++ .../apache/gluten/execution/MiscOperatorSuite.scala | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 463514fb5c0c..6d46d09c1f3f 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -200,6 +200,14 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { } } + override def genArraySortTransformer( + substraitExprName: String, + argument: ExpressionTransformer, + function: ExpressionTransformer, + expr: ArraySort): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(argument, function), expr) + } + /** Transform array exists to Substrait */ override def genArrayExistsTransformer( substraitExprName: String, diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index 04d8f7dffffc..5403b786b3ed 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -1188,6 +1188,18 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } } + test("array_sort") { + withTable("t") { + sql("create table t (a array) using parquet") + sql("insert into t values (array('a', 'acds', 'bcedf', 'dc'))") + runQueryAndCompare( + "select array_sort(a, (x, y) -> " + + "if(length(x) > length(y), 1, if(length(x) < length(y), -1, 0))) from t") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + test("Support bool type filter in scan") { withTable("t") { sql("create table t (id int, b boolean) using parquet")