From cc106af514fd8756d9126603f8755c2a6de459fc Mon Sep 17 00:00:00 2001 From: zhli1142015 Date: Wed, 24 Sep 2025 12:06:44 +0800 Subject: [PATCH 1/3] [VL] Enable div function --- .../scala/org/apache/gluten/utils/CHExpressionUtil.scala | 3 ++- .../gluten/functions/ArithmeticAnsiValidateSuite.scala | 9 +++++++++ ep/build-velox/src/get_velox.sh | 2 +- .../apache/gluten/expression/ExpressionConverter.scala | 8 ++++++++ .../apache/gluten/expression/ExpressionMappings.scala | 1 + .../org/apache/gluten/expression/ExpressionNames.scala | 2 ++ 6 files changed, 23 insertions(+), 2 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index 8365245b848d..2954a4387a4e 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -207,6 +207,7 @@ object CHExpressionUtil { MAKE_DATE -> DefaultValidator(), ARRAY_APPEND -> DefaultValidator(), JSON_OBJECT_KEYS -> DefaultValidator(), - LUHN_CHECK -> DefaultValidator() + LUHN_CHECK -> DefaultValidator(), + DIV -> DefaultValidator() ) } diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala index b3ee4d432146..c51f593b2269 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala @@ -72,4 +72,13 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { } } + test("div") { + runQueryAndCompare("SELECT int_field1 div 2 FROM datatab WHERE int_field1 IS NOT NULL") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + intercept[ArithmeticException] { + sql("SELECT 2147483647 div 0").collect() + } + } + } diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index a361910f8286..d1073a64ee90 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,7 +16,7 @@ set -exu -VELOX_REPO=https://github.com/oap-project/velox.git +VELOX_REPO=https://github.com/zhli1142015/velox.git VELOX_BRANCH=2025_09_23 VELOX_HOME="" RUN_SETUP_SCRIPT=ON diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 37f90d512e0a..07c12db2fcb5 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -695,6 +695,14 @@ object ExpressionConverter extends SQLConfHelper with Logging { a, ExpressionNames.CHECKED_DIVIDE ) + case i: IntegralDivide => + BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( + substraitExprName, + replaceWithExpressionTransformer0(i.left, attributeSeq, expressionsMap), + replaceWithExpressionTransformer0(i.right, attributeSeq, expressionsMap), + i, + ExpressionNames.CHECKED_DIV + ) case tryEval: TryEval => // This is a placeholder to handle try_eval(other expressions). BackendsApiManager.getSparkPlanExecApiInstance.genTryEvalTransformer( diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index cc7500287517..b0b7c8079315 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -37,6 +37,7 @@ object ExpressionMappings { Sig[Subtract](SUBTRACT), Sig[Multiply](MULTIPLY), Sig[Divide](DIVIDE), + Sig[IntegralDivide](DIV), Sig[UnaryPositive](POSITIVE), Sig[UnaryMinus](NEGATIVE), Sig[And](AND), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 8fcdc01e5e86..25e4aeb45e2f 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -60,6 +60,7 @@ object ExpressionNames { final val ADD = "add" final val SUBTRACT = "subtract" final val MULTIPLY = "multiply" + final val DIV = "div" final val DIVIDE = "divide" final val POSITIVE = "positive" final val NEGATIVE = "negative" @@ -91,6 +92,7 @@ object ExpressionNames { final val CHECKED_SUBTRACT = "checked_subtract" final val CHECKED_DIVIDE = "checked_divide" final val CHECKED_MULTIPLY = "checked_multiply" + final val CHECKED_DIV = "checked_div" // SparkSQL String functions final val ASCII = "ascii" From 45438f699804f09178d4944e54c0db6751585eae Mon Sep 17 00:00:00 2001 From: zhli1142015 Date: Wed, 24 Sep 2025 14:27:09 +0800 Subject: [PATCH 2/3] fix ut --- .../apache/gluten/functions/ArithmeticAnsiValidateSuite.scala | 4 ++-- .../org/apache/gluten/sql/shims/spark34/Spark34Shims.scala | 1 + .../org/apache/gluten/sql/shims/spark35/Spark35Shims.scala | 1 + .../org/apache/gluten/sql/shims/spark40/Spark40Shims.scala | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala index c51f593b2269..65b3d98066de 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala @@ -76,8 +76,8 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { runQueryAndCompare("SELECT int_field1 div 2 FROM datatab WHERE int_field1 IS NOT NULL") { checkGlutenOperatorMatch[ProjectExecTransformer] } - intercept[ArithmeticException] { - sql("SELECT 2147483647 div 0").collect() + intercept[SparkException] { + sql("SELECT 1 div 0 ").collect() } } diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index c09783343587..be6056d3656f 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -562,6 +562,7 @@ class Spark34Shims extends SparkShims { case s: Subtract => s.evalMode == EvalMode.ANSI case d: Divide => d.evalMode == EvalMode.ANSI case m: Multiply => m.evalMode == EvalMode.ANSI + case i: IntegralDivide => i.evalMode == EvalMode.ANSI case _ => false } } diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 58759949f79b..6f30df594295 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -601,6 +601,7 @@ class Spark35Shims extends SparkShims { case s: Subtract => s.evalMode == EvalMode.ANSI case d: Divide => d.evalMode == EvalMode.ANSI case m: Multiply => m.evalMode == EvalMode.ANSI + case i: IntegralDivide => i.evalMode == EvalMode.ANSI case _ => false } } diff --git a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala index 11d04198ab77..f7f9631bfb8a 100644 --- a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala +++ b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala @@ -608,6 +608,7 @@ class Spark40Shims extends SparkShims { case s: Subtract => s.evalMode == EvalMode.ANSI case d: Divide => d.evalMode == EvalMode.ANSI case m: Multiply => m.evalMode == EvalMode.ANSI + case i: IntegralDivide => i.evalMode == EvalMode.ANSI case _ => false } } From 82c3f89ea7f7df7c1740acd2cb806dce2ef54be8 Mon Sep 17 00:00:00 2001 From: zhli1142015 Date: Wed, 24 Sep 2025 15:55:54 +0800 Subject: [PATCH 3/3] minor change --- .../gluten/functions/ArithmeticAnsiValidateSuite.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala index 65b3d98066de..ed8a36d1bc2a 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala @@ -76,8 +76,10 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { runQueryAndCompare("SELECT int_field1 div 2 FROM datatab WHERE int_field1 IS NOT NULL") { checkGlutenOperatorMatch[ProjectExecTransformer] } - intercept[SparkException] { - sql("SELECT 1 div 0 ").collect() + if (isSparkVersionGE("3.4")) { + intercept[SparkException] { + sql("SELECT 1 div 0 ").collect() + } } }