Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ class DynamicOffHeapSizingSuite extends VeloxWholeStageTransformerSuite {
""".stripMargin
var totalMemory = Runtime.getRuntime().totalMemory()
var freeMemory = Runtime.getRuntime().freeMemory()
val maxMemory = Runtime.getRuntime().maxMemory()
// Ensure that the JVM memory is not too small to trigger dynamic off-heap sizing.
while (!DynamicOffHeapSizingMemoryTarget.canShrinkJVMMemory(totalMemory, freeMemory)) {
withSQLConf(("spark.gluten.enabled", "false")) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl
case _: BroadcastHashJoinExecTransformerBase => true
case _ => false
}
assert(!columnarBhj.isDefined)
assert(columnarBhj.isEmpty)

val vanillaBhj = find(plan) {
case _: BroadcastHashJoinExec => true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ class GlutenSQLCollectTailExecSuite extends WholeStageTransformerSuite {

test("ColumnarCollectTailExec - with filter") {
val df = spark.range(0, 10000, 1).toDF("id").filter("id % 2 == 0").orderBy("id")
val evenCount = 5000
val expected = (9990L to 9998L by 2).map(Row(_)).takeRight(5)
verifyTailExec(df, expected, tailCount = 5)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa
}

test("group sets") {
val result = runQueryAndCompare(
runQueryAndCompare(
"select l_orderkey, l_partkey, sum(l_suppkey) from lineitem " +
"where l_orderkey < 3 group by ROLLUP(l_orderkey, l_partkey) " +
"order by l_orderkey, l_partkey ") { _ => }
Expand Down Expand Up @@ -583,13 +583,13 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa
}

test("union two tables") {
val df = runQueryAndCompare("""
|select count(orderkey) from (
| select l_orderkey as orderkey from lineitem
| union
| select o_orderkey as orderkey from orders
|);
|""".stripMargin) {
runQueryAndCompare("""
|select count(orderkey) from (
| select l_orderkey as orderkey from lineitem
| union
| select o_orderkey as orderkey from orders
|);
|""".stripMargin) {
df =>
{
getExecutedPlan(df).exists(plan => plan.find(_.isInstanceOf[ColumnarUnionExec]).isDefined)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,9 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu
}

test("count") {
val df =
runQueryAndCompare("select count(*) from lineitem where l_partkey in (1552, 674, 1062)") {
checkGlutenOperatorMatch[HashAggregateExecTransformer]
}
runQueryAndCompare("select count(*) from lineitem where l_partkey in (1552, 674, 1062)") {
checkGlutenOperatorMatch[HashAggregateExecTransformer]
}
runQueryAndCompare("select count(l_quantity), count(distinct l_partkey) from lineitem") {
df =>
{
Expand All @@ -70,7 +69,7 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu
}

test("avg") {
val df = runQueryAndCompare("select avg(l_partkey) from lineitem where l_partkey < 1000") {
runQueryAndCompare("select avg(l_partkey) from lineitem where l_partkey < 1000") {
checkGlutenOperatorMatch[HashAggregateExecTransformer]
}
runQueryAndCompare("select avg(l_quantity), count(distinct l_partkey) from lineitem") {
Expand Down Expand Up @@ -215,7 +214,7 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu
}

test("group sets") {
val result = runQueryAndCompare(
runQueryAndCompare(
"select l_orderkey, l_partkey, sum(l_suppkey) from lineitem " +
"where l_orderkey < 3 group by ROLLUP(l_orderkey, l_partkey) " +
"order by l_orderkey, l_partkey ") { _ => }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,8 @@ class VeloxHashJoinSuite extends VeloxWholeStageTransformerSuite {
Seq("true", "false").foreach(
enabledOffheapBroadcast =>
withSQLConf(
(GlutenConfig.GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD.key -> "16"),
(VeloxConfig.VELOX_BROADCAST_BUILD_RELATION_USE_OFFHEAP.key -> enabledOffheapBroadcast)) {
GlutenConfig.GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD.key -> "16",
VeloxConfig.VELOX_BROADCAST_BUILD_RELATION_USE_OFFHEAP.key -> enabledOffheapBroadcast) {
withTable("t1", "t2") {
spark.sql("""
|CREATE TABLE t1 USING PARQUET
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,9 @@ class VeloxScanSuite extends VeloxWholeStageTransformerSuite {
val plan = df.queryExecution.executedPlan
val fileScan = collect(plan) { case s: FileSourceScanExecTransformer => s }
assert(fileScan.size == 1)
val rootPaths = fileScan(0).getRootPathsInternal
val rootPaths = fileScan.head.getRootPathsInternal
assert(rootPaths.length == 1)
assert(rootPaths(0).startsWith("file:/"))
assert(rootPaths.head.startsWith("file:/"))
assert(
VeloxFileSystemValidationJniWrapper.allSupportedByRegisteredFileSystems(
rootPaths.toArray))
Expand All @@ -107,7 +107,7 @@ class VeloxScanSuite extends VeloxWholeStageTransformerSuite {
VeloxBackendSettings.distinctRootPaths(
Seq("file:/test_path/", "test://test/s", "test://test1/s"))
assert(filteredRootPath.length == 1)
assert(filteredRootPath(0).startsWith("test://"))
assert(filteredRootPath.head.startsWith("test://"))
assert(
VeloxFileSystemValidationJniWrapper.allSupportedByRegisteredFileSystems(
Array("file:/test_path/")))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class VeloxRasSuite extends SharedSparkSession {
object ConvertRowUnaryToColumnar extends RasRule[SparkPlan] {
override def shift(node: SparkPlan): Iterable[SparkPlan] = node match {
case RowUnary(child) => List(ColumnarUnary(child))
case other => List.empty
case _ => List.empty
}

override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1)
Expand Down
Loading