From 25c2fbb506432c862634ab7cfd87a50917bf86d1 Mon Sep 17 00:00:00 2001 From: Panos Garefalakis Date: Tue, 7 Oct 2025 09:57:45 -0700 Subject: [PATCH] Add Golden Files for TPC-H Spark35 with Gluten Execution Plan --- .../v1-bhj-ras/spark35/1.txt | 162 +++ .../v1-bhj-ras/spark35/10.txt | 374 ++++++ .../v1-bhj-ras/spark35/11.txt | 559 +++++++++ .../v1-bhj-ras/spark35/12.txt | 238 ++++ .../v1-bhj-ras/spark35/13.txt | 299 +++++ .../v1-bhj-ras/spark35/14.txt | 197 +++ .../v1-bhj-ras/spark35/15.txt | 390 ++++++ .../v1-bhj-ras/spark35/16.txt | 326 +++++ .../v1-bhj-ras/spark35/17.txt | 205 ++++ .../v1-bhj-ras/spark35/18.txt | 488 ++++++++ .../v1-bhj-ras/spark35/19.txt | 192 +++ .../v1-bhj-ras/spark35/20.txt | 533 +++++++++ .../v1-bhj-ras/spark35/21.txt | 504 ++++++++ .../v1-bhj-ras/spark35/22.txt | 356 ++++++ .../v1-bhj-ras/spark35/3.txt | 298 +++++ .../v1-bhj-ras/spark35/4.txt | 248 ++++ .../v1-bhj-ras/spark35/5.txt | 552 +++++++++ .../v1-bhj-ras/spark35/6.txt | 116 ++ .../v1-bhj-ras/spark35/7.txt | 514 ++++++++ .../v1-bhj-ras/spark35/8.txt | 709 +++++++++++ .../v1-bhj-ras/spark35/9.txt | 542 +++++++++ .../tpch-approved-plan/v1-bhj/spark35/1.txt | 162 +++ .../tpch-approved-plan/v1-bhj/spark35/10.txt | 374 ++++++ .../tpch-approved-plan/v1-bhj/spark35/11.txt | 559 +++++++++ .../tpch-approved-plan/v1-bhj/spark35/12.txt | 238 ++++ .../tpch-approved-plan/v1-bhj/spark35/13.txt | 299 +++++ .../tpch-approved-plan/v1-bhj/spark35/14.txt | 197 +++ .../tpch-approved-plan/v1-bhj/spark35/15.txt | 390 ++++++ .../tpch-approved-plan/v1-bhj/spark35/16.txt | 326 +++++ .../tpch-approved-plan/v1-bhj/spark35/17.txt | 205 ++++ .../tpch-approved-plan/v1-bhj/spark35/18.txt | 488 ++++++++ .../tpch-approved-plan/v1-bhj/spark35/19.txt | 192 +++ .../tpch-approved-plan/v1-bhj/spark35/20.txt | 533 +++++++++ .../tpch-approved-plan/v1-bhj/spark35/21.txt | 504 ++++++++ .../tpch-approved-plan/v1-bhj/spark35/22.txt | 356 ++++++ .../tpch-approved-plan/v1-bhj/spark35/3.txt | 298 +++++ .../tpch-approved-plan/v1-bhj/spark35/4.txt | 248 ++++ .../tpch-approved-plan/v1-bhj/spark35/5.txt | 552 +++++++++ .../tpch-approved-plan/v1-bhj/spark35/6.txt | 116 ++ .../tpch-approved-plan/v1-bhj/spark35/7.txt | 514 ++++++++ .../tpch-approved-plan/v1-bhj/spark35/8.txt | 709 +++++++++++ .../tpch-approved-plan/v1-bhj/spark35/9.txt | 542 +++++++++ .../tpch-approved-plan/v1-ras/spark35/1.txt | 162 +++ .../tpch-approved-plan/v1-ras/spark35/10.txt | 522 ++++++++ .../tpch-approved-plan/v1-ras/spark35/11.txt | 709 +++++++++++ .../tpch-approved-plan/v1-ras/spark35/12.txt | 289 +++++ .../tpch-approved-plan/v1-ras/spark35/13.txt | 306 +++++ .../tpch-approved-plan/v1-ras/spark35/14.txt | 209 ++++ .../tpch-approved-plan/v1-ras/spark35/15.txt | 410 +++++++ .../tpch-approved-plan/v1-ras/spark35/16.txt | 382 ++++++ .../tpch-approved-plan/v1-ras/spark35/17.txt | 347 ++++++ .../tpch-approved-plan/v1-ras/spark35/18.txt | 589 +++++++++ .../tpch-approved-plan/v1-ras/spark35/19.txt | 204 ++++ .../tpch-approved-plan/v1-ras/spark35/20.txt | 734 ++++++++++++ .../tpch-approved-plan/v1-ras/spark35/21.txt | 713 +++++++++++ .../tpch-approved-plan/v1-ras/spark35/22.txt | 412 +++++++ .../tpch-approved-plan/v1-ras/spark35/3.txt | 351 ++++++ .../tpch-approved-plan/v1-ras/spark35/4.txt | 294 +++++ .../tpch-approved-plan/v1-ras/spark35/5.txt | 802 +++++++++++++ .../tpch-approved-plan/v1-ras/spark35/6.txt | 116 ++ .../tpch-approved-plan/v1-ras/spark35/7.txt | 764 ++++++++++++ .../tpch-approved-plan/v1-ras/spark35/8.txt | 1061 +++++++++++++++++ .../tpch-approved-plan/v1-ras/spark35/9.txt | 797 +++++++++++++ .../tpch-approved-plan/v1/spark35/1.txt | 162 +++ .../tpch-approved-plan/v1/spark35/10.txt | 522 ++++++++ .../tpch-approved-plan/v1/spark35/11.txt | 709 +++++++++++ .../tpch-approved-plan/v1/spark35/12.txt | 289 +++++ .../tpch-approved-plan/v1/spark35/13.txt | 306 +++++ .../tpch-approved-plan/v1/spark35/14.txt | 209 ++++ .../tpch-approved-plan/v1/spark35/15.txt | 410 +++++++ .../tpch-approved-plan/v1/spark35/16.txt | 382 ++++++ .../tpch-approved-plan/v1/spark35/17.txt | 347 ++++++ .../tpch-approved-plan/v1/spark35/18.txt | 589 +++++++++ .../tpch-approved-plan/v1/spark35/19.txt | 204 ++++ .../tpch-approved-plan/v1/spark35/20.txt | 734 ++++++++++++ .../tpch-approved-plan/v1/spark35/21.txt | 713 +++++++++++ .../tpch-approved-plan/v1/spark35/22.txt | 412 +++++++ .../tpch-approved-plan/v1/spark35/3.txt | 351 ++++++ .../tpch-approved-plan/v1/spark35/4.txt | 294 +++++ .../tpch-approved-plan/v1/spark35/5.txt | 802 +++++++++++++ .../tpch-approved-plan/v1/spark35/6.txt | 116 ++ .../tpch-approved-plan/v1/spark35/7.txt | 764 ++++++++++++ .../tpch-approved-plan/v1/spark35/8.txt | 1061 +++++++++++++++++ .../tpch-approved-plan/v1/spark35/9.txt | 797 +++++++++++++ .../gluten/execution/VeloxTPCHSuite.scala | 1 + 85 files changed, 35951 insertions(+) create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/1.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/10.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/11.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/12.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/13.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/14.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/15.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/16.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/17.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/18.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/19.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/20.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/21.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/22.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/3.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/4.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/5.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/6.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/7.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/8.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/9.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/1.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/10.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/11.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/12.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/13.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/14.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/15.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/16.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/17.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/18.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/19.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/20.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/21.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/22.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/3.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/4.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/5.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/6.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/7.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/8.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/9.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/1.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/10.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/11.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/12.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/13.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/14.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/15.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/16.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/17.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/18.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/19.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/20.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/21.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/22.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/3.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/4.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/5.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/6.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/7.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/8.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/9.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/1.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/10.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/11.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/12.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/13.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/14.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/15.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/16.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/17.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/18.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/19.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/20.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/21.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/22.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/3.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/4.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/5.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/6.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/7.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/8.txt create mode 100644 backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/9.txt diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/1.txt new file mode 100644 index 000000000000..4c768cbd8b06 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/1.txt @@ -0,0 +1,162 @@ +== Physical Plan == +AdaptiveSparkPlan (30) ++- == Final Plan == + VeloxColumnarToRow (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxResizeBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) + + +(1) ScanTransformer parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(3) ProjectExecTransformer +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(5) ProjectExecTransformer +Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(6) WholeStageCodegenTransformer (X) +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: false + +(7) VeloxResizeBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X, X + +(8) ColumnarExchange +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(9) ShuffleQueryStage +Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(10) InputAdapter +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(11) InputIteratorTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(12) RegularHashAggregateExecTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(13) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(14) VeloxResizeBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X, X + +(15) ColumnarExchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(17) InputAdapter +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(18) InputIteratorTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(19) SortExecTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(20) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(21) VeloxColumnarToRow +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(22) Scan parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(23) Filter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(24) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(25) HashAggregate +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(26) Exchange +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(27) HashAggregate +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(28) Exchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(29) Sort +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(30) AdaptiveSparkPlan +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: isFinalPlan=true diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/10.txt new file mode 100644 index 000000000000..8154ff50d7e2 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/10.txt @@ -0,0 +1,374 @@ +== Physical Plan == +AdaptiveSparkPlan (68) ++- == Final Plan == + VeloxColumnarToRow (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxResizeBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ ScanTransformer parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ ScanTransformer parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ ScanTransformer parquet (23) ++- == Initial Plan == + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) + + +(1) ScanTransformer parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(5) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(6) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: false + +(7) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(8) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(9) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(10) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(13) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(15) ProjectExecTransformer +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(16) WholeStageCodegenTransformer (X) +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(17) ColumnarBroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(18) BroadcastQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(20) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(21) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(22) ProjectExecTransformer +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(23) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(24) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(25) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(26) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(27) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(28) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(29) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(30) BroadcastHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(31) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(32) FlushableHashAggregateExecTransformer +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, _pre_X#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(33) ProjectExecTransformer +Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(34) WholeStageCodegenTransformer (X) +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: false + +(35) VeloxResizeBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X, X + +(36) ColumnarExchange +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(37) ShuffleQueryStage +Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(38) InputAdapter +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(39) InputIteratorTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(40) RegularHashAggregateExecTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(41) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(42) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(43) TakeOrderedAndProjectExecTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 + +(44) VeloxColumnarToRow +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(45) Scan parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(46) Filter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(47) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(48) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(49) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(50) BroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(51) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(52) Project +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(53) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(54) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(55) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(56) BroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(57) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(58) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(59) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(60) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(61) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(62) BroadcastHashJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(63) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(64) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(65) Exchange +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(66) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(67) TakeOrderedAndProject +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(68) AdaptiveSparkPlan +Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/11.txt new file mode 100644 index 000000000000..1ae5e4c4fa4c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/11.txt @@ -0,0 +1,559 @@ +== Physical Plan == +AdaptiveSparkPlan (60) ++- == Final Plan == + VeloxColumnarToRow (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxResizeBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxResizeBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ ScanTransformer parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ ScanTransformer parquet (12) ++- == Initial Plan == + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) + + +(1) ScanTransformer parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(3) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(5) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(6) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(7) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(11) ProjectExecTransformer +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(12) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(13) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(14) ProjectExecTransformer +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(15) WholeStageCodegenTransformer (X) +Input [1]: [n_nationkey#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(18) InputAdapter +Input [1]: [n_nationkey#X] + +(19) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [2]: [ps_partkey#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(22) FlushableHashAggregateExecTransformer +Input [2]: [ps_partkey#X, _pre_X#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(23) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(24) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: false + +(25) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(26) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(27) ShuffleQueryStage +Output [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(28) InputAdapter +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(29) InputIteratorTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(30) RegularHashAggregateExecTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(31) FilterExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(32) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(33) VeloxResizeBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X, X + +(34) ColumnarExchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(35) ShuffleQueryStage +Output [2]: [ps_partkey#X, value#X] +Arguments: X + +(36) InputAdapter +Input [2]: [ps_partkey#X, value#X] + +(37) InputIteratorTransformer +Input [2]: [ps_partkey#X, value#X] + +(38) SortExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(39) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(40) VeloxColumnarToRow +Input [2]: [ps_partkey#X, value#X] + +(41) Scan parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(42) Filter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(43) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(44) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(45) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(46) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(47) Project +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(48) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(49) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(50) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(51) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(52) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(53) Project +Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(54) HashAggregate +Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(55) Exchange +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) HashAggregate +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(57) Filter +Input [2]: [ps_partkey#X, value#X] +Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(58) Exchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(59) Sort +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(60) AdaptiveSparkPlan +Output [2]: [ps_partkey#X, value#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) ++- == Final Plan == + VeloxColumnarToRow (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxResizeBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ FilterExecTransformer (62) + : : +- ^ ScanTransformer parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) ++- == Initial Plan == + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) ScanTransformer parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(63) ReusedExchange [Reuses operator id: 6] +Output [2]: [s_suppkey#X, s_nationkey#X] + +(64) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(65) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(66) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(67) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(68) ProjectExecTransformer +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(69) ReusedExchange [Reuses operator id: 16] +Output [1]: [n_nationkey#X] + +(70) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(71) InputAdapter +Input [1]: [n_nationkey#X] + +(72) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(73) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(74) ProjectExecTransformer +Output [1]: [(ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(75) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(76) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(77) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(78) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(79) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(80) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(81) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(82) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(83) ProjectExecTransformer +Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(84) WholeStageCodegenTransformer (X) +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: false + +(85) VeloxColumnarToRow +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(86) Scan parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(87) Filter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(88) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(89) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(90) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(91) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(92) Project +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(93) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(94) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(95) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(96) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(97) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(98) Project +Output [2]: [ps_availqty#X, ps_supplycost#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(99) HashAggregate +Input [2]: [ps_availqty#X, ps_supplycost#X] +Keys: [] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(100) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(101) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(102) AdaptiveSparkPlan +Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/12.txt new file mode 100644 index 000000000000..d5dc00e6dc4c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/12.txt @@ -0,0 +1,238 @@ +== Physical Plan == +AdaptiveSparkPlan (44) ++- == Final Plan == + VeloxColumnarToRow (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxResizeBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxResizeBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ ScanTransformer parquet (8) ++- == Initial Plan == + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) + + +(1) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: isnotnull(o_orderkey#X) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) InputAdapter +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(7) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(8) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(10) ProjectExecTransformer +Output [2]: [l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [3]: [l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(13) FlushableHashAggregateExecTransformer +Input [3]: [l_shipmode#X, _pre_X#X, _pre_X#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(14) ProjectExecTransformer +Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(15) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: false + +(16) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X, X + +(17) ColumnarExchange +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [shuffle_writer_type=hash] + +(18) ShuffleQueryStage +Output [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(20) InputIteratorTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(21) RegularHashAggregateExecTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(22) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(23) VeloxResizeBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X, X + +(24) ColumnarExchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(26) InputAdapter +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(27) InputIteratorTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(28) SortExecTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(29) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(30) VeloxColumnarToRow +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(31) Scan parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(32) Filter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Condition : isnotnull(o_orderkey#X) + +(33) BroadcastExchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(34) Scan parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(35) Filter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(36) Project +Output [2]: [l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(37) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(38) Project +Output [2]: [o_orderpriority#X, l_shipmode#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(39) HashAggregate +Input [2]: [o_orderpriority#X, l_shipmode#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(40) Exchange +Input [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(41) HashAggregate +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(42) Exchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) Sort +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(44) AdaptiveSparkPlan +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/13.txt new file mode 100644 index 000000000000..8848ac70c05f --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/13.txt @@ -0,0 +1,299 @@ +== Physical Plan == +AdaptiveSparkPlan (55) ++- == Final Plan == + VeloxColumnarToRow (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxResizeBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxResizeBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ ScanTransformer parquet (2) ++- == Initial Plan == + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) + + +(1) ScanTransformer parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(2) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(3) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(4) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(5) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: false + +(6) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(7) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(9) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(11) ProjectExecTransformer +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(12) FlushableHashAggregateExecTransformer +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [partial_count(o_orderkey#X)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_custkey#X, count#X] + +(13) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] +Input [2]: [c_custkey#X, count#X] + +(14) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: false + +(15) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X, X + +(16) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [2]: [c_custkey#X, count#X] +Arguments: X + +(18) InputAdapter +Input [2]: [c_custkey#X, count#X] + +(19) InputIteratorTransformer +Input [2]: [c_custkey#X, count#X] + +(20) RegularHashAggregateExecTransformer +Input [2]: [c_custkey#X, count#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(21) ProjectExecTransformer +Output [1]: [count(o_orderkey#X)#X AS c_count#X] +Input [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(22) FlushableHashAggregateExecTransformer +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(23) ProjectExecTransformer +Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] +Input [2]: [c_count#X, count#X] + +(24) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: false + +(25) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X, X + +(26) ColumnarExchange +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(27) ShuffleQueryStage +Output [2]: [c_count#X, count#X] +Arguments: X + +(28) InputAdapter +Input [2]: [c_count#X, count#X] + +(29) InputIteratorTransformer +Input [2]: [c_count#X, count#X] + +(30) RegularHashAggregateExecTransformer +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(31) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(32) VeloxResizeBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X, X + +(33) ColumnarExchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(34) ShuffleQueryStage +Output [2]: [c_count#X, custdist#X] +Arguments: X + +(35) InputAdapter +Input [2]: [c_count#X, custdist#X] + +(36) InputIteratorTransformer +Input [2]: [c_count#X, custdist#X] + +(37) SortExecTransformer +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(38) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(39) VeloxColumnarToRow +Input [2]: [c_count#X, custdist#X] + +(40) Scan parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(41) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(42) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(43) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(44) BroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(45) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(46) Project +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(47) HashAggregate +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [partial_count(o_orderkey#X)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_custkey#X, count#X] + +(48) Exchange +Input [2]: [c_custkey#X, count#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(49) HashAggregate +Input [2]: [c_custkey#X, count#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [1]: [count(o_orderkey#X)#X AS c_count#X] + +(50) HashAggregate +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(51) Exchange +Input [2]: [c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(52) HashAggregate +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(53) Exchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(54) Sort +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(55) AdaptiveSparkPlan +Output [2]: [c_count#X, custdist#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/14.txt new file mode 100644 index 000000000000..17ce5be290b0 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/14.txt @@ -0,0 +1,197 @@ +== Physical Plan == +AdaptiveSparkPlan (35) ++- == Final Plan == + VeloxColumnarToRow (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ ScanTransformer parquet (4) ++- == Initial Plan == + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) + + +(1) ScanTransformer parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(3) ProjectExecTransformer +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(5) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: isnotnull(p_partkey#X) + +(6) WholeStageCodegenTransformer (X) +Input [2]: [p_partkey#X, p_type#X] +Arguments: false + +(7) ColumnarBroadcastExchange +Input [2]: [p_partkey#X, p_type#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(8) BroadcastQueryStage +Output [2]: [p_partkey#X, p_type#X] +Arguments: X + +(9) InputAdapter +Input [2]: [p_partkey#X, p_type#X] + +(10) InputIteratorTransformer +Input [2]: [p_partkey#X, p_type#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [2]: [CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(13) FlushableHashAggregateExecTransformer +Input [2]: [_pre_X#X, _pre_X#X] +Keys: [] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(14) WholeStageCodegenTransformer (X) +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: false + +(15) VeloxResizeBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X, X + +(16) ColumnarExchange +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(18) InputAdapter +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(19) InputIteratorTransformer +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(20) RegularHashAggregateExecTransformer +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(21) ProjectExecTransformer +Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] +Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(22) WholeStageCodegenTransformer (X) +Input [1]: [promo_revenue#X] +Arguments: false + +(23) VeloxColumnarToRow +Input [1]: [promo_revenue#X] + +(24) Scan parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(25) Filter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(26) Project +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(27) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(28) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : isnotnull(p_partkey#X) + +(29) BroadcastExchange +Input [2]: [p_partkey#X, p_type#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(30) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(31) Project +Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(32) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Keys: [] +Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(33) Exchange +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(34) HashAggregate +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] + +(35) AdaptiveSparkPlan +Output [1]: [promo_revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/15.txt new file mode 100644 index 000000000000..ad3c0445bde5 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/15.txt @@ -0,0 +1,390 @@ +== Physical Plan == +AdaptiveSparkPlan (43) ++- == Final Plan == + VeloxColumnarToRow (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ ScanTransformer parquet (8) ++- == Initial Plan == + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: isnotnull(s_suppkey#X) + +(3) WholeStageCodegenTransformer (X) +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) InputAdapter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(7) InputIteratorTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(8) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(10) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(12) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(13) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(18) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(19) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(20) FilterExecTransformer +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(21) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(22) ProjectExecTransformer +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(23) WholeStageCodegenTransformer (X) +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: false + +(24) VeloxResizeBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X, X + +(25) ColumnarExchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) AQEShuffleRead +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: local + +(28) VeloxColumnarToRow +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] + +(29) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(30) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Condition : isnotnull(s_suppkey#X) + +(31) BroadcastExchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(32) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(33) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(34) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(35) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(36) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(37) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(38) Filter +Input [2]: [supplier_no#X, total_revenue#X] +Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(39) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(40) Project +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(41) Exchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(42) Sort +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 + +(43) AdaptiveSparkPlan +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (68) ++- == Final Plan == + VeloxColumnarToRow (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxResizeBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ ScanTransformer parquet (44) ++- == Initial Plan == + HashAggregate (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) + + +(44) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(46) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(47) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(48) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(49) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(50) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(51) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(52) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(54) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(55) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(56) ProjectExecTransformer +Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] +Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(57) RegularHashAggregateExecTransformer +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(58) WholeStageCodegenTransformer (X) +Input [1]: [max(total_revenue)#X] +Arguments: false + +(59) VeloxColumnarToRow +Input [1]: [max(total_revenue)#X] + +(60) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(61) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(62) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(63) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(64) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(65) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(66) HashAggregate +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [partial_max(total_revenue#X)] +Aggregate Attributes [1]: [max#X] +Results [1]: [max#X] + +(67) HashAggregate +Input [1]: [max#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(68) AdaptiveSparkPlan +Output [1]: [max(total_revenue)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/16.txt new file mode 100644 index 000000000000..3a9b17afd60c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/16.txt @@ -0,0 +1,326 @@ +== Physical Plan == +AdaptiveSparkPlan (59) ++- == Final Plan == + VeloxColumnarToRow (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ ScanTransformer parquet (3) ++- == Initial Plan == + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) + + +(1) ScanTransformer parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: isnotnull(ps_partkey#X) + +(3) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(5) WholeStageCodegenTransformer (X) +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: false + +(6) ColumnarBroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(7) BroadcastQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(8) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(9) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(11) ProjectExecTransformer +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(12) FlushableHashAggregateExecTransformer +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(13) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(14) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: false + +(15) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X, X + +(16) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(18) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(19) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(20) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(21) FlushableHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(22) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(23) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: false + +(24) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X, X + +(25) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(27) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(28) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(29) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(34) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(35) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(36) SortExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(37) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(38) VeloxColumnarToRow +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(39) Scan parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(40) Filter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Condition : isnotnull(ps_partkey#X) + +(41) Scan parquet +Output [2]: [s_suppkey#X, s_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct + +(42) Filter +Input [2]: [s_suppkey#X, s_comment#X] +Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) + +(43) Project +Output [1]: [s_suppkey#X] +Input [2]: [s_suppkey#X, s_comment#X] + +(44) BroadcastExchange +Input [1]: [s_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] + +(45) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: LeftAnti +Join condition: None + +(46) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(47) Filter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(48) BroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(49) BroadcastHashJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(50) Project +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(51) HashAggregate +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(52) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(53) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(54) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(55) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(57) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(58) Sort +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(59) AdaptiveSparkPlan +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/17.txt new file mode 100644 index 000000000000..b66a80731b7b --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/17.txt @@ -0,0 +1,205 @@ +== Physical Plan == +AdaptiveSparkPlan (36) ++- == Final Plan == + VeloxColumnarToRow (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxResizeBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) ++- == Initial Plan == + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) + + +(1) LocalTableScan +Output [1]: [l_extendedprice#X] +Arguments: , [l_extendedprice#X] + +(2) RowToVeloxColumnar +Input [1]: [l_extendedprice#X] + +(3) InputAdapter +Input [1]: [l_extendedprice#X] + +(4) InputIteratorTransformer +Input [1]: [l_extendedprice#X] + +(5) FlushableHashAggregateExecTransformer +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(6) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(7) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(8) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(9) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(10) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(11) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(12) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [sum(l_extendedprice#X)#X] + +(13) ProjectExecTransformer +Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] +Input [1]: [sum(l_extendedprice#X)#X] + +(14) WholeStageCodegenTransformer (X) +Input [1]: [avg_yearly#X] +Arguments: false + +(15) VeloxColumnarToRow +Input [1]: [avg_yearly#X] + +(16) Scan parquet +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(17) Filter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) + +(18) Scan parquet +Output [3]: [p_partkey#X, p_brand#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +ReadSchema: struct + +(19) Filter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) + +(20) Project +Output [1]: [p_partkey#X] +Input [3]: [p_partkey#X, p_brand#X, p_container#X] + +(21) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(22) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(23) Project +Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] + +(24) Scan parquet +Output [2]: [l_partkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(25) Filter +Input [2]: [l_partkey#X, l_quantity#X] +Condition : isnotnull(l_partkey#X) + +(26) HashAggregate +Input [2]: [l_partkey#X, l_quantity#X] +Keys [1]: [l_partkey#X] +Functions [1]: [partial_avg(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [3]: [l_partkey#X, sum#X, count#X] + +(27) Exchange +Input [3]: [l_partkey#X, sum#X, count#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(28) HashAggregate +Input [3]: [l_partkey#X, sum#X, count#X] +Keys [1]: [l_partkey#X] +Functions [1]: [avg(l_quantity#X)] +Aggregate Attributes [1]: [avg(l_quantity#X)#X] +Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] + +(29) Filter +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Condition : isnotnull((0.2 * avg(l_quantity))#X) + +(30) BroadcastExchange +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(31) BroadcastHashJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) + +(32) Project +Output [1]: [l_extendedprice#X] +Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] + +(33) HashAggregate +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(34) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(35) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] + +(36) AdaptiveSparkPlan +Output [1]: [avg_yearly#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/18.txt new file mode 100644 index 000000000000..c3ab86e5acfb --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/18.txt @@ -0,0 +1,488 @@ +== Physical Plan == +AdaptiveSparkPlan (88) ++- == Final Plan == + VeloxColumnarToRow (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxResizeBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ ScanTransformer parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ ScanTransformer parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) ++- == Initial Plan == + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_name#X] +Arguments: isnotnull(c_custkey#X) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_name#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [c_custkey#X, c_name#X] +Arguments: X + +(6) InputAdapter +Input [2]: [c_custkey#X, c_name#X] + +(7) InputIteratorTransformer +Input [2]: [c_custkey#X, c_name#X] + +(8) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(10) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(11) FlushableHashAggregateExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(12) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(13) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(18) InputIteratorTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(19) RegularHashAggregateExecTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(20) FilterExecTransformer +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(21) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(22) WholeStageCodegenTransformer (X) +Input [1]: [l_orderkey#X] +Arguments: false + +(23) ColumnarBroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(24) BroadcastQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(25) InputAdapter +Input [1]: [l_orderkey#X] + +(26) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(27) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(28) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(29) ProjectExecTransformer +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(30) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(31) FilterExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: isnotnull(l_orderkey#X) + +(32) ReusedExchange [Reuses operator id: 23] +Output [1]: [l_orderkey#X] + +(33) BroadcastQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(34) InputAdapter +Input [1]: [l_orderkey#X] + +(35) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(36) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(37) WholeStageCodegenTransformer (X) +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: false + +(38) ColumnarBroadcastExchange +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(39) BroadcastQueryStage +Output [2]: [l_orderkey#X, l_quantity#X] +Arguments: X + +(40) InputAdapter +Input [2]: [l_orderkey#X, l_quantity#X] + +(41) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_quantity#X] + +(42) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(43) ProjectExecTransformer +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(44) FlushableHashAggregateExecTransformer +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(45) ProjectExecTransformer +Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(46) WholeStageCodegenTransformer (X) +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: false + +(47) VeloxResizeBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X, X + +(48) ColumnarExchange +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(49) ShuffleQueryStage +Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(50) InputAdapter +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(51) InputIteratorTransformer +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(52) RegularHashAggregateExecTransformer +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(53) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(54) TakeOrderedAndProjectExecTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 + +(55) VeloxColumnarToRow +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(56) Scan parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(57) Filter +Input [2]: [c_custkey#X, c_name#X] +Condition : isnotnull(c_custkey#X) + +(58) BroadcastExchange +Input [2]: [c_custkey#X, c_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(59) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(60) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(61) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(62) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(63) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(64) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(65) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(66) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(67) BroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(68) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(69) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(70) Project +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(71) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(72) Filter +Input [2]: [l_orderkey#X, l_quantity#X] +Condition : isnotnull(l_orderkey#X) + +(73) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(74) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(75) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(76) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(77) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(78) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(79) BroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(80) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(81) BroadcastExchange +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(82) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(83) Project +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(84) HashAggregate +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(85) Exchange +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(86) HashAggregate +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(87) TakeOrderedAndProject +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(88) AdaptiveSparkPlan +Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/19.txt new file mode 100644 index 000000000000..a93b6458dc62 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/19.txt @@ -0,0 +1,192 @@ +== Physical Plan == +AdaptiveSparkPlan (34) ++- == Final Plan == + VeloxColumnarToRow (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ ScanTransformer parquet (4) ++- == Initial Plan == + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) + + +(1) ScanTransformer parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(2) FilterExecTransformer +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(3) ProjectExecTransformer +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(4) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(5) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(6) WholeStageCodegenTransformer (X) +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: false + +(7) ColumnarBroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(8) BroadcastQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(9) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(10) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(12) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(13) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(14) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(15) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(16) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(18) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(19) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(20) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(21) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(22) VeloxColumnarToRow +Input [1]: [revenue#X] + +(23) Scan parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(24) Filter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(25) Project +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(26) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(27) Filter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(28) BroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(29) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(30) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(31) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(32) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(33) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(34) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/20.txt new file mode 100644 index 000000000000..ff8473e56f04 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/20.txt @@ -0,0 +1,533 @@ +== Physical Plan == +AdaptiveSparkPlan (98) ++- == Final Plan == + VeloxColumnarToRow (62) + +- AQEShuffleRead (61) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (55) + :- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (45) + : :- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (44) + : +- BroadcastQueryStage (42), Statistics(X) + : +- ColumnarBroadcastExchange (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (38) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ FilterExecTransformer (4) + : : : +- ^ ScanTransformer parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ FilterExecTransformer (6) + : : +- ^ ScanTransformer parquet (5) + : +- ^ FilterExecTransformer (37) + : +- ^ ProjectExecTransformer (36) + : +- ^ RegularHashAggregateExecTransformer (35) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- VeloxResizeBatches (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ ScanTransformer parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (54) + +- BroadcastQueryStage (52), Statistics(X) + +- ColumnarBroadcastExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ ScanTransformer parquet (47) ++- == Initial Plan == + Sort (97) + +- Exchange (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (87) + : +- Project (86) + : +- BroadcastHashJoin Inner BuildLeft (85) + : :- BroadcastExchange (72) + : : +- BroadcastHashJoin LeftSemi BuildRight (71) + : : :- Filter (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (70) + : : +- Project (69) + : : +- Filter (68) + : : +- Scan parquet (67) + : +- Filter (84) + : +- HashAggregate (83) + : +- Exchange (82) + : +- HashAggregate (81) + : +- BroadcastHashJoin LeftSemi BuildRight (80) + : :- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: isnotnull(s_nationkey#X) + +(3) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(5) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(6) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(7) ProjectExecTransformer +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(8) WholeStageCodegenTransformer (X) +Input [1]: [p_partkey#X] +Arguments: false + +(9) ColumnarBroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(10) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(11) InputAdapter +Input [1]: [p_partkey#X] + +(12) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(13) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(14) WholeStageCodegenTransformer (X) +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: false + +(15) ColumnarBroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(16) BroadcastQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(18) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(19) ScanTransformer parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(20) FilterExecTransformer +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(21) ProjectExecTransformer +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(22) ReusedExchange [Reuses operator id: 9] +Output [1]: [p_partkey#X] + +(23) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(24) InputAdapter +Input [1]: [p_partkey#X] + +(25) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(26) BroadcastHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(27) FlushableHashAggregateExecTransformer +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(28) ProjectExecTransformer +Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(29) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(30) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(31) ColumnarExchange +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(32) ShuffleQueryStage +Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(33) InputAdapter +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(34) InputIteratorTransformer +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(35) RegularHashAggregateExecTransformer +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(36) ProjectExecTransformer +Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(37) FilterExecTransformer +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: isnotnull((0.5 * sum(l_quantity))#X) + +(38) BroadcastHashJoinExecTransformer +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(39) ProjectExecTransformer +Output [1]: [ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(40) WholeStageCodegenTransformer (X) +Input [1]: [ps_suppkey#X] +Arguments: false + +(41) ColumnarBroadcastExchange +Input [1]: [ps_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(42) BroadcastQueryStage +Output [1]: [ps_suppkey#X] +Arguments: X + +(43) InputAdapter +Input [1]: [ps_suppkey#X] + +(44) InputIteratorTransformer +Input [1]: [ps_suppkey#X] + +(45) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(46) ProjectExecTransformer +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(47) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(48) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(49) ProjectExecTransformer +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(50) WholeStageCodegenTransformer (X) +Input [1]: [n_nationkey#X] +Arguments: false + +(51) ColumnarBroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(52) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(53) InputAdapter +Input [1]: [n_nationkey#X] + +(54) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(55) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(56) ProjectExecTransformer +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(57) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, s_address#X] +Arguments: false + +(58) VeloxResizeBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X, X + +(59) ColumnarExchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(60) ShuffleQueryStage +Output [2]: [s_name#X, s_address#X] +Arguments: X + +(61) AQEShuffleRead +Input [2]: [s_name#X, s_address#X] +Arguments: local + +(62) VeloxColumnarToRow +Input [2]: [s_name#X, s_address#X] + +(63) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(64) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Condition : isnotnull(s_nationkey#X) + +(65) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(66) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(67) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(68) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(69) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(70) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(71) BroadcastHashJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(72) BroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(73) Scan parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(74) Filter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(75) Project +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(76) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(77) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(78) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(79) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(80) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(81) HashAggregate +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(82) Exchange +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(83) HashAggregate +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(84) Filter +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Condition : isnotnull((0.5 * sum(l_quantity))#X) + +(85) BroadcastHashJoin +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(86) Project +Output [1]: [ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(87) BroadcastExchange +Input [1]: [ps_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(88) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(89) Project +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(90) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(91) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(92) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(93) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(94) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(95) Project +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(96) Exchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(97) Sort +Input [2]: [s_name#X, s_address#X] +Arguments: [s_name#X ASC NULLS FIRST], true, 0 + +(98) AdaptiveSparkPlan +Output [2]: [s_name#X, s_address#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/21.txt new file mode 100644 index 000000000000..5fab9fa9da42 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/21.txt @@ -0,0 +1,504 @@ +== Physical Plan == +AdaptiveSparkPlan (92) ++- == Final Plan == + VeloxColumnarToRow (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxResizeBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ ScanTransformer parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ ScanTransformer parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ ScanTransformer parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ ScanTransformer parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ ScanTransformer parquet (39) ++- == Initial Plan == + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) + + +(1) ScanTransformer parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) WholeStageCodegenTransformer (X) +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) InputAdapter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(7) InputIteratorTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(8) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(10) ProjectExecTransformer +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(12) WholeStageCodegenTransformer (X) +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: false + +(13) ColumnarBroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(14) BroadcastQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(16) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(17) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(18) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(19) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(20) ProjectExecTransformer +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(21) WholeStageCodegenTransformer (X) +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: false + +(22) ColumnarBroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(23) BroadcastQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(24) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(25) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(26) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(27) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(28) ProjectExecTransformer +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(29) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(30) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(31) ProjectExecTransformer +Output [1]: [o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(32) WholeStageCodegenTransformer (X) +Input [1]: [o_orderkey#X] +Arguments: false + +(33) ColumnarBroadcastExchange +Input [1]: [o_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(34) BroadcastQueryStage +Output [1]: [o_orderkey#X] +Arguments: X + +(35) InputAdapter +Input [1]: [o_orderkey#X] + +(36) InputIteratorTransformer +Input [1]: [o_orderkey#X] + +(37) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(38) ProjectExecTransformer +Output [2]: [s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(39) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(40) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(41) ProjectExecTransformer +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) +Input [1]: [n_nationkey#X] +Arguments: false + +(43) ColumnarBroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(44) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(45) InputAdapter +Input [1]: [n_nationkey#X] + +(46) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(47) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(48) ProjectExecTransformer +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(49) FlushableHashAggregateExecTransformer +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(50) ProjectExecTransformer +Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] +Input [2]: [s_name#X, count#X] + +(51) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: false + +(52) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X, X + +(53) ColumnarExchange +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(54) ShuffleQueryStage +Output [2]: [s_name#X, count#X] +Arguments: X + +(55) InputAdapter +Input [2]: [s_name#X, count#X] + +(56) InputIteratorTransformer +Input [2]: [s_name#X, count#X] + +(57) RegularHashAggregateExecTransformer +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(58) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, numwait#X] +Arguments: false + +(59) VeloxColumnarToRow +Input [2]: [s_name#X, numwait#X] + +(60) Scan parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(61) Filter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(62) BroadcastExchange +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(63) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(64) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(65) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(66) Scan parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(67) BroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(68) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(69) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(70) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(71) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(72) BroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(73) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(74) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(75) Project +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(76) Scan parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(77) Filter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(78) Project +Output [1]: [o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(79) BroadcastExchange +Input [1]: [o_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(80) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(81) Project +Output [2]: [s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(82) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(83) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(84) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(85) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(86) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(87) Project +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(88) HashAggregate +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(89) Exchange +Input [2]: [s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(90) HashAggregate +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(91) TakeOrderedAndProject +Input [2]: [s_name#X, numwait#X] +Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] + +(92) AdaptiveSparkPlan +Output [2]: [s_name#X, numwait#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/22.txt new file mode 100644 index 000000000000..0c2ce3c56b5a --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/22.txt @@ -0,0 +1,356 @@ +== Physical Plan == +AdaptiveSparkPlan (40) ++- == Final Plan == + VeloxColumnarToRow (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxResizeBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ ScanTransformer parquet (3) ++- == Initial Plan == + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) + + +(1) ScanTransformer parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(3) ScanTransformer parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(4) WholeStageCodegenTransformer (X) +Input [1]: [o_custkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [o_custkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [o_custkey#X] + +(8) InputIteratorTransformer +Input [1]: [o_custkey#X] + +(9) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(10) ProjectExecTransformer +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(11) FlushableHashAggregateExecTransformer +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(12) ProjectExecTransformer +Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(17) InputAdapter +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(18) InputIteratorTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(19) RegularHashAggregateExecTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(20) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(21) VeloxResizeBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X, X + +(22) ColumnarExchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(23) ShuffleQueryStage +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(24) InputAdapter +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(25) InputIteratorTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(26) SortExecTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(27) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(28) VeloxColumnarToRow +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(29) Scan parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(30) Filter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(31) Scan parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(32) BroadcastExchange +Input [1]: [o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(33) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(34) Project +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(35) HashAggregate +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(36) Exchange +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(37) HashAggregate +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(38) Exchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(39) Sort +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(40) AdaptiveSparkPlan +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (60) ++- == Final Plan == + VeloxColumnarToRow (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxResizeBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ ScanTransformer parquet (41) ++- == Initial Plan == + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) + + +(41) ScanTransformer parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(42) FilterExecTransformer +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(43) ProjectExecTransformer +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(44) FlushableHashAggregateExecTransformer +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(45) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, count#X] +Arguments: false + +(46) VeloxResizeBatches +Input [2]: [sum#X, count#X] +Arguments: X, X + +(47) ColumnarExchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(48) ShuffleQueryStage +Output [2]: [sum#X, count#X] +Arguments: X + +(49) InputAdapter +Input [2]: [sum#X, count#X] + +(50) InputIteratorTransformer +Input [2]: [sum#X, count#X] + +(51) RegularHashAggregateExecTransformer +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(52) WholeStageCodegenTransformer (X) +Input [1]: [avg(c_acctbal)#X] +Arguments: false + +(53) VeloxColumnarToRow +Input [1]: [avg(c_acctbal)#X] + +(54) Scan parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(55) Filter +Input [2]: [c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(56) Project +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(57) HashAggregate +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(58) Exchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(59) HashAggregate +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(60) AdaptiveSparkPlan +Output [1]: [avg(c_acctbal)#X] +Arguments: isFinalPlan=true + +Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (60) ++- == Final Plan == + VeloxColumnarToRow (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxResizeBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ ScanTransformer parquet (41) ++- == Initial Plan == + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/3.txt new file mode 100644 index 000000000000..26db3004e2ee --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/3.txt @@ -0,0 +1,298 @@ +== Physical Plan == +AdaptiveSparkPlan (54) ++- == Final Plan == + VeloxColumnarToRow (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxResizeBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ ScanTransformer parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ ScanTransformer parquet (13) ++- == Initial Plan == + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(3) ProjectExecTransformer +Output [1]: [c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [c_custkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [c_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [c_custkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [c_custkey#X] + +(8) InputIteratorTransformer +Input [1]: [c_custkey#X] + +(9) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(13) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(15) ProjectExecTransformer +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(16) WholeStageCodegenTransformer (X) +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(17) ColumnarBroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(18) BroadcastQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(20) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(21) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(22) ProjectExecTransformer +Output [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(23) FlushableHashAggregateExecTransformer +Input [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, _pre_X#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(24) ProjectExecTransformer +Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(25) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: false + +(26) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X, X + +(27) ColumnarExchange +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(28) ShuffleQueryStage +Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(29) InputAdapter +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(30) InputIteratorTransformer +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(31) RegularHashAggregateExecTransformer +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(32) ProjectExecTransformer +Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] +Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(33) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(34) TakeOrderedAndProjectExecTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 + +(35) VeloxColumnarToRow +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(36) Scan parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(37) Filter +Input [2]: [c_custkey#X, c_mktsegment#X] +Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(38) Project +Output [1]: [c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(39) BroadcastExchange +Input [1]: [c_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(40) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(41) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(42) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(43) Project +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(44) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(45) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(46) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(47) BroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(48) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(49) Project +Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(50) HashAggregate +Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(51) Exchange +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(52) HashAggregate +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] + +(53) TakeOrderedAndProject +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(54) AdaptiveSparkPlan +Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/4.txt new file mode 100644 index 000000000000..a77b48298d37 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/4.txt @@ -0,0 +1,248 @@ +== Physical Plan == +AdaptiveSparkPlan (46) ++- == Final Plan == + VeloxColumnarToRow (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxResizeBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ ScanTransformer parquet (4) ++- == Initial Plan == + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) + + +(1) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(3) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(4) ScanTransformer parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(5) FilterExecTransformer +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(6) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(7) WholeStageCodegenTransformer (X) +Input [1]: [l_orderkey#X] +Arguments: false + +(8) ColumnarBroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(9) BroadcastQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(10) InputAdapter +Input [1]: [l_orderkey#X] + +(11) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(12) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(13) ProjectExecTransformer +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(14) FlushableHashAggregateExecTransformer +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(15) ProjectExecTransformer +Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] +Input [2]: [o_orderpriority#X, count#X] + +(16) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: false + +(17) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X, X + +(18) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(19) ShuffleQueryStage +Output [2]: [o_orderpriority#X, count#X] +Arguments: X + +(20) InputAdapter +Input [2]: [o_orderpriority#X, count#X] + +(21) InputIteratorTransformer +Input [2]: [o_orderpriority#X, count#X] + +(22) RegularHashAggregateExecTransformer +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(23) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(24) VeloxResizeBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X, X + +(25) ColumnarExchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(27) InputAdapter +Input [2]: [o_orderpriority#X, order_count#X] + +(28) InputIteratorTransformer +Input [2]: [o_orderpriority#X, order_count#X] + +(29) SortExecTransformer +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(30) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(31) VeloxColumnarToRow +Input [2]: [o_orderpriority#X, order_count#X] + +(32) Scan parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(33) Filter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(34) Project +Output [2]: [o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(35) Scan parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(36) Filter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(37) Project +Output [1]: [l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(38) BroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(39) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(40) Project +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(41) HashAggregate +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(42) Exchange +Input [2]: [o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) HashAggregate +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(44) Exchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(45) Sort +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(46) AdaptiveSparkPlan +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/5.txt new file mode 100644 index 000000000000..60f009173f69 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/5.txt @@ -0,0 +1,552 @@ +== Physical Plan == +AdaptiveSparkPlan (102) ++- == Final Plan == + VeloxColumnarToRow (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxResizeBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxResizeBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ ScanTransformer parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ ScanTransformer parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ ScanTransformer parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ ScanTransformer parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ ScanTransformer parquet (40) ++- == Initial Plan == + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(6) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(7) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(8) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(10) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [2]: [c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(13) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(15) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(18) InputAdapter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(19) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(22) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(23) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(24) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(25) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(26) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(27) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(28) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(29) BroadcastHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(30) ProjectExecTransformer +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(31) ScanTransformer parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(32) FilterExecTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(33) WholeStageCodegenTransformer (X) +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: false + +(34) ColumnarBroadcastExchange +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(35) BroadcastQueryStage +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(36) InputAdapter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(37) InputIteratorTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(38) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(39) ProjectExecTransformer +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(40) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(41) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(42) ProjectExecTransformer +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(43) WholeStageCodegenTransformer (X) +Input [1]: [r_regionkey#X] +Arguments: false + +(44) ColumnarBroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(45) BroadcastQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(46) InputAdapter +Input [1]: [r_regionkey#X] + +(47) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(48) BroadcastHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(49) ProjectExecTransformer +Output [2]: [n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(50) FlushableHashAggregateExecTransformer +Input [2]: [n_name#X, _pre_X#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(51) ProjectExecTransformer +Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(52) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: false + +(53) VeloxResizeBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X, X + +(54) ColumnarExchange +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(55) ShuffleQueryStage +Output [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: X + +(56) InputAdapter +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(57) InputIteratorTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(58) RegularHashAggregateExecTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(59) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(60) VeloxResizeBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X, X + +(61) ColumnarExchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(62) ShuffleQueryStage +Output [2]: [n_name#X, revenue#X] +Arguments: X + +(63) InputAdapter +Input [2]: [n_name#X, revenue#X] + +(64) InputIteratorTransformer +Input [2]: [n_name#X, revenue#X] + +(65) SortExecTransformer +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(66) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(67) VeloxColumnarToRow +Input [2]: [n_name#X, revenue#X] + +(68) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(69) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(70) BroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(71) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(72) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(73) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(74) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(75) Project +Output [2]: [c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(76) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(77) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(78) BroadcastExchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(79) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(80) Project +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(81) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(82) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(83) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(84) BroadcastHashJoin +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(85) Project +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(86) Scan parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(87) Filter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(88) BroadcastExchange +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(89) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(90) Project +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(91) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(92) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(93) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(94) BroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(95) BroadcastHashJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(96) Project +Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(97) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(98) Exchange +Input [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(99) HashAggregate +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(100) Exchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(101) Sort +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(102) AdaptiveSparkPlan +Output [2]: [n_name#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/6.txt new file mode 100644 index 000000000000..b85e63e10655 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/6.txt @@ -0,0 +1,116 @@ +== Physical Plan == +AdaptiveSparkPlan (20) ++- == Final Plan == + VeloxColumnarToRow (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxResizeBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) + + +(1) ScanTransformer parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(3) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * l_discount#X) AS _pre_X#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(5) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(6) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(7) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(8) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(9) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(10) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(11) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(12) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(13) VeloxColumnarToRow +Input [1]: [revenue#X] + +(14) Scan parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(15) Filter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(16) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(17) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(18) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(19) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(20) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/7.txt new file mode 100644 index 000000000000..eaa84cfa46c2 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/7.txt @@ -0,0 +1,514 @@ +== Physical Plan == +AdaptiveSparkPlan (95) ++- == Final Plan == + VeloxColumnarToRow (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxResizeBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxResizeBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ ScanTransformer parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ ScanTransformer parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ ScanTransformer parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ ScanTransformer parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) ++- == Initial Plan == + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) + + +(1) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(7) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(8) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(11) ProjectExecTransformer +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(12) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(13) FilterExecTransformer +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(14) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: false + +(15) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(16) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(19) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(21) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(22) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(23) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: false + +(24) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(25) BroadcastQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(26) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(27) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(28) BroadcastHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(29) ProjectExecTransformer +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(30) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(31) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(32) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(33) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(34) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(35) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(36) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(37) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(38) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(39) ReusedExchange [Reuses operator id: 33] +Output [2]: [n_nationkey#X, n_name#X] + +(40) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(41) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(42) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(43) BroadcastHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(44) ProjectExecTransformer +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(45) FlushableHashAggregateExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(46) ProjectExecTransformer +Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(47) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: false + +(48) VeloxResizeBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(49) ColumnarExchange +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(51) InputAdapter +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(52) InputIteratorTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(53) RegularHashAggregateExecTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(54) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(55) VeloxResizeBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X, X + +(56) ColumnarExchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(57) ShuffleQueryStage +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(58) InputAdapter +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(59) InputIteratorTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(60) SortExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(61) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(62) VeloxColumnarToRow +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(63) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(64) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(65) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(66) Scan parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(67) Filter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(68) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(69) Project +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(70) Scan parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(71) Filter +Input [2]: [o_orderkey#X, o_custkey#X] +Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(72) BroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(73) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(74) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(75) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(76) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(77) BroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(78) BroadcastHashJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(79) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(80) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(81) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(82) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(83) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(84) Project +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(85) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] +ReadSchema: struct + +(86) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) + +(87) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(88) BroadcastHashJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(89) Project +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(90) HashAggregate +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(91) Exchange +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(92) HashAggregate +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(93) Exchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(94) Sort +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(95) AdaptiveSparkPlan +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/8.txt new file mode 100644 index 000000000000..1e76f5d7254e --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/8.txt @@ -0,0 +1,709 @@ +== Physical Plan == +AdaptiveSparkPlan (131) ++- == Final Plan == + VeloxColumnarToRow (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxResizeBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxResizeBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ ScanTransformer parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ ScanTransformer parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ ScanTransformer parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ ScanTransformer parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ ScanTransformer parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ ScanTransformer parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ ScanTransformer parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ ScanTransformer parquet (58) ++- == Initial Plan == + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [p_partkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [p_partkey#X] + +(8) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(9) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(13) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(15) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(18) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(19) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(22) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(23) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(24) WholeStageCodegenTransformer (X) +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: false + +(25) ColumnarBroadcastExchange +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(26) BroadcastQueryStage +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(27) InputAdapter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(28) InputIteratorTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(29) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(30) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(31) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(32) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(33) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: false + +(34) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(35) BroadcastQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(36) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(37) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(38) BroadcastHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(39) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(40) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(41) FilterExecTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(42) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: false + +(43) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(44) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_regionkey#X] +Arguments: X + +(45) InputAdapter +Input [2]: [n_nationkey#X, n_regionkey#X] + +(46) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] + +(47) BroadcastHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(48) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(49) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(50) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(51) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(52) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(53) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(54) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(55) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(56) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(57) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(58) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(59) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(60) ProjectExecTransformer +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(61) WholeStageCodegenTransformer (X) +Input [1]: [r_regionkey#X] +Arguments: false + +(62) ColumnarBroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(63) BroadcastQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(64) InputAdapter +Input [1]: [r_regionkey#X] + +(65) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(66) BroadcastHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(67) ProjectExecTransformer +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(68) FlushableHashAggregateExecTransformer +Input [3]: [o_year#X, volume#X, _pre_X#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(69) ProjectExecTransformer +Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(70) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: false + +(71) VeloxResizeBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X, X + +(72) ColumnarExchange +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(73) ShuffleQueryStage +Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(74) InputAdapter +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(75) InputIteratorTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(76) RegularHashAggregateExecTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(77) ProjectExecTransformer +Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] +Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(78) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(79) VeloxResizeBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X, X + +(80) ColumnarExchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(81) ShuffleQueryStage +Output [2]: [o_year#X, mkt_share#X] +Arguments: X + +(82) InputAdapter +Input [2]: [o_year#X, mkt_share#X] + +(83) InputIteratorTransformer +Input [2]: [o_year#X, mkt_share#X] + +(84) SortExecTransformer +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(85) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(86) VeloxColumnarToRow +Input [2]: [o_year#X, mkt_share#X] + +(87) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(88) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(89) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(90) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(91) Scan parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(92) Filter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(93) BroadcastHashJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(94) Project +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(95) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(96) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(97) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(98) BroadcastHashJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(99) Project +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(100) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(101) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(102) BroadcastExchange +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(103) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(104) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(105) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(106) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(107) BroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(108) BroadcastHashJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(109) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(110) Scan parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(111) Filter +Input [2]: [n_nationkey#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(112) BroadcastExchange +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(113) BroadcastHashJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(114) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(115) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(116) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(117) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(118) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(119) Project +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(120) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(121) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(122) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(123) BroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(124) BroadcastHashJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(125) Project +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(126) HashAggregate +Input [3]: [o_year#X, volume#X, nation#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(127) Exchange +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(128) HashAggregate +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] + +(129) Exchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(130) Sort +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(131) AdaptiveSparkPlan +Output [2]: [o_year#X, mkt_share#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/9.txt new file mode 100644 index 000000000000..8baec4075ad3 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark35/9.txt @@ -0,0 +1,542 @@ +== Physical Plan == +AdaptiveSparkPlan (100) ++- == Final Plan == + VeloxColumnarToRow (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxResizeBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxResizeBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ ScanTransformer parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ ScanTransformer parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ ScanTransformer parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ ScanTransformer parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ ScanTransformer parquet (40) ++- == Initial Plan == + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [p_partkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [p_partkey#X] + +(8) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(9) ScanTransformer parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(13) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(15) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(18) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(19) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(22) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(23) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(24) WholeStageCodegenTransformer (X) +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: false + +(25) ColumnarBroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] + +(26) BroadcastQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(27) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(28) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(29) BroadcastHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(30) ProjectExecTransformer +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(31) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(32) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: isnotnull(o_orderkey#X) + +(33) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: false + +(34) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(35) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_orderdate#X] +Arguments: X + +(36) InputAdapter +Input [2]: [o_orderkey#X, o_orderdate#X] + +(37) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] + +(38) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(39) ProjectExecTransformer +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(40) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(41) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(42) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(43) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(44) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(45) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(46) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(47) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(48) ProjectExecTransformer +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(49) FlushableHashAggregateExecTransformer +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(50) ProjectExecTransformer +Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(51) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: false + +(52) VeloxResizeBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(53) ColumnarExchange +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(54) ShuffleQueryStage +Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(55) InputAdapter +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(56) InputIteratorTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(57) RegularHashAggregateExecTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(58) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(59) VeloxResizeBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X, X + +(60) ColumnarExchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(61) ShuffleQueryStage +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(62) InputAdapter +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(63) InputIteratorTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(64) SortExecTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(65) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(66) VeloxColumnarToRow +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(67) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(68) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(69) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(70) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(71) Scan parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(72) Filter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(73) BroadcastHashJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(74) Project +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(75) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(76) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(77) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(78) BroadcastHashJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(79) Project +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(80) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(81) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(82) BroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] + +(83) BroadcastHashJoin +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(84) Project +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(85) Scan parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(86) Filter +Input [2]: [o_orderkey#X, o_orderdate#X] +Condition : isnotnull(o_orderkey#X) + +(87) BroadcastExchange +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(88) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(89) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(90) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(91) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(92) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(93) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(94) Project +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(95) HashAggregate +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(96) Exchange +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(97) HashAggregate +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(98) Exchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(99) Sort +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(100) AdaptiveSparkPlan +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/1.txt new file mode 100644 index 000000000000..fce60c054ea8 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/1.txt @@ -0,0 +1,162 @@ +== Physical Plan == +AdaptiveSparkPlan (30) ++- == Final Plan == + VeloxColumnarToRow (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxResizeBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) + + +(1) ScanTransformer parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(3) ProjectExecTransformer +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(5) ProjectExecTransformer +Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(6) WholeStageCodegenTransformer (X) +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: false + +(7) VeloxResizeBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X, X + +(8) ColumnarExchange +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(9) ShuffleQueryStage +Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(10) InputAdapter +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(11) InputIteratorTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(12) RegularHashAggregateExecTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(13) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(14) VeloxResizeBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X, X + +(15) ColumnarExchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(17) InputAdapter +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(18) InputIteratorTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(19) SortExecTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(20) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(21) VeloxColumnarToRow +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(22) Scan parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(23) Filter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(24) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(25) HashAggregate +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(26) Exchange +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(27) HashAggregate +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(28) Exchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(29) Sort +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(30) AdaptiveSparkPlan +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/10.txt new file mode 100644 index 000000000000..8154ff50d7e2 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/10.txt @@ -0,0 +1,374 @@ +== Physical Plan == +AdaptiveSparkPlan (68) ++- == Final Plan == + VeloxColumnarToRow (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxResizeBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ ScanTransformer parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ ScanTransformer parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ ScanTransformer parquet (23) ++- == Initial Plan == + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) + + +(1) ScanTransformer parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(5) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(6) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: false + +(7) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(8) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(9) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(10) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(13) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(15) ProjectExecTransformer +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(16) WholeStageCodegenTransformer (X) +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(17) ColumnarBroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(18) BroadcastQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(20) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(21) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(22) ProjectExecTransformer +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(23) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(24) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(25) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(26) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(27) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(28) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(29) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(30) BroadcastHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(31) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(32) FlushableHashAggregateExecTransformer +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, _pre_X#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(33) ProjectExecTransformer +Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(34) WholeStageCodegenTransformer (X) +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: false + +(35) VeloxResizeBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X, X + +(36) ColumnarExchange +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(37) ShuffleQueryStage +Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(38) InputAdapter +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(39) InputIteratorTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(40) RegularHashAggregateExecTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(41) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(42) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(43) TakeOrderedAndProjectExecTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 + +(44) VeloxColumnarToRow +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(45) Scan parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(46) Filter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(47) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(48) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(49) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(50) BroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(51) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(52) Project +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(53) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(54) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(55) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(56) BroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(57) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(58) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(59) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(60) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(61) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(62) BroadcastHashJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(63) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(64) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(65) Exchange +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(66) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(67) TakeOrderedAndProject +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(68) AdaptiveSparkPlan +Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/11.txt new file mode 100644 index 000000000000..1ae5e4c4fa4c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/11.txt @@ -0,0 +1,559 @@ +== Physical Plan == +AdaptiveSparkPlan (60) ++- == Final Plan == + VeloxColumnarToRow (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxResizeBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxResizeBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ ScanTransformer parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ ScanTransformer parquet (12) ++- == Initial Plan == + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) + + +(1) ScanTransformer parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(3) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(5) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(6) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(7) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(11) ProjectExecTransformer +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(12) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(13) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(14) ProjectExecTransformer +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(15) WholeStageCodegenTransformer (X) +Input [1]: [n_nationkey#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(18) InputAdapter +Input [1]: [n_nationkey#X] + +(19) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [2]: [ps_partkey#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(22) FlushableHashAggregateExecTransformer +Input [2]: [ps_partkey#X, _pre_X#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(23) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(24) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: false + +(25) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(26) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(27) ShuffleQueryStage +Output [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(28) InputAdapter +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(29) InputIteratorTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(30) RegularHashAggregateExecTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(31) FilterExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(32) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(33) VeloxResizeBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X, X + +(34) ColumnarExchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(35) ShuffleQueryStage +Output [2]: [ps_partkey#X, value#X] +Arguments: X + +(36) InputAdapter +Input [2]: [ps_partkey#X, value#X] + +(37) InputIteratorTransformer +Input [2]: [ps_partkey#X, value#X] + +(38) SortExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(39) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(40) VeloxColumnarToRow +Input [2]: [ps_partkey#X, value#X] + +(41) Scan parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(42) Filter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(43) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(44) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(45) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(46) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(47) Project +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(48) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(49) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(50) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(51) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(52) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(53) Project +Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(54) HashAggregate +Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(55) Exchange +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) HashAggregate +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(57) Filter +Input [2]: [ps_partkey#X, value#X] +Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(58) Exchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(59) Sort +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(60) AdaptiveSparkPlan +Output [2]: [ps_partkey#X, value#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) ++- == Final Plan == + VeloxColumnarToRow (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxResizeBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ FilterExecTransformer (62) + : : +- ^ ScanTransformer parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) ++- == Initial Plan == + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) ScanTransformer parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(63) ReusedExchange [Reuses operator id: 6] +Output [2]: [s_suppkey#X, s_nationkey#X] + +(64) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(65) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(66) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(67) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(68) ProjectExecTransformer +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(69) ReusedExchange [Reuses operator id: 16] +Output [1]: [n_nationkey#X] + +(70) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(71) InputAdapter +Input [1]: [n_nationkey#X] + +(72) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(73) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(74) ProjectExecTransformer +Output [1]: [(ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(75) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(76) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(77) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(78) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(79) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(80) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(81) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(82) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(83) ProjectExecTransformer +Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(84) WholeStageCodegenTransformer (X) +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: false + +(85) VeloxColumnarToRow +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(86) Scan parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(87) Filter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(88) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(89) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(90) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(91) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(92) Project +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(93) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(94) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(95) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(96) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(97) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(98) Project +Output [2]: [ps_availqty#X, ps_supplycost#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(99) HashAggregate +Input [2]: [ps_availqty#X, ps_supplycost#X] +Keys: [] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(100) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(101) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(102) AdaptiveSparkPlan +Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/12.txt new file mode 100644 index 000000000000..d5dc00e6dc4c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/12.txt @@ -0,0 +1,238 @@ +== Physical Plan == +AdaptiveSparkPlan (44) ++- == Final Plan == + VeloxColumnarToRow (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxResizeBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxResizeBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ ScanTransformer parquet (8) ++- == Initial Plan == + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) + + +(1) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: isnotnull(o_orderkey#X) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) InputAdapter +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(7) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(8) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(10) ProjectExecTransformer +Output [2]: [l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [3]: [l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(13) FlushableHashAggregateExecTransformer +Input [3]: [l_shipmode#X, _pre_X#X, _pre_X#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(14) ProjectExecTransformer +Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(15) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: false + +(16) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X, X + +(17) ColumnarExchange +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [shuffle_writer_type=hash] + +(18) ShuffleQueryStage +Output [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(20) InputIteratorTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(21) RegularHashAggregateExecTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(22) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(23) VeloxResizeBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X, X + +(24) ColumnarExchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(26) InputAdapter +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(27) InputIteratorTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(28) SortExecTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(29) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(30) VeloxColumnarToRow +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(31) Scan parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(32) Filter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Condition : isnotnull(o_orderkey#X) + +(33) BroadcastExchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(34) Scan parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(35) Filter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(36) Project +Output [2]: [l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(37) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(38) Project +Output [2]: [o_orderpriority#X, l_shipmode#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(39) HashAggregate +Input [2]: [o_orderpriority#X, l_shipmode#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(40) Exchange +Input [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(41) HashAggregate +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(42) Exchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) Sort +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(44) AdaptiveSparkPlan +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/13.txt new file mode 100644 index 000000000000..8848ac70c05f --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/13.txt @@ -0,0 +1,299 @@ +== Physical Plan == +AdaptiveSparkPlan (55) ++- == Final Plan == + VeloxColumnarToRow (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxResizeBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxResizeBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ ScanTransformer parquet (2) ++- == Initial Plan == + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) + + +(1) ScanTransformer parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(2) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(3) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(4) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(5) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: false + +(6) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(7) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(9) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(11) ProjectExecTransformer +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(12) FlushableHashAggregateExecTransformer +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [partial_count(o_orderkey#X)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_custkey#X, count#X] + +(13) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] +Input [2]: [c_custkey#X, count#X] + +(14) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: false + +(15) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X, X + +(16) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [2]: [c_custkey#X, count#X] +Arguments: X + +(18) InputAdapter +Input [2]: [c_custkey#X, count#X] + +(19) InputIteratorTransformer +Input [2]: [c_custkey#X, count#X] + +(20) RegularHashAggregateExecTransformer +Input [2]: [c_custkey#X, count#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(21) ProjectExecTransformer +Output [1]: [count(o_orderkey#X)#X AS c_count#X] +Input [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(22) FlushableHashAggregateExecTransformer +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(23) ProjectExecTransformer +Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] +Input [2]: [c_count#X, count#X] + +(24) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: false + +(25) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X, X + +(26) ColumnarExchange +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(27) ShuffleQueryStage +Output [2]: [c_count#X, count#X] +Arguments: X + +(28) InputAdapter +Input [2]: [c_count#X, count#X] + +(29) InputIteratorTransformer +Input [2]: [c_count#X, count#X] + +(30) RegularHashAggregateExecTransformer +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(31) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(32) VeloxResizeBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X, X + +(33) ColumnarExchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(34) ShuffleQueryStage +Output [2]: [c_count#X, custdist#X] +Arguments: X + +(35) InputAdapter +Input [2]: [c_count#X, custdist#X] + +(36) InputIteratorTransformer +Input [2]: [c_count#X, custdist#X] + +(37) SortExecTransformer +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(38) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(39) VeloxColumnarToRow +Input [2]: [c_count#X, custdist#X] + +(40) Scan parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(41) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(42) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(43) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(44) BroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(45) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(46) Project +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(47) HashAggregate +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [partial_count(o_orderkey#X)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_custkey#X, count#X] + +(48) Exchange +Input [2]: [c_custkey#X, count#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(49) HashAggregate +Input [2]: [c_custkey#X, count#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [1]: [count(o_orderkey#X)#X AS c_count#X] + +(50) HashAggregate +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(51) Exchange +Input [2]: [c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(52) HashAggregate +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(53) Exchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(54) Sort +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(55) AdaptiveSparkPlan +Output [2]: [c_count#X, custdist#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/14.txt new file mode 100644 index 000000000000..17ce5be290b0 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/14.txt @@ -0,0 +1,197 @@ +== Physical Plan == +AdaptiveSparkPlan (35) ++- == Final Plan == + VeloxColumnarToRow (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ ScanTransformer parquet (4) ++- == Initial Plan == + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) + + +(1) ScanTransformer parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(3) ProjectExecTransformer +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(5) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: isnotnull(p_partkey#X) + +(6) WholeStageCodegenTransformer (X) +Input [2]: [p_partkey#X, p_type#X] +Arguments: false + +(7) ColumnarBroadcastExchange +Input [2]: [p_partkey#X, p_type#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(8) BroadcastQueryStage +Output [2]: [p_partkey#X, p_type#X] +Arguments: X + +(9) InputAdapter +Input [2]: [p_partkey#X, p_type#X] + +(10) InputIteratorTransformer +Input [2]: [p_partkey#X, p_type#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [2]: [CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(13) FlushableHashAggregateExecTransformer +Input [2]: [_pre_X#X, _pre_X#X] +Keys: [] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(14) WholeStageCodegenTransformer (X) +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: false + +(15) VeloxResizeBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X, X + +(16) ColumnarExchange +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(18) InputAdapter +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(19) InputIteratorTransformer +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(20) RegularHashAggregateExecTransformer +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(21) ProjectExecTransformer +Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] +Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(22) WholeStageCodegenTransformer (X) +Input [1]: [promo_revenue#X] +Arguments: false + +(23) VeloxColumnarToRow +Input [1]: [promo_revenue#X] + +(24) Scan parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(25) Filter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(26) Project +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(27) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(28) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : isnotnull(p_partkey#X) + +(29) BroadcastExchange +Input [2]: [p_partkey#X, p_type#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(30) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(31) Project +Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(32) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Keys: [] +Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(33) Exchange +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(34) HashAggregate +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] + +(35) AdaptiveSparkPlan +Output [1]: [promo_revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/15.txt new file mode 100644 index 000000000000..ad3c0445bde5 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/15.txt @@ -0,0 +1,390 @@ +== Physical Plan == +AdaptiveSparkPlan (43) ++- == Final Plan == + VeloxColumnarToRow (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ ScanTransformer parquet (8) ++- == Initial Plan == + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: isnotnull(s_suppkey#X) + +(3) WholeStageCodegenTransformer (X) +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) InputAdapter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(7) InputIteratorTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(8) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(10) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(12) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(13) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(18) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(19) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(20) FilterExecTransformer +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(21) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(22) ProjectExecTransformer +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(23) WholeStageCodegenTransformer (X) +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: false + +(24) VeloxResizeBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X, X + +(25) ColumnarExchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) AQEShuffleRead +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: local + +(28) VeloxColumnarToRow +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] + +(29) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(30) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Condition : isnotnull(s_suppkey#X) + +(31) BroadcastExchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(32) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(33) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(34) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(35) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(36) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(37) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(38) Filter +Input [2]: [supplier_no#X, total_revenue#X] +Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(39) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(40) Project +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(41) Exchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(42) Sort +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 + +(43) AdaptiveSparkPlan +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (68) ++- == Final Plan == + VeloxColumnarToRow (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxResizeBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ ScanTransformer parquet (44) ++- == Initial Plan == + HashAggregate (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) + + +(44) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(46) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(47) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(48) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(49) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(50) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(51) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(52) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(54) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(55) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(56) ProjectExecTransformer +Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] +Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(57) RegularHashAggregateExecTransformer +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(58) WholeStageCodegenTransformer (X) +Input [1]: [max(total_revenue)#X] +Arguments: false + +(59) VeloxColumnarToRow +Input [1]: [max(total_revenue)#X] + +(60) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(61) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(62) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(63) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(64) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(65) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(66) HashAggregate +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [partial_max(total_revenue#X)] +Aggregate Attributes [1]: [max#X] +Results [1]: [max#X] + +(67) HashAggregate +Input [1]: [max#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(68) AdaptiveSparkPlan +Output [1]: [max(total_revenue)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/16.txt new file mode 100644 index 000000000000..3a9b17afd60c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/16.txt @@ -0,0 +1,326 @@ +== Physical Plan == +AdaptiveSparkPlan (59) ++- == Final Plan == + VeloxColumnarToRow (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ ScanTransformer parquet (3) ++- == Initial Plan == + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) + + +(1) ScanTransformer parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: isnotnull(ps_partkey#X) + +(3) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(5) WholeStageCodegenTransformer (X) +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: false + +(6) ColumnarBroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(7) BroadcastQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(8) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(9) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(11) ProjectExecTransformer +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(12) FlushableHashAggregateExecTransformer +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(13) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(14) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: false + +(15) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X, X + +(16) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(18) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(19) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(20) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(21) FlushableHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(22) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(23) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: false + +(24) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X, X + +(25) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(27) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(28) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(29) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(34) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(35) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(36) SortExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(37) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(38) VeloxColumnarToRow +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(39) Scan parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(40) Filter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Condition : isnotnull(ps_partkey#X) + +(41) Scan parquet +Output [2]: [s_suppkey#X, s_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct + +(42) Filter +Input [2]: [s_suppkey#X, s_comment#X] +Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) + +(43) Project +Output [1]: [s_suppkey#X] +Input [2]: [s_suppkey#X, s_comment#X] + +(44) BroadcastExchange +Input [1]: [s_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] + +(45) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: LeftAnti +Join condition: None + +(46) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(47) Filter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(48) BroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(49) BroadcastHashJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(50) Project +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(51) HashAggregate +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(52) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(53) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(54) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(55) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(57) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(58) Sort +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(59) AdaptiveSparkPlan +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/17.txt new file mode 100644 index 000000000000..b66a80731b7b --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/17.txt @@ -0,0 +1,205 @@ +== Physical Plan == +AdaptiveSparkPlan (36) ++- == Final Plan == + VeloxColumnarToRow (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxResizeBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) ++- == Initial Plan == + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) + + +(1) LocalTableScan +Output [1]: [l_extendedprice#X] +Arguments: , [l_extendedprice#X] + +(2) RowToVeloxColumnar +Input [1]: [l_extendedprice#X] + +(3) InputAdapter +Input [1]: [l_extendedprice#X] + +(4) InputIteratorTransformer +Input [1]: [l_extendedprice#X] + +(5) FlushableHashAggregateExecTransformer +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(6) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(7) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(8) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(9) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(10) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(11) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(12) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [sum(l_extendedprice#X)#X] + +(13) ProjectExecTransformer +Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] +Input [1]: [sum(l_extendedprice#X)#X] + +(14) WholeStageCodegenTransformer (X) +Input [1]: [avg_yearly#X] +Arguments: false + +(15) VeloxColumnarToRow +Input [1]: [avg_yearly#X] + +(16) Scan parquet +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(17) Filter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) + +(18) Scan parquet +Output [3]: [p_partkey#X, p_brand#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +ReadSchema: struct + +(19) Filter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) + +(20) Project +Output [1]: [p_partkey#X] +Input [3]: [p_partkey#X, p_brand#X, p_container#X] + +(21) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(22) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(23) Project +Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] + +(24) Scan parquet +Output [2]: [l_partkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(25) Filter +Input [2]: [l_partkey#X, l_quantity#X] +Condition : isnotnull(l_partkey#X) + +(26) HashAggregate +Input [2]: [l_partkey#X, l_quantity#X] +Keys [1]: [l_partkey#X] +Functions [1]: [partial_avg(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [3]: [l_partkey#X, sum#X, count#X] + +(27) Exchange +Input [3]: [l_partkey#X, sum#X, count#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(28) HashAggregate +Input [3]: [l_partkey#X, sum#X, count#X] +Keys [1]: [l_partkey#X] +Functions [1]: [avg(l_quantity#X)] +Aggregate Attributes [1]: [avg(l_quantity#X)#X] +Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] + +(29) Filter +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Condition : isnotnull((0.2 * avg(l_quantity))#X) + +(30) BroadcastExchange +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] + +(31) BroadcastHashJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) + +(32) Project +Output [1]: [l_extendedprice#X] +Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] + +(33) HashAggregate +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(34) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(35) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] + +(36) AdaptiveSparkPlan +Output [1]: [avg_yearly#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/18.txt new file mode 100644 index 000000000000..c3ab86e5acfb --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/18.txt @@ -0,0 +1,488 @@ +== Physical Plan == +AdaptiveSparkPlan (88) ++- == Final Plan == + VeloxColumnarToRow (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxResizeBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ ScanTransformer parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ ScanTransformer parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) ++- == Initial Plan == + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_name#X] +Arguments: isnotnull(c_custkey#X) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_name#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [c_custkey#X, c_name#X] +Arguments: X + +(6) InputAdapter +Input [2]: [c_custkey#X, c_name#X] + +(7) InputIteratorTransformer +Input [2]: [c_custkey#X, c_name#X] + +(8) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(10) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(11) FlushableHashAggregateExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(12) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(13) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(18) InputIteratorTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(19) RegularHashAggregateExecTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(20) FilterExecTransformer +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(21) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(22) WholeStageCodegenTransformer (X) +Input [1]: [l_orderkey#X] +Arguments: false + +(23) ColumnarBroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(24) BroadcastQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(25) InputAdapter +Input [1]: [l_orderkey#X] + +(26) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(27) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(28) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(29) ProjectExecTransformer +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(30) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(31) FilterExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: isnotnull(l_orderkey#X) + +(32) ReusedExchange [Reuses operator id: 23] +Output [1]: [l_orderkey#X] + +(33) BroadcastQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(34) InputAdapter +Input [1]: [l_orderkey#X] + +(35) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(36) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(37) WholeStageCodegenTransformer (X) +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: false + +(38) ColumnarBroadcastExchange +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(39) BroadcastQueryStage +Output [2]: [l_orderkey#X, l_quantity#X] +Arguments: X + +(40) InputAdapter +Input [2]: [l_orderkey#X, l_quantity#X] + +(41) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_quantity#X] + +(42) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(43) ProjectExecTransformer +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(44) FlushableHashAggregateExecTransformer +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(45) ProjectExecTransformer +Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(46) WholeStageCodegenTransformer (X) +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: false + +(47) VeloxResizeBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X, X + +(48) ColumnarExchange +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(49) ShuffleQueryStage +Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(50) InputAdapter +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(51) InputIteratorTransformer +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(52) RegularHashAggregateExecTransformer +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(53) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(54) TakeOrderedAndProjectExecTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 + +(55) VeloxColumnarToRow +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(56) Scan parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(57) Filter +Input [2]: [c_custkey#X, c_name#X] +Condition : isnotnull(c_custkey#X) + +(58) BroadcastExchange +Input [2]: [c_custkey#X, c_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(59) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(60) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(61) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(62) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(63) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(64) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(65) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(66) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(67) BroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(68) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(69) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(70) Project +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(71) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(72) Filter +Input [2]: [l_orderkey#X, l_quantity#X] +Condition : isnotnull(l_orderkey#X) + +(73) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(74) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(75) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(76) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(77) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(78) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(79) BroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(80) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(81) BroadcastExchange +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(82) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(83) Project +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(84) HashAggregate +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(85) Exchange +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(86) HashAggregate +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(87) TakeOrderedAndProject +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(88) AdaptiveSparkPlan +Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/19.txt new file mode 100644 index 000000000000..a93b6458dc62 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/19.txt @@ -0,0 +1,192 @@ +== Physical Plan == +AdaptiveSparkPlan (34) ++- == Final Plan == + VeloxColumnarToRow (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxResizeBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ ScanTransformer parquet (4) ++- == Initial Plan == + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) + + +(1) ScanTransformer parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(2) FilterExecTransformer +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(3) ProjectExecTransformer +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(4) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(5) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(6) WholeStageCodegenTransformer (X) +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: false + +(7) ColumnarBroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(8) BroadcastQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(9) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(10) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(12) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(13) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(14) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(15) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(16) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(17) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(18) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(19) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(20) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(21) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(22) VeloxColumnarToRow +Input [1]: [revenue#X] + +(23) Scan parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(24) Filter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(25) Project +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(26) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(27) Filter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(28) BroadcastExchange +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(29) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(30) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(31) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(32) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(33) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(34) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/20.txt new file mode 100644 index 000000000000..ff8473e56f04 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/20.txt @@ -0,0 +1,533 @@ +== Physical Plan == +AdaptiveSparkPlan (98) ++- == Final Plan == + VeloxColumnarToRow (62) + +- AQEShuffleRead (61) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (55) + :- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (45) + : :- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (44) + : +- BroadcastQueryStage (42), Statistics(X) + : +- ColumnarBroadcastExchange (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (38) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ FilterExecTransformer (4) + : : : +- ^ ScanTransformer parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ FilterExecTransformer (6) + : : +- ^ ScanTransformer parquet (5) + : +- ^ FilterExecTransformer (37) + : +- ^ ProjectExecTransformer (36) + : +- ^ RegularHashAggregateExecTransformer (35) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- VeloxResizeBatches (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ ScanTransformer parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (54) + +- BroadcastQueryStage (52), Statistics(X) + +- ColumnarBroadcastExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ ScanTransformer parquet (47) ++- == Initial Plan == + Sort (97) + +- Exchange (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (87) + : +- Project (86) + : +- BroadcastHashJoin Inner BuildLeft (85) + : :- BroadcastExchange (72) + : : +- BroadcastHashJoin LeftSemi BuildRight (71) + : : :- Filter (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (70) + : : +- Project (69) + : : +- Filter (68) + : : +- Scan parquet (67) + : +- Filter (84) + : +- HashAggregate (83) + : +- Exchange (82) + : +- HashAggregate (81) + : +- BroadcastHashJoin LeftSemi BuildRight (80) + : :- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: isnotnull(s_nationkey#X) + +(3) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(4) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(5) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(6) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(7) ProjectExecTransformer +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(8) WholeStageCodegenTransformer (X) +Input [1]: [p_partkey#X] +Arguments: false + +(9) ColumnarBroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(10) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(11) InputAdapter +Input [1]: [p_partkey#X] + +(12) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(13) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(14) WholeStageCodegenTransformer (X) +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: false + +(15) ColumnarBroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(16) BroadcastQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(18) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(19) ScanTransformer parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(20) FilterExecTransformer +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(21) ProjectExecTransformer +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(22) ReusedExchange [Reuses operator id: 9] +Output [1]: [p_partkey#X] + +(23) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(24) InputAdapter +Input [1]: [p_partkey#X] + +(25) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(26) BroadcastHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(27) FlushableHashAggregateExecTransformer +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(28) ProjectExecTransformer +Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(29) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(30) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(31) ColumnarExchange +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(32) ShuffleQueryStage +Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(33) InputAdapter +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(34) InputIteratorTransformer +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(35) RegularHashAggregateExecTransformer +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(36) ProjectExecTransformer +Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(37) FilterExecTransformer +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: isnotnull((0.5 * sum(l_quantity))#X) + +(38) BroadcastHashJoinExecTransformer +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(39) ProjectExecTransformer +Output [1]: [ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(40) WholeStageCodegenTransformer (X) +Input [1]: [ps_suppkey#X] +Arguments: false + +(41) ColumnarBroadcastExchange +Input [1]: [ps_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(42) BroadcastQueryStage +Output [1]: [ps_suppkey#X] +Arguments: X + +(43) InputAdapter +Input [1]: [ps_suppkey#X] + +(44) InputIteratorTransformer +Input [1]: [ps_suppkey#X] + +(45) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(46) ProjectExecTransformer +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(47) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(48) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(49) ProjectExecTransformer +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(50) WholeStageCodegenTransformer (X) +Input [1]: [n_nationkey#X] +Arguments: false + +(51) ColumnarBroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(52) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(53) InputAdapter +Input [1]: [n_nationkey#X] + +(54) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(55) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(56) ProjectExecTransformer +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(57) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, s_address#X] +Arguments: false + +(58) VeloxResizeBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X, X + +(59) ColumnarExchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(60) ShuffleQueryStage +Output [2]: [s_name#X, s_address#X] +Arguments: X + +(61) AQEShuffleRead +Input [2]: [s_name#X, s_address#X] +Arguments: local + +(62) VeloxColumnarToRow +Input [2]: [s_name#X, s_address#X] + +(63) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(64) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Condition : isnotnull(s_nationkey#X) + +(65) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(66) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(67) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(68) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(69) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(70) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(71) BroadcastHashJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(72) BroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(73) Scan parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(74) Filter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(75) Project +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(76) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(77) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(78) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(79) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(80) BroadcastHashJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(81) HashAggregate +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(82) Exchange +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(83) HashAggregate +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(84) Filter +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Condition : isnotnull((0.5 * sum(l_quantity))#X) + +(85) BroadcastHashJoin +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(86) Project +Output [1]: [ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(87) BroadcastExchange +Input [1]: [ps_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(88) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(89) Project +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(90) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(91) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(92) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(93) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(94) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(95) Project +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(96) Exchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(97) Sort +Input [2]: [s_name#X, s_address#X] +Arguments: [s_name#X ASC NULLS FIRST], true, 0 + +(98) AdaptiveSparkPlan +Output [2]: [s_name#X, s_address#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/21.txt new file mode 100644 index 000000000000..5fab9fa9da42 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/21.txt @@ -0,0 +1,504 @@ +== Physical Plan == +AdaptiveSparkPlan (92) ++- == Final Plan == + VeloxColumnarToRow (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxResizeBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ ScanTransformer parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ ScanTransformer parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ ScanTransformer parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ ScanTransformer parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ ScanTransformer parquet (39) ++- == Initial Plan == + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) + + +(1) ScanTransformer parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) WholeStageCodegenTransformer (X) +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) InputAdapter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(7) InputIteratorTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(8) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(10) ProjectExecTransformer +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(12) WholeStageCodegenTransformer (X) +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: false + +(13) ColumnarBroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(14) BroadcastQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(16) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(17) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(18) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(19) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(20) ProjectExecTransformer +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(21) WholeStageCodegenTransformer (X) +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: false + +(22) ColumnarBroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(23) BroadcastQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(24) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(25) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(26) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(27) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(28) ProjectExecTransformer +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(29) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(30) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(31) ProjectExecTransformer +Output [1]: [o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(32) WholeStageCodegenTransformer (X) +Input [1]: [o_orderkey#X] +Arguments: false + +(33) ColumnarBroadcastExchange +Input [1]: [o_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(34) BroadcastQueryStage +Output [1]: [o_orderkey#X] +Arguments: X + +(35) InputAdapter +Input [1]: [o_orderkey#X] + +(36) InputIteratorTransformer +Input [1]: [o_orderkey#X] + +(37) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(38) ProjectExecTransformer +Output [2]: [s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(39) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(40) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(41) ProjectExecTransformer +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) +Input [1]: [n_nationkey#X] +Arguments: false + +(43) ColumnarBroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(44) BroadcastQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(45) InputAdapter +Input [1]: [n_nationkey#X] + +(46) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(47) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(48) ProjectExecTransformer +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(49) FlushableHashAggregateExecTransformer +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(50) ProjectExecTransformer +Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] +Input [2]: [s_name#X, count#X] + +(51) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: false + +(52) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X, X + +(53) ColumnarExchange +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(54) ShuffleQueryStage +Output [2]: [s_name#X, count#X] +Arguments: X + +(55) InputAdapter +Input [2]: [s_name#X, count#X] + +(56) InputIteratorTransformer +Input [2]: [s_name#X, count#X] + +(57) RegularHashAggregateExecTransformer +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(58) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, numwait#X] +Arguments: false + +(59) VeloxColumnarToRow +Input [2]: [s_name#X, numwait#X] + +(60) Scan parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(61) Filter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(62) BroadcastExchange +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(63) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(64) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(65) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(66) Scan parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(67) BroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(68) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(69) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(70) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(71) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(72) BroadcastExchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(73) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(74) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(75) Project +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(76) Scan parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(77) Filter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(78) Project +Output [1]: [o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(79) BroadcastExchange +Input [1]: [o_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(80) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(81) Project +Output [2]: [s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(82) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(83) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(84) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(85) BroadcastExchange +Input [1]: [n_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(86) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(87) Project +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(88) HashAggregate +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(89) Exchange +Input [2]: [s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(90) HashAggregate +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(91) TakeOrderedAndProject +Input [2]: [s_name#X, numwait#X] +Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] + +(92) AdaptiveSparkPlan +Output [2]: [s_name#X, numwait#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/22.txt new file mode 100644 index 000000000000..0c2ce3c56b5a --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/22.txt @@ -0,0 +1,356 @@ +== Physical Plan == +AdaptiveSparkPlan (40) ++- == Final Plan == + VeloxColumnarToRow (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxResizeBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ ScanTransformer parquet (3) ++- == Initial Plan == + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) + + +(1) ScanTransformer parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(3) ScanTransformer parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(4) WholeStageCodegenTransformer (X) +Input [1]: [o_custkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [o_custkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [o_custkey#X] + +(8) InputIteratorTransformer +Input [1]: [o_custkey#X] + +(9) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(10) ProjectExecTransformer +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(11) FlushableHashAggregateExecTransformer +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(12) ProjectExecTransformer +Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(17) InputAdapter +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(18) InputIteratorTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(19) RegularHashAggregateExecTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(20) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(21) VeloxResizeBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X, X + +(22) ColumnarExchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(23) ShuffleQueryStage +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(24) InputAdapter +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(25) InputIteratorTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(26) SortExecTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(27) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(28) VeloxColumnarToRow +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(29) Scan parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(30) Filter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(31) Scan parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(32) BroadcastExchange +Input [1]: [o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(33) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(34) Project +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(35) HashAggregate +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(36) Exchange +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(37) HashAggregate +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(38) Exchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(39) Sort +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(40) AdaptiveSparkPlan +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (60) ++- == Final Plan == + VeloxColumnarToRow (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxResizeBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ ScanTransformer parquet (41) ++- == Initial Plan == + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) + + +(41) ScanTransformer parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(42) FilterExecTransformer +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(43) ProjectExecTransformer +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(44) FlushableHashAggregateExecTransformer +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(45) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, count#X] +Arguments: false + +(46) VeloxResizeBatches +Input [2]: [sum#X, count#X] +Arguments: X, X + +(47) ColumnarExchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(48) ShuffleQueryStage +Output [2]: [sum#X, count#X] +Arguments: X + +(49) InputAdapter +Input [2]: [sum#X, count#X] + +(50) InputIteratorTransformer +Input [2]: [sum#X, count#X] + +(51) RegularHashAggregateExecTransformer +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(52) WholeStageCodegenTransformer (X) +Input [1]: [avg(c_acctbal)#X] +Arguments: false + +(53) VeloxColumnarToRow +Input [1]: [avg(c_acctbal)#X] + +(54) Scan parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(55) Filter +Input [2]: [c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(56) Project +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(57) HashAggregate +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(58) Exchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(59) HashAggregate +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(60) AdaptiveSparkPlan +Output [1]: [avg(c_acctbal)#X] +Arguments: isFinalPlan=true + +Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (60) ++- == Final Plan == + VeloxColumnarToRow (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxResizeBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ ScanTransformer parquet (41) ++- == Initial Plan == + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/3.txt new file mode 100644 index 000000000000..26db3004e2ee --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/3.txt @@ -0,0 +1,298 @@ +== Physical Plan == +AdaptiveSparkPlan (54) ++- == Final Plan == + VeloxColumnarToRow (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxResizeBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ ScanTransformer parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ ScanTransformer parquet (13) ++- == Initial Plan == + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(3) ProjectExecTransformer +Output [1]: [c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [c_custkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [c_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [c_custkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [c_custkey#X] + +(8) InputIteratorTransformer +Input [1]: [c_custkey#X] + +(9) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(13) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(15) ProjectExecTransformer +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(16) WholeStageCodegenTransformer (X) +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(17) ColumnarBroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(18) BroadcastQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(20) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(21) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(22) ProjectExecTransformer +Output [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(23) FlushableHashAggregateExecTransformer +Input [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, _pre_X#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(24) ProjectExecTransformer +Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(25) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: false + +(26) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X, X + +(27) ColumnarExchange +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(28) ShuffleQueryStage +Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(29) InputAdapter +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(30) InputIteratorTransformer +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(31) RegularHashAggregateExecTransformer +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(32) ProjectExecTransformer +Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] +Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(33) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(34) TakeOrderedAndProjectExecTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 + +(35) VeloxColumnarToRow +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(36) Scan parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(37) Filter +Input [2]: [c_custkey#X, c_mktsegment#X] +Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(38) Project +Output [1]: [c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(39) BroadcastExchange +Input [1]: [c_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(40) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(41) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(42) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(43) Project +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(44) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(45) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(46) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(47) BroadcastExchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(48) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(49) Project +Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(50) HashAggregate +Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(51) Exchange +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(52) HashAggregate +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] + +(53) TakeOrderedAndProject +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(54) AdaptiveSparkPlan +Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/4.txt new file mode 100644 index 000000000000..a77b48298d37 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/4.txt @@ -0,0 +1,248 @@ +== Physical Plan == +AdaptiveSparkPlan (46) ++- == Final Plan == + VeloxColumnarToRow (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxResizeBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ ScanTransformer parquet (4) ++- == Initial Plan == + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) + + +(1) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(3) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(4) ScanTransformer parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(5) FilterExecTransformer +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(6) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(7) WholeStageCodegenTransformer (X) +Input [1]: [l_orderkey#X] +Arguments: false + +(8) ColumnarBroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(9) BroadcastQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(10) InputAdapter +Input [1]: [l_orderkey#X] + +(11) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(12) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(13) ProjectExecTransformer +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(14) FlushableHashAggregateExecTransformer +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(15) ProjectExecTransformer +Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] +Input [2]: [o_orderpriority#X, count#X] + +(16) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: false + +(17) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X, X + +(18) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(19) ShuffleQueryStage +Output [2]: [o_orderpriority#X, count#X] +Arguments: X + +(20) InputAdapter +Input [2]: [o_orderpriority#X, count#X] + +(21) InputIteratorTransformer +Input [2]: [o_orderpriority#X, count#X] + +(22) RegularHashAggregateExecTransformer +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(23) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(24) VeloxResizeBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X, X + +(25) ColumnarExchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(27) InputAdapter +Input [2]: [o_orderpriority#X, order_count#X] + +(28) InputIteratorTransformer +Input [2]: [o_orderpriority#X, order_count#X] + +(29) SortExecTransformer +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(30) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(31) VeloxColumnarToRow +Input [2]: [o_orderpriority#X, order_count#X] + +(32) Scan parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(33) Filter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(34) Project +Output [2]: [o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(35) Scan parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(36) Filter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(37) Project +Output [1]: [l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(38) BroadcastExchange +Input [1]: [l_orderkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(39) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(40) Project +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(41) HashAggregate +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(42) Exchange +Input [2]: [o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) HashAggregate +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(44) Exchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(45) Sort +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(46) AdaptiveSparkPlan +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/5.txt new file mode 100644 index 000000000000..60f009173f69 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/5.txt @@ -0,0 +1,552 @@ +== Physical Plan == +AdaptiveSparkPlan (102) ++- == Final Plan == + VeloxColumnarToRow (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxResizeBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxResizeBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ ScanTransformer parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ ScanTransformer parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ ScanTransformer parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ ScanTransformer parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ ScanTransformer parquet (40) ++- == Initial Plan == + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(6) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(7) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(8) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(10) ProjectExecTransformer +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [2]: [c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(13) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(15) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(18) InputAdapter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(19) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(22) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(23) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(24) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(25) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(26) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(27) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(28) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(29) BroadcastHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(30) ProjectExecTransformer +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(31) ScanTransformer parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(32) FilterExecTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(33) WholeStageCodegenTransformer (X) +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: false + +(34) ColumnarBroadcastExchange +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(35) BroadcastQueryStage +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(36) InputAdapter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(37) InputIteratorTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(38) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(39) ProjectExecTransformer +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(40) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(41) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(42) ProjectExecTransformer +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(43) WholeStageCodegenTransformer (X) +Input [1]: [r_regionkey#X] +Arguments: false + +(44) ColumnarBroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(45) BroadcastQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(46) InputAdapter +Input [1]: [r_regionkey#X] + +(47) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(48) BroadcastHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(49) ProjectExecTransformer +Output [2]: [n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(50) FlushableHashAggregateExecTransformer +Input [2]: [n_name#X, _pre_X#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(51) ProjectExecTransformer +Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(52) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: false + +(53) VeloxResizeBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X, X + +(54) ColumnarExchange +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(55) ShuffleQueryStage +Output [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: X + +(56) InputAdapter +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(57) InputIteratorTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(58) RegularHashAggregateExecTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(59) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(60) VeloxResizeBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X, X + +(61) ColumnarExchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(62) ShuffleQueryStage +Output [2]: [n_name#X, revenue#X] +Arguments: X + +(63) InputAdapter +Input [2]: [n_name#X, revenue#X] + +(64) InputIteratorTransformer +Input [2]: [n_name#X, revenue#X] + +(65) SortExecTransformer +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(66) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(67) VeloxColumnarToRow +Input [2]: [n_name#X, revenue#X] + +(68) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(69) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(70) BroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(71) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(72) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(73) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(74) BroadcastHashJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(75) Project +Output [2]: [c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(76) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(77) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(78) BroadcastExchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(79) BroadcastHashJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(80) Project +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(81) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(82) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(83) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] + +(84) BroadcastHashJoin +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(85) Project +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(86) Scan parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(87) Filter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(88) BroadcastExchange +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(89) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(90) Project +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(91) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(92) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(93) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(94) BroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(95) BroadcastHashJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(96) Project +Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(97) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(98) Exchange +Input [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(99) HashAggregate +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(100) Exchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(101) Sort +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(102) AdaptiveSparkPlan +Output [2]: [n_name#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/6.txt new file mode 100644 index 000000000000..b85e63e10655 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/6.txt @@ -0,0 +1,116 @@ +== Physical Plan == +AdaptiveSparkPlan (20) ++- == Final Plan == + VeloxColumnarToRow (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxResizeBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) + + +(1) ScanTransformer parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(3) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * l_discount#X) AS _pre_X#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(5) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(6) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(7) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(8) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(9) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(10) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(11) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(12) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(13) VeloxColumnarToRow +Input [1]: [revenue#X] + +(14) Scan parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(15) Filter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(16) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(17) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(18) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(19) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(20) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/7.txt new file mode 100644 index 000000000000..eaa84cfa46c2 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/7.txt @@ -0,0 +1,514 @@ +== Physical Plan == +AdaptiveSparkPlan (95) ++- == Final Plan == + VeloxColumnarToRow (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxResizeBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxResizeBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ ScanTransformer parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ ScanTransformer parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ ScanTransformer parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ ScanTransformer parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) ++- == Initial Plan == + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) + + +(1) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(4) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(5) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(7) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(8) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(9) FilterExecTransformer +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(10) BroadcastHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(11) ProjectExecTransformer +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(12) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(13) FilterExecTransformer +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(14) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: false + +(15) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(16) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(19) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(21) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(22) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(23) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: false + +(24) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(25) BroadcastQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(26) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(27) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(28) BroadcastHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(29) ProjectExecTransformer +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(30) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(31) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(32) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(33) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(34) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(35) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(36) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(37) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(38) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(39) ReusedExchange [Reuses operator id: 33] +Output [2]: [n_nationkey#X, n_name#X] + +(40) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(41) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(42) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(43) BroadcastHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(44) ProjectExecTransformer +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(45) FlushableHashAggregateExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(46) ProjectExecTransformer +Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(47) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: false + +(48) VeloxResizeBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(49) ColumnarExchange +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(51) InputAdapter +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(52) InputIteratorTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(53) RegularHashAggregateExecTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(54) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(55) VeloxResizeBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X, X + +(56) ColumnarExchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(57) ShuffleQueryStage +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(58) InputAdapter +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(59) InputIteratorTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(60) SortExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(61) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(62) VeloxColumnarToRow +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(63) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(64) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(65) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(66) Scan parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(67) Filter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(68) BroadcastHashJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(69) Project +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(70) Scan parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(71) Filter +Input [2]: [o_orderkey#X, o_custkey#X] +Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(72) BroadcastExchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(73) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(74) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(75) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(76) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(77) BroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(78) BroadcastHashJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(79) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(80) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(81) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(82) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(83) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(84) Project +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(85) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] +ReadSchema: struct + +(86) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) + +(87) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(88) BroadcastHashJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(89) Project +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(90) HashAggregate +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(91) Exchange +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(92) HashAggregate +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(93) Exchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(94) Sort +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(95) AdaptiveSparkPlan +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/8.txt new file mode 100644 index 000000000000..1e76f5d7254e --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/8.txt @@ -0,0 +1,709 @@ +== Physical Plan == +AdaptiveSparkPlan (131) ++- == Final Plan == + VeloxColumnarToRow (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxResizeBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxResizeBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ ScanTransformer parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ ScanTransformer parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ ScanTransformer parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ ScanTransformer parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ ScanTransformer parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ ScanTransformer parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ ScanTransformer parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ ScanTransformer parquet (58) ++- == Initial Plan == + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [p_partkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [p_partkey#X] + +(8) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(9) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(13) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(15) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(18) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(19) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(22) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(23) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(24) WholeStageCodegenTransformer (X) +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: false + +(25) ColumnarBroadcastExchange +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(26) BroadcastQueryStage +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(27) InputAdapter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(28) InputIteratorTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(29) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(30) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(31) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(32) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(33) WholeStageCodegenTransformer (X) +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: false + +(34) ColumnarBroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(35) BroadcastQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(36) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(37) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(38) BroadcastHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(39) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(40) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(41) FilterExecTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(42) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: false + +(43) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(44) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_regionkey#X] +Arguments: X + +(45) InputAdapter +Input [2]: [n_nationkey#X, n_regionkey#X] + +(46) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] + +(47) BroadcastHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(48) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(49) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(50) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(51) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(52) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(53) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(54) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(55) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(56) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(57) ProjectExecTransformer +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(58) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(59) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(60) ProjectExecTransformer +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(61) WholeStageCodegenTransformer (X) +Input [1]: [r_regionkey#X] +Arguments: false + +(62) ColumnarBroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(63) BroadcastQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(64) InputAdapter +Input [1]: [r_regionkey#X] + +(65) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(66) BroadcastHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(67) ProjectExecTransformer +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(68) FlushableHashAggregateExecTransformer +Input [3]: [o_year#X, volume#X, _pre_X#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(69) ProjectExecTransformer +Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(70) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: false + +(71) VeloxResizeBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X, X + +(72) ColumnarExchange +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(73) ShuffleQueryStage +Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(74) InputAdapter +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(75) InputIteratorTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(76) RegularHashAggregateExecTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(77) ProjectExecTransformer +Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] +Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(78) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(79) VeloxResizeBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X, X + +(80) ColumnarExchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(81) ShuffleQueryStage +Output [2]: [o_year#X, mkt_share#X] +Arguments: X + +(82) InputAdapter +Input [2]: [o_year#X, mkt_share#X] + +(83) InputIteratorTransformer +Input [2]: [o_year#X, mkt_share#X] + +(84) SortExecTransformer +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(85) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(86) VeloxColumnarToRow +Input [2]: [o_year#X, mkt_share#X] + +(87) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(88) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(89) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(90) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(91) Scan parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(92) Filter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(93) BroadcastHashJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(94) Project +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(95) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(96) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(97) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(98) BroadcastHashJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(99) Project +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(100) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(101) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(102) BroadcastExchange +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(103) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(104) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(105) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(106) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(107) BroadcastExchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(108) BroadcastHashJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(109) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(110) Scan parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(111) Filter +Input [2]: [n_nationkey#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(112) BroadcastExchange +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(113) BroadcastHashJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(114) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(115) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(116) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(117) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(118) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(119) Project +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(120) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(121) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(122) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(123) BroadcastExchange +Input [1]: [r_regionkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(124) BroadcastHashJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(125) Project +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(126) HashAggregate +Input [3]: [o_year#X, volume#X, nation#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(127) Exchange +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(128) HashAggregate +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] + +(129) Exchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(130) Sort +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(131) AdaptiveSparkPlan +Output [2]: [o_year#X, mkt_share#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/9.txt new file mode 100644 index 000000000000..8baec4075ad3 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark35/9.txt @@ -0,0 +1,542 @@ +== Physical Plan == +AdaptiveSparkPlan (100) ++- == Final Plan == + VeloxColumnarToRow (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxResizeBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxResizeBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ ScanTransformer parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ ScanTransformer parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ ScanTransformer parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ ScanTransformer parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ ScanTransformer parquet (40) ++- == Initial Plan == + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [p_partkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [p_partkey#X] + +(8) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(9) ScanTransformer parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(11) BroadcastHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(12) ProjectExecTransformer +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(13) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(14) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(15) WholeStageCodegenTransformer (X) +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: false + +(16) ColumnarBroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(17) BroadcastQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(18) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(19) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(20) BroadcastHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(21) ProjectExecTransformer +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(22) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(23) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(24) WholeStageCodegenTransformer (X) +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: false + +(25) ColumnarBroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] + +(26) BroadcastQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(27) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(28) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(29) BroadcastHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(30) ProjectExecTransformer +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(31) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(32) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: isnotnull(o_orderkey#X) + +(33) WholeStageCodegenTransformer (X) +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: false + +(34) ColumnarBroadcastExchange +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(35) BroadcastQueryStage +Output [2]: [o_orderkey#X, o_orderdate#X] +Arguments: X + +(36) InputAdapter +Input [2]: [o_orderkey#X, o_orderdate#X] + +(37) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] + +(38) BroadcastHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(39) ProjectExecTransformer +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(40) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(41) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(42) WholeStageCodegenTransformer (X) +Input [2]: [n_nationkey#X, n_name#X] +Arguments: false + +(43) ColumnarBroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(44) BroadcastQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(45) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(46) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(47) BroadcastHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(48) ProjectExecTransformer +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(49) FlushableHashAggregateExecTransformer +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(50) ProjectExecTransformer +Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(51) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: false + +(52) VeloxResizeBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(53) ColumnarExchange +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(54) ShuffleQueryStage +Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(55) InputAdapter +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(56) InputIteratorTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(57) RegularHashAggregateExecTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(58) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(59) VeloxResizeBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X, X + +(60) ColumnarExchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(61) ShuffleQueryStage +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(62) InputAdapter +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(63) InputIteratorTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(64) SortExecTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(65) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(66) VeloxColumnarToRow +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(67) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(68) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(69) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(70) BroadcastExchange +Input [1]: [p_partkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] + +(71) Scan parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(72) Filter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(73) BroadcastHashJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(74) Project +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(75) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(76) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(77) BroadcastExchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(78) BroadcastHashJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(79) Project +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(80) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(81) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(82) BroadcastExchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] + +(83) BroadcastHashJoin +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(84) Project +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(85) Scan parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(86) Filter +Input [2]: [o_orderkey#X, o_orderdate#X] +Condition : isnotnull(o_orderkey#X) + +(87) BroadcastExchange +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(88) BroadcastHashJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(89) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(90) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(91) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(92) BroadcastExchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] + +(93) BroadcastHashJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(94) Project +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(95) HashAggregate +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(96) Exchange +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(97) HashAggregate +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(98) Exchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(99) Sort +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(100) AdaptiveSparkPlan +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/1.txt new file mode 100644 index 000000000000..fce60c054ea8 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/1.txt @@ -0,0 +1,162 @@ +== Physical Plan == +AdaptiveSparkPlan (30) ++- == Final Plan == + VeloxColumnarToRow (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxResizeBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) + + +(1) ScanTransformer parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(3) ProjectExecTransformer +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(5) ProjectExecTransformer +Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(6) WholeStageCodegenTransformer (X) +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: false + +(7) VeloxResizeBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X, X + +(8) ColumnarExchange +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(9) ShuffleQueryStage +Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(10) InputAdapter +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(11) InputIteratorTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(12) RegularHashAggregateExecTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(13) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(14) VeloxResizeBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X, X + +(15) ColumnarExchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(17) InputAdapter +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(18) InputIteratorTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(19) SortExecTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(20) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(21) VeloxColumnarToRow +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(22) Scan parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(23) Filter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(24) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(25) HashAggregate +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(26) Exchange +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(27) HashAggregate +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(28) Exchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(29) Sort +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(30) AdaptiveSparkPlan +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/10.txt new file mode 100644 index 000000000000..855ddf391138 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/10.txt @@ -0,0 +1,522 @@ +== Physical Plan == +AdaptiveSparkPlan (100) ++- == Final Plan == + VeloxColumnarToRow (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxResizeBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxResizeBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxResizeBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxResizeBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ ScanTransformer parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxResizeBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ ScanTransformer parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxResizeBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ ScanTransformer parquet (44) ++- == Initial Plan == + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) + + +(1) ScanTransformer parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) ProjectExecTransformer +Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(4) WholeStageCodegenTransformer (X) +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: false + +(5) VeloxResizeBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X, X + +(6) ColumnarExchange +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(8) InputAdapter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(9) InputIteratorTransformer +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(10) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(25) InputAdapter +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] + +(26) InputIteratorTransformer +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] + +(27) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(29) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(34) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(35) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(38) WholeStageCodegenTransformer (X) +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(39) VeloxResizeBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(40) ColumnarExchange +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(42) InputAdapter +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] + +(43) InputIteratorTransformer +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] + +(44) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(46) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(47) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(48) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(49) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(51) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(52) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(55) FlushableHashAggregateExecTransformer +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, _pre_X#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(56) ProjectExecTransformer +Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(57) WholeStageCodegenTransformer (X) +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: false + +(58) VeloxResizeBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X, X + +(59) ColumnarExchange +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(60) ShuffleQueryStage +Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(61) InputAdapter +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(62) InputIteratorTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(63) RegularHashAggregateExecTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(64) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(65) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(66) TakeOrderedAndProjectExecTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 + +(67) VeloxColumnarToRow +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(68) Scan parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(69) Filter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(70) Exchange +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(73) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(74) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(75) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(78) Project +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(79) Exchange +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(82) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(83) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(84) Exchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(87) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(88) Exchange +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(91) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(92) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(95) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(96) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(97) Exchange +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(98) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(99) TakeOrderedAndProject +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(100) AdaptiveSparkPlan +Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/11.txt new file mode 100644 index 000000000000..99539cde1a75 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/11.txt @@ -0,0 +1,709 @@ +== Physical Plan == +AdaptiveSparkPlan (82) ++- == Final Plan == + VeloxColumnarToRow (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxResizeBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxResizeBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxResizeBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ ScanTransformer parquet (27) ++- == Initial Plan == + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) + + +(1) ScanTransformer parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(3) ProjectExecTransformer +Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(8) InputAdapter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(9) InputIteratorTransformer +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(10) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(18) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(25) InputAdapter +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(26) InputIteratorTransformer +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(27) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(29) ProjectExecTransformer +Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(30) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(34) InputAdapter +Input [1]: [n_nationkey#X] + +(35) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [2]: [ps_partkey#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(38) FlushableHashAggregateExecTransformer +Input [2]: [ps_partkey#X, _pre_X#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(39) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(40) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: false + +(41) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(42) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(43) ShuffleQueryStage +Output [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(44) InputAdapter +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(45) InputIteratorTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(46) RegularHashAggregateExecTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(47) FilterExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(48) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(49) VeloxResizeBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X, X + +(50) ColumnarExchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(51) ShuffleQueryStage +Output [2]: [ps_partkey#X, value#X] +Arguments: X + +(52) InputAdapter +Input [2]: [ps_partkey#X, value#X] + +(53) InputIteratorTransformer +Input [2]: [ps_partkey#X, value#X] + +(54) SortExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(55) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(56) VeloxColumnarToRow +Input [2]: [ps_partkey#X, value#X] + +(57) Scan parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(58) Filter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(59) Exchange +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(62) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(63) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(66) Project +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(67) Exchange +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(70) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(71) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(72) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(75) Project +Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(76) HashAggregate +Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(77) Exchange +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(78) HashAggregate +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(79) Filter +Input [2]: [ps_partkey#X, value#X] +Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(80) Exchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(81) Sort +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(82) AdaptiveSparkPlan +Output [2]: [ps_partkey#X, value#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (135) ++- == Final Plan == + VeloxColumnarToRow (113) + +- ^ ProjectExecTransformer (111) + +- ^ RegularHashAggregateExecTransformer (110) + +- ^ ProjectExecTransformer (109) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) + :- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxResizeBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) + : :- ^ InputIteratorTransformer (91) + : : +- ShuffleQueryStage (89), Statistics(X) + : : +- ColumnarExchange (88) + : : +- VeloxResizeBatches (87) + : : +- ^ ProjectExecTransformer (85) + : : +- ^ FilterExecTransformer (84) + : : +- ^ ScanTransformer parquet (83) + : +- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ReusedExchange (92) + +- ^ InputIteratorTransformer (107) + +- ShuffleQueryStage (105), Statistics(X) + +- ReusedExchange (104) ++- == Initial Plan == + HashAggregate (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (117) + : : +- Exchange (116) + : : +- Filter (115) + : : +- Scan parquet (114) + : +- Sort (121) + : +- Exchange (120) + : +- Filter (119) + : +- Scan parquet (118) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) + + +(83) ScanTransformer parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(84) FilterExecTransformer +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(85) ProjectExecTransformer +Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(86) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: false + +(87) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X, X + +(88) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(89) ShuffleQueryStage +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(90) InputAdapter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(91) InputIteratorTransformer +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(92) ReusedExchange [Reuses operator id: 15] +Output [2]: [s_suppkey#X, s_nationkey#X] + +(93) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(94) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(95) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(96) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(97) ProjectExecTransformer +Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(98) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: false + +(99) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X, X + +(100) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(102) InputAdapter +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(103) InputIteratorTransformer +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(104) ReusedExchange [Reuses operator id: 32] +Output [1]: [n_nationkey#X] + +(105) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(106) InputAdapter +Input [1]: [n_nationkey#X] + +(107) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(108) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(109) ProjectExecTransformer +Output [1]: [(ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(110) RegularHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [sum(_pre_X#X)] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(111) ProjectExecTransformer +Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(112) WholeStageCodegenTransformer (X) +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: false + +(113) VeloxColumnarToRow +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(114) Scan parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(115) Filter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(116) Exchange +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(117) Sort +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(118) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(119) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(120) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(121) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(123) Project +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(124) Exchange +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(125) Sort +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(127) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(128) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(129) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(132) Project +Output [2]: [ps_availqty#X, ps_supplycost#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(133) HashAggregate +Input [2]: [ps_availqty#X, ps_supplycost#X] +Keys: [] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(134) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(135) AdaptiveSparkPlan +Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/12.txt new file mode 100644 index 000000000000..9cb9c145f1bb --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/12.txt @@ -0,0 +1,289 @@ +== Physical Plan == +AdaptiveSparkPlan (55) ++- == Final Plan == + VeloxColumnarToRow (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) + + +(1) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: isnotnull(o_orderkey#X) + +(3) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: X + +(8) InputAdapter +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(9) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(10) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_shipmode#X] +Arguments: X + +(17) InputAdapter +Input [2]: [l_orderkey#X, l_shipmode#X] + +(18) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_shipmode#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [3]: [l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(21) FlushableHashAggregateExecTransformer +Input [3]: [l_shipmode#X, _pre_X#X, _pre_X#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(22) ProjectExecTransformer +Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(23) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: false + +(24) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X, X + +(25) ColumnarExchange +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: X + +(27) InputAdapter +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(28) InputIteratorTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(29) RegularHashAggregateExecTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(34) InputAdapter +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(35) InputIteratorTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(36) SortExecTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(37) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(38) VeloxColumnarToRow +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(39) Scan parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(40) Filter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Condition : isnotnull(o_orderkey#X) + +(41) Exchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(44) Filter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(45) Project +Output [2]: [l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(46) Exchange +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(49) Project +Output [2]: [o_orderpriority#X, l_shipmode#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(50) HashAggregate +Input [2]: [o_orderpriority#X, l_shipmode#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(51) Exchange +Input [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(52) HashAggregate +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(53) Exchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(54) Sort +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(55) AdaptiveSparkPlan +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/13.txt new file mode 100644 index 000000000000..59abfb682c42 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/13.txt @@ -0,0 +1,306 @@ +== Physical Plan == +AdaptiveSparkPlan (57) ++- == Final Plan == + VeloxColumnarToRow (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxResizeBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxResizeBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxResizeBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxResizeBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ ScanTransformer parquet (9) ++- == Initial Plan == + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin LeftOuter (48) + :- Sort (42) + : +- Exchange (41) + : +- Scan parquet (40) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) + + +(1) ScanTransformer parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(2) ProjectExecTransformer +Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] +Input [1]: [c_custkey#X] + +(3) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: false + +(4) VeloxResizeBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X, X + +(5) ColumnarExchange +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(6) ShuffleQueryStage +Output [1]: [c_custkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [c_custkey#X] + +(8) InputIteratorTransformer +Input [1]: [c_custkey#X] + +(9) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(11) ProjectExecTransformer +Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(12) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(13) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(14) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(15) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(16) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(17) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(19) ProjectExecTransformer +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(20) RegularHashAggregateExecTransformer +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(21) ProjectExecTransformer +Output [1]: [count(o_orderkey#X)#X AS c_count#X] +Input [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(22) FlushableHashAggregateExecTransformer +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(23) ProjectExecTransformer +Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] +Input [2]: [c_count#X, count#X] + +(24) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: false + +(25) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X, X + +(26) ColumnarExchange +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(27) ShuffleQueryStage +Output [2]: [c_count#X, count#X] +Arguments: X + +(28) InputAdapter +Input [2]: [c_count#X, count#X] + +(29) InputIteratorTransformer +Input [2]: [c_count#X, count#X] + +(30) RegularHashAggregateExecTransformer +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(31) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(32) VeloxResizeBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X, X + +(33) ColumnarExchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(34) ShuffleQueryStage +Output [2]: [c_count#X, custdist#X] +Arguments: X + +(35) InputAdapter +Input [2]: [c_count#X, custdist#X] + +(36) InputIteratorTransformer +Input [2]: [c_count#X, custdist#X] + +(37) SortExecTransformer +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(38) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(39) VeloxColumnarToRow +Input [2]: [c_count#X, custdist#X] + +(40) Scan parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(41) Exchange +Input [1]: [c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(42) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(44) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(45) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(46) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(47) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(49) Project +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(50) HashAggregate +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [partial_count(o_orderkey#X)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_custkey#X, count#X] + +(51) HashAggregate +Input [2]: [c_custkey#X, count#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [1]: [count(o_orderkey#X)#X AS c_count#X] + +(52) HashAggregate +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(53) Exchange +Input [2]: [c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(54) HashAggregate +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(55) Exchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) Sort +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(57) AdaptiveSparkPlan +Output [2]: [c_count#X, custdist#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/14.txt new file mode 100644 index 000000000000..058ec94ab093 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/14.txt @@ -0,0 +1,209 @@ +== Physical Plan == +AdaptiveSparkPlan (38) ++- == Final Plan == + VeloxColumnarToRow (24) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) + + +(1) ScanTransformer parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(3) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(8) InputAdapter +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] + +(9) InputIteratorTransformer +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] + +(10) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: isnotnull(p_partkey#X) + +(12) ProjectExecTransformer +Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] +Input [2]: [p_partkey#X, p_type#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [p_partkey#X, p_type#X] +Arguments: X + +(17) InputAdapter +Input [2]: [p_partkey#X, p_type#X] + +(18) InputIteratorTransformer +Input [2]: [p_partkey#X, p_type#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [2]: [CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(21) RegularHashAggregateExecTransformer +Input [2]: [_pre_X#X, _pre_X#X] +Keys: [] +Functions [2]: [sum(_pre_X#X), sum(_pre_X#X)] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(22) ProjectExecTransformer +Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] +Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(23) WholeStageCodegenTransformer (X) +Input [1]: [promo_revenue#X] +Arguments: false + +(24) VeloxColumnarToRow +Input [1]: [promo_revenue#X] + +(25) Scan parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(26) Filter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(27) Project +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(28) Exchange +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(29) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(31) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : isnotnull(p_partkey#X) + +(32) Exchange +Input [2]: [p_partkey#X, p_type#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(33) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(35) Project +Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(36) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Keys: [] +Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(37) HashAggregate +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] + +(38) AdaptiveSparkPlan +Output [1]: [promo_revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/15.txt new file mode 100644 index 000000000000..e0c0204c0b87 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/15.txt @@ -0,0 +1,410 @@ +== Physical Plan == +AdaptiveSparkPlan (47) ++- == Final Plan == + VeloxColumnarToRow (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxResizeBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxResizeBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (46) + +- Exchange (45) + +- Project (44) + +- SortMergeJoin Inner (43) + :- Sort (34) + : +- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Sort (42) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: isnotnull(s_suppkey#X) + +(3) ProjectExecTransformer +Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(8) InputAdapter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(9) InputIteratorTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(10) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(12) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(13) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(14) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(15) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(16) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(17) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(18) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(20) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(21) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(22) FilterExecTransformer +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(23) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(24) ProjectExecTransformer +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(25) WholeStageCodegenTransformer (X) +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: false + +(26) VeloxResizeBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X, X + +(27) ColumnarExchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(28) ShuffleQueryStage +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(29) AQEShuffleRead +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: local + +(30) VeloxColumnarToRow +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] + +(31) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(32) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Condition : isnotnull(s_suppkey#X) + +(33) Exchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(34) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(35) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(36) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(37) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(38) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(39) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(40) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(41) Filter +Input [2]: [supplier_no#X, total_revenue#X] +Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(42) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(43) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(44) Project +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(45) Exchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(46) Sort +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 + +(47) AdaptiveSparkPlan +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (72) ++- == Final Plan == + VeloxColumnarToRow (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ ProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxResizeBatches (54) + +- ^ ProjectExecTransformer (52) + +- ^ FlushableHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ FilterExecTransformer (49) + +- ^ ScanTransformer parquet (48) ++- == Initial Plan == + HashAggregate (71) + +- HashAggregate (70) + +- HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) + + +(48) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(49) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(50) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(51) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(52) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(53) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(54) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(55) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(56) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(57) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(58) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(59) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(60) ProjectExecTransformer +Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] +Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(61) RegularHashAggregateExecTransformer +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(62) WholeStageCodegenTransformer (X) +Input [1]: [max(total_revenue)#X] +Arguments: false + +(63) VeloxColumnarToRow +Input [1]: [max(total_revenue)#X] + +(64) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(65) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(66) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(67) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(68) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(69) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(70) HashAggregate +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [partial_max(total_revenue#X)] +Aggregate Attributes [1]: [max#X] +Results [1]: [max#X] + +(71) HashAggregate +Input [1]: [max#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(72) AdaptiveSparkPlan +Output [1]: [max(total_revenue)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/16.txt new file mode 100644 index 000000000000..3441216f140f --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/16.txt @@ -0,0 +1,382 @@ +== Physical Plan == +AdaptiveSparkPlan (71) ++- == Final Plan == + VeloxColumnarToRow (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxResizeBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxResizeBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) + + +(1) ScanTransformer parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: isnotnull(ps_partkey#X) + +(3) ProjectExecTransformer +Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Input [2]: [ps_partkey#X, ps_suppkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [ps_partkey#X, ps_suppkey#X] + +(9) InputIteratorTransformer +Input [2]: [ps_partkey#X, ps_suppkey#X] + +(10) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(12) ProjectExecTransformer +Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(17) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(18) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(21) FlushableHashAggregateExecTransformer +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(22) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(23) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: false + +(24) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X, X + +(25) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(27) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(28) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(29) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(30) FlushableHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(31) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(32) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: false + +(33) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X, X + +(34) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(35) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(36) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(37) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(38) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(39) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(40) VeloxResizeBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X, X + +(41) ColumnarExchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(42) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(43) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(44) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(45) SortExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(46) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(47) VeloxColumnarToRow +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(48) Scan parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(49) Filter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Condition : isnotnull(ps_partkey#X) + +(50) Scan parquet +Output [2]: [s_suppkey#X, s_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct + +(51) Filter +Input [2]: [s_suppkey#X, s_comment#X] +Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) + +(52) Project +Output [1]: [s_suppkey#X] +Input [2]: [s_suppkey#X, s_comment#X] + +(53) BroadcastExchange +Input [1]: [s_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] + +(54) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: LeftAnti +Join condition: None + +(55) Exchange +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(58) Filter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(59) Exchange +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(62) Project +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(63) HashAggregate +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(64) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(65) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(66) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(67) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(68) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(69) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(70) Sort +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(71) AdaptiveSparkPlan +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/17.txt new file mode 100644 index 000000000000..c1b43b7fc421 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/17.txt @@ -0,0 +1,347 @@ +== Physical Plan == +AdaptiveSparkPlan (62) ++- == Final Plan == + VeloxColumnarToRow (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxResizeBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ ScanTransformer parquet (21) ++- == Initial Plan == + HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- SortMergeJoin Inner (58) + :- Project (50) + : +- SortMergeJoin Inner (49) + : :- Sort (43) + : : +- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Sort (48) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Sort (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Filter (52) + +- Scan parquet (51) + + +(1) ScanTransformer parquet +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) + +(3) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(8) InputAdapter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(9) InputIteratorTransformer +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(10) ScanTransformer parquet +Output [3]: [p_partkey#X, p_brand#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) + +(12) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [3]: [p_partkey#X, p_brand#X, p_container#X] + +(13) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(17) InputAdapter +Input [1]: [p_partkey#X] + +(18) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] + +(21) ScanTransformer parquet +Output [2]: [l_partkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(22) FilterExecTransformer +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: isnotnull(l_partkey#X) + +(23) FlushableHashAggregateExecTransformer +Input [2]: [l_partkey#X, l_quantity#X] +Keys [1]: [l_partkey#X] +Functions [1]: [partial_avg(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [3]: [l_partkey#X, sum#X, count#X] + +(24) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] +Input [3]: [l_partkey#X, sum#X, count#X] + +(25) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: false + +(26) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X, X + +(27) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(28) ShuffleQueryStage +Output [3]: [l_partkey#X, sum#X, count#X] +Arguments: X + +(29) InputAdapter +Input [3]: [l_partkey#X, sum#X, count#X] + +(30) InputIteratorTransformer +Input [3]: [l_partkey#X, sum#X, count#X] + +(31) RegularHashAggregateExecTransformer +Input [3]: [l_partkey#X, sum#X, count#X] +Keys [1]: [l_partkey#X] +Functions [1]: [avg(l_quantity#X)] +Aggregate Attributes [1]: [avg(l_quantity#X)#X] +Results [2]: [l_partkey#X, avg(l_quantity#X)#X] + +(32) ProjectExecTransformer +Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] +Input [2]: [l_partkey#X, avg(l_quantity#X)#X] + +(33) FilterExecTransformer +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: isnotnull((0.2 * avg(l_quantity))#X) + +(34) ShuffledHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) + +(35) ProjectExecTransformer +Output [1]: [l_extendedprice#X] +Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] + +(36) RegularHashAggregateExecTransformer +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [sum(l_extendedprice#X)#X] + +(37) ProjectExecTransformer +Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] +Input [1]: [sum(l_extendedprice#X)#X] + +(38) WholeStageCodegenTransformer (X) +Input [1]: [avg_yearly#X] +Arguments: false + +(39) VeloxColumnarToRow +Input [1]: [avg_yearly#X] + +(40) Scan parquet +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(41) Filter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) + +(42) Exchange +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet +Output [3]: [p_partkey#X, p_brand#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +ReadSchema: struct + +(45) Filter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) + +(46) Project +Output [1]: [p_partkey#X] +Input [3]: [p_partkey#X, p_brand#X, p_container#X] + +(47) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(48) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(50) Project +Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] + +(51) Scan parquet +Output [2]: [l_partkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(52) Filter +Input [2]: [l_partkey#X, l_quantity#X] +Condition : isnotnull(l_partkey#X) + +(53) HashAggregate +Input [2]: [l_partkey#X, l_quantity#X] +Keys [1]: [l_partkey#X] +Functions [1]: [partial_avg(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [3]: [l_partkey#X, sum#X, count#X] + +(54) Exchange +Input [3]: [l_partkey#X, sum#X, count#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(55) HashAggregate +Input [3]: [l_partkey#X, sum#X, count#X] +Keys [1]: [l_partkey#X] +Functions [1]: [avg(l_quantity#X)] +Aggregate Attributes [1]: [avg(l_quantity#X)#X] +Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] + +(56) Filter +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Condition : isnotnull((0.2 * avg(l_quantity))#X) + +(57) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(58) SortMergeJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) + +(59) Project +Output [1]: [l_extendedprice#X] +Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] + +(60) HashAggregate +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(61) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] + +(62) AdaptiveSparkPlan +Output [1]: [avg_yearly#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/18.txt new file mode 100644 index 000000000000..c1a6b181bb34 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/18.txt @@ -0,0 +1,589 @@ +== Physical Plan == +AdaptiveSparkPlan (109) ++- == Final Plan == + VeloxColumnarToRow (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxResizeBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxResizeBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxResizeBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ ScanTransformer parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxResizeBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ ScanTransformer parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxResizeBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ScanTransformer parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) ++- == Initial Plan == + TakeOrderedAndProject (108) + +- HashAggregate (107) + +- HashAggregate (106) + +- Project (105) + +- SortMergeJoin Inner (104) + :- Sort (91) + : +- Exchange (90) + : +- Project (89) + : +- SortMergeJoin Inner (88) + : :- Sort (73) + : : +- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Sort (87) + : +- Exchange (86) + : +- SortMergeJoin LeftSemi (85) + : :- Sort (77) + : : +- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Sort (84) + : +- Project (83) + : +- Filter (82) + : +- HashAggregate (81) + : +- Exchange (80) + : +- HashAggregate (79) + : +- Scan parquet (78) + +- SortMergeJoin LeftSemi (103) + :- Sort (95) + : +- Exchange (94) + : +- Filter (93) + : +- Scan parquet (92) + +- Sort (102) + +- Project (101) + +- Filter (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Scan parquet (96) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_name#X] +Arguments: isnotnull(c_custkey#X) + +(3) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] +Input [2]: [c_custkey#X, c_name#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [c_custkey#X, c_name#X] +Arguments: X + +(8) InputAdapter +Input [2]: [c_custkey#X, c_name#X] + +(9) InputIteratorTransformer +Input [2]: [c_custkey#X, c_name#X] + +(10) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(17) InputAdapter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(18) InputIteratorTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(19) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(20) FlushableHashAggregateExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(21) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(22) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: false + +(23) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(24) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) InputAdapter +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(27) InputIteratorTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(28) RegularHashAggregateExecTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(29) FilterExecTransformer +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(30) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(31) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(32) ProjectExecTransformer +Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: false + +(34) VeloxResizeBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X, X + +(35) ColumnarExchange +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(36) ShuffleQueryStage +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(37) InputAdapter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(38) InputIteratorTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(39) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(40) ProjectExecTransformer +Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(41) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: false + +(42) VeloxResizeBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X, X + +(43) ColumnarExchange +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(44) ShuffleQueryStage +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(45) InputAdapter +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] + +(46) InputIteratorTransformer +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] + +(47) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(48) FilterExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: isnotnull(l_orderkey#X) + +(49) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] +Input [2]: [l_orderkey#X, l_quantity#X] + +(50) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: false + +(51) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X, X + +(52) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [shuffle_writer_type=hash] + +(53) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_quantity#X] +Arguments: X + +(54) InputAdapter +Input [2]: [l_orderkey#X, l_quantity#X] + +(55) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_quantity#X] + +(56) ReusedExchange [Reuses operator id: 24] +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(57) ShuffleQueryStage +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(58) InputAdapter +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(59) InputIteratorTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(60) RegularHashAggregateExecTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(61) FilterExecTransformer +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(62) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(63) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(64) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(65) ProjectExecTransformer +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(66) RegularHashAggregateExecTransformer +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(67) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(68) TakeOrderedAndProjectExecTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 + +(69) VeloxColumnarToRow +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(70) Scan parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(71) Filter +Input [2]: [c_custkey#X, c_name#X] +Condition : isnotnull(c_custkey#X) + +(72) Exchange +Input [2]: [c_custkey#X, c_name#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(73) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(74) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(75) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(76) Exchange +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(77) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(78) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(79) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(80) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(81) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(82) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(83) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(84) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(85) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(86) Exchange +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(87) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(88) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(89) Project +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(90) Exchange +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(91) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(92) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(93) Filter +Input [2]: [l_orderkey#X, l_quantity#X] +Condition : isnotnull(l_orderkey#X) + +(94) Exchange +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(95) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(97) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(98) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(99) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(100) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(101) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(102) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(103) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(104) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(105) Project +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(106) HashAggregate +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(107) HashAggregate +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(108) TakeOrderedAndProject +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(109) AdaptiveSparkPlan +Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/19.txt new file mode 100644 index 000000000000..420c980f3273 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/19.txt @@ -0,0 +1,204 @@ +== Physical Plan == +AdaptiveSparkPlan (37) ++- == Final Plan == + VeloxColumnarToRow (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- SortMergeJoin Inner (33) + :- Sort (28) + : +- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Sort (32) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) + + +(1) ScanTransformer parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(2) FilterExecTransformer +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(3) ProjectExecTransformer +Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(8) InputAdapter +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(9) InputIteratorTransformer +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(10) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(12) ProjectExecTransformer +Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(17) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(18) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(20) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(21) RegularHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [sum(_pre_X#X)] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(22) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(23) VeloxColumnarToRow +Input [1]: [revenue#X] + +(24) Scan parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(25) Filter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(26) Project +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(27) Exchange +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(28) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(29) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(30) Filter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(31) Exchange +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(32) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(33) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(34) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(35) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(36) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(37) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/20.txt new file mode 100644 index 000000000000..4d8eee1f1fb7 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/20.txt @@ -0,0 +1,734 @@ +== Physical Plan == +AdaptiveSparkPlan (142) ++- == Final Plan == + VeloxColumnarToRow (92) + +- AQEShuffleRead (91) + +- ShuffleQueryStage (90), Statistics(X) + +- ColumnarExchange (89) + +- VeloxResizeBatches (88) + +- ^ ProjectExecTransformer (86) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) + :- ^ InputIteratorTransformer (75) + : +- ShuffleQueryStage (73), Statistics(X) + : +- ColumnarExchange (72) + : +- VeloxResizeBatches (71) + : +- ^ ProjectExecTransformer (69) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (68) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (67) + : +- ShuffleQueryStage (65), Statistics(X) + : +- ColumnarExchange (64) + : +- VeloxResizeBatches (63) + : +- ^ ProjectExecTransformer (61) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (60) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxResizeBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxResizeBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ ScanTransformer parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxResizeBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ ScanTransformer parquet (19) + : +- ^ InputIteratorTransformer (59) + : +- ShuffleQueryStage (57), Statistics(X) + : +- ColumnarExchange (56) + : +- VeloxResizeBatches (55) + : +- ^ ProjectExecTransformer (53) + : +- ^ FilterExecTransformer (52) + : +- ^ ProjectExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxResizeBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ ScanTransformer parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (84) + +- ShuffleQueryStage (82), Statistics(X) + +- ColumnarExchange (81) + +- VeloxResizeBatches (80) + +- ^ ProjectExecTransformer (78) + +- ^ FilterExecTransformer (77) + +- ^ ScanTransformer parquet (76) ++- == Initial Plan == + Sort (141) + +- Exchange (140) + +- Project (139) + +- SortMergeJoin Inner (138) + :- Sort (132) + : +- Exchange (131) + : +- Project (130) + : +- SortMergeJoin LeftSemi (129) + : :- Sort (96) + : : +- Exchange (95) + : : +- Filter (94) + : : +- Scan parquet (93) + : +- Sort (128) + : +- Exchange (127) + : +- Project (126) + : +- SortMergeJoin Inner (125) + : :- Sort (108) + : : +- Exchange (107) + : : +- SortMergeJoin LeftSemi (106) + : : :- Sort (100) + : : : +- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Sort (105) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Sort (124) + : +- Exchange (123) + : +- Filter (122) + : +- HashAggregate (121) + : +- HashAggregate (120) + : +- SortMergeJoin LeftSemi (119) + : :- Sort (113) + : : +- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Sort (118) + : +- Exchange (117) + : +- Project (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Sort (137) + +- Exchange (136) + +- Project (135) + +- Filter (134) + +- Scan parquet (133) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: isnotnull(s_nationkey#X) + +(3) ProjectExecTransformer +Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(10) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(12) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(13) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(18) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(19) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(20) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(21) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(22) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(23) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(24) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(26) InputAdapter +Input [1]: [p_partkey#X] + +(27) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(28) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(29) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(34) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(35) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(36) ScanTransformer parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(37) FilterExecTransformer +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(38) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(39) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: false + +(40) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X, X + +(41) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [shuffle_writer_type=hash] + +(42) ShuffleQueryStage +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(43) InputAdapter +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] + +(44) InputIteratorTransformer +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] + +(45) ReusedExchange [Reuses operator id: 24] +Output [1]: [p_partkey#X] + +(46) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(47) InputAdapter +Input [1]: [p_partkey#X] + +(48) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(49) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(50) RegularHashAggregateExecTransformer +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(51) ProjectExecTransformer +Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(52) FilterExecTransformer +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: isnotnull((0.5 * sum(l_quantity))#X) + +(53) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(54) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: false + +(55) VeloxResizeBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X, X + +(56) ColumnarExchange +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(57) ShuffleQueryStage +Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(58) InputAdapter +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(59) InputIteratorTransformer +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(60) ShuffledHashJoinExecTransformer +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(61) ProjectExecTransformer +Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(62) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: false + +(63) VeloxResizeBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X, X + +(64) ColumnarExchange +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(65) ShuffleQueryStage +Output [1]: [ps_suppkey#X] +Arguments: X + +(66) InputAdapter +Input [1]: [ps_suppkey#X] + +(67) InputIteratorTransformer +Input [1]: [ps_suppkey#X] + +(68) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(69) ProjectExecTransformer +Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(70) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: false + +(71) VeloxResizeBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X, X + +(72) ColumnarExchange +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(73) ShuffleQueryStage +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(74) InputAdapter +Input [3]: [s_name#X, s_address#X, s_nationkey#X] + +(75) InputIteratorTransformer +Input [3]: [s_name#X, s_address#X, s_nationkey#X] + +(76) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(77) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(78) ProjectExecTransformer +Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(79) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: false + +(80) VeloxResizeBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X, X + +(81) ColumnarExchange +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(82) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(83) InputAdapter +Input [1]: [n_nationkey#X] + +(84) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(85) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(86) ProjectExecTransformer +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(87) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, s_address#X] +Arguments: false + +(88) VeloxResizeBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X, X + +(89) ColumnarExchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(90) ShuffleQueryStage +Output [2]: [s_name#X, s_address#X] +Arguments: X + +(91) AQEShuffleRead +Input [2]: [s_name#X, s_address#X] +Arguments: local + +(92) VeloxColumnarToRow +Input [2]: [s_name#X, s_address#X] + +(93) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(94) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Condition : isnotnull(s_nationkey#X) + +(95) Exchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(96) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(98) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(99) Exchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(100) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(102) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(103) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(104) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(105) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(106) SortMergeJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(107) Exchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(108) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(109) Scan parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(110) Filter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(111) Project +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(112) Exchange +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(113) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(115) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(116) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(117) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(118) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(119) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(120) HashAggregate +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(121) HashAggregate +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(122) Filter +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Condition : isnotnull((0.5 * sum(l_quantity))#X) + +(123) Exchange +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(124) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(125) SortMergeJoin +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(126) Project +Output [1]: [ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(127) Exchange +Input [1]: [ps_suppkey#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(128) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(129) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(130) Project +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(131) Exchange +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(132) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(133) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(134) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(135) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(136) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(137) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(138) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(139) Project +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(140) Exchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(141) Sort +Input [2]: [s_name#X, s_address#X] +Arguments: [s_name#X ASC NULLS FIRST], true, 0 + +(142) AdaptiveSparkPlan +Output [2]: [s_name#X, s_address#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/21.txt new file mode 100644 index 000000000000..96750b2b3ecc --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/21.txt @@ -0,0 +1,713 @@ +== Physical Plan == +AdaptiveSparkPlan (137) ++- == Final Plan == + VeloxColumnarToRow (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxResizeBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxResizeBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxResizeBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxResizeBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ ScanTransformer parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxResizeBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ ScanTransformer parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxResizeBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ ScanTransformer parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxResizeBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ ScanTransformer parquet (70) ++- == Initial Plan == + TakeOrderedAndProject (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (116) + : : +- Exchange (115) + : : +- Project (114) + : : +- SortMergeJoin Inner (113) + : : :- Sort (95) + : : : +- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftAnti (110) + : : :- SortMergeJoin LeftSemi (104) + : : : :- Sort (100) + : : : : +- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Sort (103) + : : : +- Exchange (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) + + +(1) ScanTransformer parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) ProjectExecTransformer +Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(10) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(18) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(19) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(20) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [2]: [l_orderkey#X, l_suppkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(25) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(26) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(27) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(28) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(29) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(30) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(31) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(32) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(33) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(34) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(35) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(36) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(37) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(38) ProjectExecTransformer +Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [2]: [l_orderkey#X, l_suppkey#X] + +(39) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(40) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(41) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(42) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(43) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(44) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(45) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(46) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(47) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: false + +(48) VeloxResizeBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X, X + +(49) ColumnarExchange +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(51) InputAdapter +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] + +(52) InputIteratorTransformer +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] + +(53) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(54) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(55) ProjectExecTransformer +Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(56) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: false + +(57) VeloxResizeBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X, X + +(58) ColumnarExchange +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(59) ShuffleQueryStage +Output [1]: [o_orderkey#X] +Arguments: X + +(60) InputAdapter +Input [1]: [o_orderkey#X] + +(61) InputIteratorTransformer +Input [1]: [o_orderkey#X] + +(62) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(63) ProjectExecTransformer +Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [s_name#X, s_nationkey#X] +Arguments: X + +(68) InputAdapter +Input [2]: [s_name#X, s_nationkey#X] + +(69) InputIteratorTransformer +Input [2]: [s_name#X, s_nationkey#X] + +(70) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(71) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(72) ProjectExecTransformer +Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(73) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: false + +(74) VeloxResizeBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X, X + +(75) ColumnarExchange +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(76) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(77) InputAdapter +Input [1]: [n_nationkey#X] + +(78) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(79) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(80) ProjectExecTransformer +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(81) FlushableHashAggregateExecTransformer +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(82) ProjectExecTransformer +Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] +Input [2]: [s_name#X, count#X] + +(83) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: false + +(84) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X, X + +(85) ColumnarExchange +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(86) ShuffleQueryStage +Output [2]: [s_name#X, count#X] +Arguments: X + +(87) InputAdapter +Input [2]: [s_name#X, count#X] + +(88) InputIteratorTransformer +Input [2]: [s_name#X, count#X] + +(89) RegularHashAggregateExecTransformer +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(90) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, numwait#X] +Arguments: false + +(91) VeloxColumnarToRow +Input [2]: [s_name#X, numwait#X] + +(92) Scan parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(93) Filter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(94) Exchange +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(95) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(97) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(98) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(99) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(100) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(102) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(103) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(105) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(106) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(107) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(108) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(109) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(111) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(112) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(113) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(114) Project +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(115) Exchange +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(116) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(117) Scan parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(118) Filter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(119) Project +Output [1]: [o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(120) Exchange +Input [1]: [o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(121) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(123) Project +Output [2]: [s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(124) Exchange +Input [2]: [s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(125) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(127) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(128) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(129) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(132) Project +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(133) HashAggregate +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(134) Exchange +Input [2]: [s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(135) HashAggregate +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(136) TakeOrderedAndProject +Input [2]: [s_name#X, numwait#X] +Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] + +(137) AdaptiveSparkPlan +Output [2]: [s_name#X, numwait#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/22.txt new file mode 100644 index 000000000000..c6e147da04d8 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/22.txt @@ -0,0 +1,412 @@ +== Physical Plan == +AdaptiveSparkPlan (52) ++- == Final Plan == + VeloxColumnarToRow (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxResizeBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxResizeBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxResizeBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) + + +(1) ScanTransformer parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(3) ProjectExecTransformer +Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(8) InputAdapter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(9) InputIteratorTransformer +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(10) ScanTransformer parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(11) ProjectExecTransformer +Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] +Input [1]: [o_custkey#X] + +(12) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: false + +(13) VeloxResizeBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X, X + +(14) ColumnarExchange +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(15) ShuffleQueryStage +Output [1]: [o_custkey#X] +Arguments: X + +(16) InputAdapter +Input [1]: [o_custkey#X] + +(17) InputIteratorTransformer +Input [1]: [o_custkey#X] + +(18) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(19) ProjectExecTransformer +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(20) FlushableHashAggregateExecTransformer +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(21) ProjectExecTransformer +Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(22) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: false + +(23) VeloxResizeBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X, X + +(24) ColumnarExchange +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(26) InputAdapter +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(27) InputIteratorTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(28) RegularHashAggregateExecTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(29) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(30) VeloxResizeBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X, X + +(31) ColumnarExchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(32) ShuffleQueryStage +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(33) InputAdapter +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(34) InputIteratorTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(35) SortExecTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(36) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(37) VeloxColumnarToRow +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(38) Scan parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(39) Filter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(40) Exchange +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(43) Exchange +Input [1]: [o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(46) Project +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(47) HashAggregate +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(48) Exchange +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(49) HashAggregate +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(50) Exchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(51) Sort +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(52) AdaptiveSparkPlan +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (72) ++- == Final Plan == + VeloxColumnarToRow (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ ScanTransformer parquet (53) ++- == Initial Plan == + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) + + +(53) ScanTransformer parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(54) FilterExecTransformer +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(55) ProjectExecTransformer +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(56) FlushableHashAggregateExecTransformer +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(57) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, count#X] +Arguments: false + +(58) VeloxResizeBatches +Input [2]: [sum#X, count#X] +Arguments: X, X + +(59) ColumnarExchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(60) ShuffleQueryStage +Output [2]: [sum#X, count#X] +Arguments: X + +(61) InputAdapter +Input [2]: [sum#X, count#X] + +(62) InputIteratorTransformer +Input [2]: [sum#X, count#X] + +(63) RegularHashAggregateExecTransformer +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(64) WholeStageCodegenTransformer (X) +Input [1]: [avg(c_acctbal)#X] +Arguments: false + +(65) VeloxColumnarToRow +Input [1]: [avg(c_acctbal)#X] + +(66) Scan parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(67) Filter +Input [2]: [c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(68) Project +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(69) HashAggregate +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(70) Exchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(71) HashAggregate +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(72) AdaptiveSparkPlan +Output [1]: [avg(c_acctbal)#X] +Arguments: isFinalPlan=true + +Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (72) ++- == Final Plan == + VeloxColumnarToRow (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ ScanTransformer parquet (53) ++- == Initial Plan == + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/3.txt new file mode 100644 index 000000000000..d6573f7483ba --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/3.txt @@ -0,0 +1,351 @@ +== Physical Plan == +AdaptiveSparkPlan (66) ++- == Final Plan == + VeloxColumnarToRow (42) + +- TakeOrderedAndProjectExecTransformer (41) + +- ^ ProjectExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxResizeBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ ScanTransformer parquet (27) ++- == Initial Plan == + TakeOrderedAndProject (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (55) + : +- Exchange (54) + : +- Project (53) + : +- SortMergeJoin Inner (52) + : :- Sort (47) + : : +- Exchange (46) + : : +- Project (45) + : : +- Filter (44) + : : +- Scan parquet (43) + : +- Sort (51) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Sort (60) + +- Exchange (59) + +- Project (58) + +- Filter (57) + +- Scan parquet (56) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(3) ProjectExecTransformer +Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(4) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [1]: [c_custkey#X] +Arguments: X + +(8) InputAdapter +Input [1]: [c_custkey#X] + +(9) InputIteratorTransformer +Input [1]: [c_custkey#X] + +(10) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(17) InputAdapter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(18) InputIteratorTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(21) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(22) VeloxResizeBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X, X + +(23) ColumnarExchange +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(25) InputAdapter +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] + +(26) InputIteratorTransformer +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] + +(27) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(29) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(34) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(35) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(38) RegularHashAggregateExecTransformer +Input [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, _pre_X#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum(_pre_X#X)] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(39) ProjectExecTransformer +Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] +Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(40) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(41) TakeOrderedAndProjectExecTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 + +(42) VeloxColumnarToRow +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(43) Scan parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(44) Filter +Input [2]: [c_custkey#X, c_mktsegment#X] +Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(45) Project +Output [1]: [c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(46) Exchange +Input [1]: [c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(47) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(48) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(49) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(50) Exchange +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(51) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(52) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(53) Project +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(54) Exchange +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(55) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(56) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(57) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(58) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(59) Exchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(60) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(62) Project +Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(63) HashAggregate +Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(64) HashAggregate +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] + +(65) TakeOrderedAndProject +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(66) AdaptiveSparkPlan +Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/4.txt new file mode 100644 index 000000000000..71a8f6974ed1 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/4.txt @@ -0,0 +1,294 @@ +== Physical Plan == +AdaptiveSparkPlan (56) ++- == Final Plan == + VeloxColumnarToRow (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) + + +(1) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(3) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: X + +(8) InputAdapter +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(9) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(10) ScanTransformer parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(12) ProjectExecTransformer +Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(17) InputAdapter +Input [1]: [l_orderkey#X] + +(18) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(20) ProjectExecTransformer +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(21) FlushableHashAggregateExecTransformer +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(22) ProjectExecTransformer +Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] +Input [2]: [o_orderpriority#X, count#X] + +(23) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: false + +(24) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X, X + +(25) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [2]: [o_orderpriority#X, count#X] +Arguments: X + +(27) InputAdapter +Input [2]: [o_orderpriority#X, count#X] + +(28) InputIteratorTransformer +Input [2]: [o_orderpriority#X, count#X] + +(29) RegularHashAggregateExecTransformer +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(30) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(31) VeloxResizeBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X, X + +(32) ColumnarExchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(34) InputAdapter +Input [2]: [o_orderpriority#X, order_count#X] + +(35) InputIteratorTransformer +Input [2]: [o_orderpriority#X, order_count#X] + +(36) SortExecTransformer +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(37) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(38) VeloxColumnarToRow +Input [2]: [o_orderpriority#X, order_count#X] + +(39) Scan parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(40) Filter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(41) Project +Output [2]: [o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(42) Exchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(45) Filter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(46) Project +Output [1]: [l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(47) Exchange +Input [1]: [l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(50) Project +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(51) HashAggregate +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(52) Exchange +Input [2]: [o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(53) HashAggregate +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(54) Exchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(55) Sort +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(56) AdaptiveSparkPlan +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/5.txt new file mode 100644 index 000000000000..a78e477ace2c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/5.txt @@ -0,0 +1,802 @@ +== Physical Plan == +AdaptiveSparkPlan (156) ++- == Final Plan == + VeloxColumnarToRow (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxResizeBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxResizeBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxResizeBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxResizeBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxResizeBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxResizeBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxResizeBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxResizeBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ ScanTransformer parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ ScanTransformer parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ ScanTransformer parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxResizeBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ ScanTransformer parquet (78) ++- == Initial Plan == + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] +Input [2]: [c_custkey#X, c_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(9) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(10) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [2]: [c_nationkey#X, o_orderkey#X] +Arguments: X + +(25) InputAdapter +Input [2]: [c_nationkey#X, o_orderkey#X] + +(26) InputIteratorTransformer +Input [2]: [c_nationkey#X, o_orderkey#X] + +(27) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(29) ProjectExecTransformer +Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(30) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(31) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(32) ColumnarExchange +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(34) InputAdapter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(35) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(38) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(39) VeloxResizeBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(40) ColumnarExchange +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(42) InputAdapter +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(43) InputIteratorTransformer +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(44) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(46) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(47) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(48) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(49) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(51) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(52) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(55) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: false + +(56) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X, X + +(57) ColumnarExchange +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(59) InputAdapter +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] + +(60) InputIteratorTransformer +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] + +(61) ScanTransformer parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(63) ProjectExecTransformer +Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(64) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: false + +(65) VeloxResizeBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X, X + +(66) ColumnarExchange +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(68) InputAdapter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(69) InputIteratorTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(72) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: false + +(73) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X, X + +(74) ColumnarExchange +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(76) InputAdapter +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] + +(77) InputIteratorTransformer +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] + +(78) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(79) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(80) ProjectExecTransformer +Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(81) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: false + +(82) VeloxResizeBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X, X + +(83) ColumnarExchange +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(84) ShuffleQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(85) InputAdapter +Input [1]: [r_regionkey#X] + +(86) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(87) ShuffledHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(88) ProjectExecTransformer +Output [2]: [n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(89) FlushableHashAggregateExecTransformer +Input [2]: [n_name#X, _pre_X#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(90) ProjectExecTransformer +Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(91) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: false + +(92) VeloxResizeBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X, X + +(93) ColumnarExchange +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(94) ShuffleQueryStage +Output [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: X + +(95) InputAdapter +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(96) InputIteratorTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(97) RegularHashAggregateExecTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(98) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(99) VeloxResizeBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X, X + +(100) ColumnarExchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [2]: [n_name#X, revenue#X] +Arguments: X + +(102) InputAdapter +Input [2]: [n_name#X, revenue#X] + +(103) InputIteratorTransformer +Input [2]: [n_name#X, revenue#X] + +(104) SortExecTransformer +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(105) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(106) VeloxColumnarToRow +Input [2]: [n_name#X, revenue#X] + +(107) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(108) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(109) Exchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(112) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(113) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(114) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(117) Project +Output [2]: [c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(118) Exchange +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(121) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(122) Exchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(125) Project +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(126) Exchange +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(129) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(130) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(133) Project +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(134) Exchange +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(137) Filter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(138) Exchange +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(141) Project +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(142) Exchange +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(145) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(146) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(147) Exchange +Input [1]: [r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(150) Project +Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(151) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(152) Exchange +Input [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(153) HashAggregate +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(154) Exchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(155) Sort +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(156) AdaptiveSparkPlan +Output [2]: [n_name#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/6.txt new file mode 100644 index 000000000000..b85e63e10655 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/6.txt @@ -0,0 +1,116 @@ +== Physical Plan == +AdaptiveSparkPlan (20) ++- == Final Plan == + VeloxColumnarToRow (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxResizeBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) + + +(1) ScanTransformer parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(3) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * l_discount#X) AS _pre_X#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(5) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(6) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(7) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(8) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(9) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(10) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(11) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(12) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(13) VeloxColumnarToRow +Input [1]: [revenue#X] + +(14) Scan parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(15) Filter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(16) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(17) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(18) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(19) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(20) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/7.txt new file mode 100644 index 000000000000..a0894489f0a5 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/7.txt @@ -0,0 +1,764 @@ +== Physical Plan == +AdaptiveSparkPlan (149) ++- == Final Plan == + VeloxColumnarToRow (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxResizeBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxResizeBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxResizeBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxResizeBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxResizeBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxResizeBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxResizeBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxResizeBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ ScanTransformer parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ ScanTransformer parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ ScanTransformer parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) ++- == Initial Plan == + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) + + +(1) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(10) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: false + +(14) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X, X + +(15) ColumnarExchange +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(17) InputAdapter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(18) InputIteratorTransformer +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(21) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: false + +(22) VeloxResizeBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X, X + +(23) ColumnarExchange +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(25) InputAdapter +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) InputIteratorTransformer +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(27) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(29) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [2]: [o_orderkey#X, o_custkey#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(34) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(35) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(38) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: false + +(39) VeloxResizeBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X, X + +(40) ColumnarExchange +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(42) InputAdapter +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] + +(43) InputIteratorTransformer +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] + +(44) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(46) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] +Input [2]: [c_custkey#X, c_nationkey#X] + +(47) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: false + +(48) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X, X + +(49) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(51) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(52) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(55) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: false + +(56) VeloxResizeBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X, X + +(57) ColumnarExchange +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(59) InputAdapter +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] + +(60) InputIteratorTransformer +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] + +(61) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(62) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(63) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(68) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(69) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(72) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: false + +(73) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X, X + +(74) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(76) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] + +(77) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] + +(78) ReusedExchange [Reuses operator id: 66] +Output [2]: [n_nationkey#X, n_name#X] + +(79) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(80) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(81) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(82) ShuffledHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(83) ProjectExecTransformer +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(84) FlushableHashAggregateExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(85) ProjectExecTransformer +Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(86) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: false + +(87) VeloxResizeBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(88) ColumnarExchange +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(89) ShuffleQueryStage +Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(90) InputAdapter +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(91) InputIteratorTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(92) RegularHashAggregateExecTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(93) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(94) VeloxResizeBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X, X + +(95) ColumnarExchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(96) ShuffleQueryStage +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(97) InputAdapter +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(98) InputIteratorTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(99) SortExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(100) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(101) VeloxColumnarToRow +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(102) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(103) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(104) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(107) Filter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(108) Exchange +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(111) Project +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(112) Exchange +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(115) Filter +Input [2]: [o_orderkey#X, o_custkey#X] +Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(116) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(119) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(120) Exchange +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(123) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(124) Exchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(127) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(128) Exchange +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(131) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(132) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(135) Project +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(136) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] +ReadSchema: struct + +(139) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) + +(140) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(143) Project +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(144) HashAggregate +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(145) Exchange +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(146) HashAggregate +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(147) Exchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(148) Sort +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(149) AdaptiveSparkPlan +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/8.txt new file mode 100644 index 000000000000..88a6a79e074e --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/8.txt @@ -0,0 +1,1061 @@ +== Physical Plan == +AdaptiveSparkPlan (207) ++- == Final Plan == + VeloxColumnarToRow (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxResizeBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxResizeBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxResizeBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxResizeBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxResizeBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxResizeBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxResizeBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxResizeBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxResizeBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ ScanTransformer parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxResizeBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ ScanTransformer parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxResizeBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ ScanTransformer parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxResizeBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ ScanTransformer parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxResizeBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ ScanTransformer parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxResizeBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ ScanTransformer parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxResizeBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ ScanTransformer parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxResizeBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ ScanTransformer parquet (112) ++- == Initial Plan == + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(4) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(8) InputAdapter +Input [1]: [p_partkey#X] + +(9) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(10) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(13) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(14) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(15) ColumnarExchange +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(17) InputAdapter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(18) InputIteratorTransformer +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(21) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(22) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(23) ColumnarExchange +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(25) InputAdapter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(27) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(29) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(34) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(35) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(38) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: false + +(39) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X, X + +(40) ColumnarExchange +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(42) InputAdapter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(43) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(44) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(46) ProjectExecTransformer +Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(47) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: false + +(48) VeloxResizeBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X, X + +(49) ColumnarExchange +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(51) InputAdapter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(52) InputIteratorTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(55) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: false + +(56) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X, X + +(57) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(59) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] + +(60) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] + +(61) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(63) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] +Input [2]: [c_custkey#X, c_nationkey#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(68) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(69) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(72) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: false + +(73) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X, X + +(74) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(76) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] + +(77) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] + +(78) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(79) FilterExecTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(80) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Input [2]: [n_nationkey#X, n_regionkey#X] + +(81) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: false + +(82) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X, X + +(83) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(84) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_regionkey#X] +Arguments: X + +(85) InputAdapter +Input [2]: [n_nationkey#X, n_regionkey#X] + +(86) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] + +(87) ShuffledHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(88) ProjectExecTransformer +Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(89) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: false + +(90) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X, X + +(91) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(92) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(93) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] + +(94) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] + +(95) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(96) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(97) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(98) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(99) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(100) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(102) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(103) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(104) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(105) ProjectExecTransformer +Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(106) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: false + +(107) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X, X + +(108) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(109) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(110) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] + +(111) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] + +(112) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(113) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(114) ProjectExecTransformer +Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(115) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: false + +(116) VeloxResizeBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X, X + +(117) ColumnarExchange +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(118) ShuffleQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(119) InputAdapter +Input [1]: [r_regionkey#X] + +(120) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(121) ShuffledHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(122) ProjectExecTransformer +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(123) FlushableHashAggregateExecTransformer +Input [3]: [o_year#X, volume#X, _pre_X#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(124) ProjectExecTransformer +Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(125) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: false + +(126) VeloxResizeBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X, X + +(127) ColumnarExchange +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(128) ShuffleQueryStage +Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(129) InputAdapter +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(130) InputIteratorTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(131) RegularHashAggregateExecTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(132) ProjectExecTransformer +Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] +Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(133) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(134) VeloxResizeBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X, X + +(135) ColumnarExchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(136) ShuffleQueryStage +Output [2]: [o_year#X, mkt_share#X] +Arguments: X + +(137) InputAdapter +Input [2]: [o_year#X, mkt_share#X] + +(138) InputIteratorTransformer +Input [2]: [o_year#X, mkt_share#X] + +(139) SortExecTransformer +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(140) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(141) VeloxColumnarToRow +Input [2]: [o_year#X, mkt_share#X] + +(142) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(143) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(144) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(145) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(148) Filter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(149) Exchange +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(152) Project +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(153) Exchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(156) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(157) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(160) Project +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(161) Exchange +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(164) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(165) Exchange +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(168) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(169) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(172) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(173) Exchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(176) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(177) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(180) Filter +Input [2]: [n_nationkey#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(181) Exchange +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(184) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(185) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(188) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(189) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(192) Project +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(193) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(196) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(197) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(198) Exchange +Input [1]: [r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(201) Project +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(202) HashAggregate +Input [3]: [o_year#X, volume#X, nation#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(203) Exchange +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(204) HashAggregate +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] + +(205) Exchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(206) Sort +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(207) AdaptiveSparkPlan +Output [2]: [o_year#X, mkt_share#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/9.txt new file mode 100644 index 000000000000..4f537e91e132 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark35/9.txt @@ -0,0 +1,797 @@ +== Physical Plan == +AdaptiveSparkPlan (155) ++- == Final Plan == + VeloxColumnarToRow (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxResizeBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxResizeBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxResizeBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxResizeBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxResizeBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxResizeBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxResizeBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxResizeBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ ScanTransformer parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ ScanTransformer parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ ScanTransformer parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxResizeBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ ScanTransformer parquet (78) ++- == Initial Plan == + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(4) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(8) InputAdapter +Input [1]: [p_partkey#X] + +(9) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(10) ScanTransformer parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(13) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(14) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(15) ColumnarExchange +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(17) InputAdapter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(18) InputIteratorTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(21) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(22) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(23) ColumnarExchange +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(25) InputAdapter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(26) InputIteratorTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(27) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(29) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(34) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(35) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(38) WholeStageCodegenTransformer (X) +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: false + +(39) VeloxResizeBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X, X + +(40) ColumnarExchange +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(42) InputAdapter +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(43) InputIteratorTransformer +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(44) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(46) ProjectExecTransformer +Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(47) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: false + +(48) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X, X + +(49) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(51) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(52) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(55) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: false + +(56) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X, X + +(57) ColumnarExchange +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(59) InputAdapter +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] + +(60) InputIteratorTransformer +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] + +(61) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: isnotnull(o_orderkey#X) + +(63) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Input [2]: [o_orderkey#X, o_orderdate#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_orderdate#X] +Arguments: X + +(68) InputAdapter +Input [2]: [o_orderkey#X, o_orderdate#X] + +(69) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(72) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: false + +(73) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X, X + +(74) ColumnarExchange +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(76) InputAdapter +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] + +(77) InputIteratorTransformer +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] + +(78) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(79) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(80) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(81) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(82) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(83) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(84) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(85) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(86) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(87) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(88) ProjectExecTransformer +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(89) FlushableHashAggregateExecTransformer +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(90) ProjectExecTransformer +Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(91) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: false + +(92) VeloxResizeBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(93) ColumnarExchange +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(94) ShuffleQueryStage +Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(95) InputAdapter +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(96) InputIteratorTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(97) RegularHashAggregateExecTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(98) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(99) VeloxResizeBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X, X + +(100) ColumnarExchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(102) InputAdapter +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(103) InputIteratorTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(104) SortExecTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(105) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(106) VeloxColumnarToRow +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(107) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(108) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(109) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(110) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(113) Filter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(114) Exchange +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(117) Project +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(118) Exchange +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(121) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(122) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(125) Project +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(126) Exchange +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(129) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(130) Exchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(133) Project +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(134) Exchange +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(137) Filter +Input [2]: [o_orderkey#X, o_orderdate#X] +Condition : isnotnull(o_orderkey#X) + +(138) Exchange +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(141) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(142) Exchange +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(145) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(146) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(149) Project +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(150) HashAggregate +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(151) Exchange +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(152) HashAggregate +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(153) Exchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(154) Sort +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(155) AdaptiveSparkPlan +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/1.txt new file mode 100644 index 000000000000..fce60c054ea8 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/1.txt @@ -0,0 +1,162 @@ +== Physical Plan == +AdaptiveSparkPlan (30) ++- == Final Plan == + VeloxColumnarToRow (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxResizeBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) + + +(1) ScanTransformer parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(3) ProjectExecTransformer +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(5) ProjectExecTransformer +Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(6) WholeStageCodegenTransformer (X) +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: false + +(7) VeloxResizeBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X, X + +(8) ColumnarExchange +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(9) ShuffleQueryStage +Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(10) InputAdapter +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(11) InputIteratorTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(12) RegularHashAggregateExecTransformer +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(13) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(14) VeloxResizeBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X, X + +(15) ColumnarExchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(17) InputAdapter +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(18) InputIteratorTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(19) SortExecTransformer +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(20) WholeStageCodegenTransformer (X) +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: false + +(21) VeloxColumnarToRow +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] + +(22) Scan parquet +Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(23) Filter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) + +(24) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(25) HashAggregate +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] +Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] + +(26) Exchange +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(27) HashAggregate +Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Keys [2]: [l_returnflag#X, l_linestatus#X] +Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] +Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] + +(28) Exchange +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(29) Sort +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 + +(30) AdaptiveSparkPlan +Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/10.txt new file mode 100644 index 000000000000..855ddf391138 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/10.txt @@ -0,0 +1,522 @@ +== Physical Plan == +AdaptiveSparkPlan (100) ++- == Final Plan == + VeloxColumnarToRow (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxResizeBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxResizeBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxResizeBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxResizeBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ ScanTransformer parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxResizeBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ ScanTransformer parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxResizeBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ ScanTransformer parquet (44) ++- == Initial Plan == + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) + + +(1) ScanTransformer parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) ProjectExecTransformer +Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(4) WholeStageCodegenTransformer (X) +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: false + +(5) VeloxResizeBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X, X + +(6) ColumnarExchange +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(8) InputAdapter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(9) InputIteratorTransformer +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(10) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(25) InputAdapter +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] + +(26) InputIteratorTransformer +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] + +(27) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(29) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(34) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(35) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(38) WholeStageCodegenTransformer (X) +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(39) VeloxResizeBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(40) ColumnarExchange +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(42) InputAdapter +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] + +(43) InputIteratorTransformer +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] + +(44) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(46) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(47) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(48) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(49) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(51) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(52) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(55) FlushableHashAggregateExecTransformer +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, _pre_X#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(56) ProjectExecTransformer +Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(57) WholeStageCodegenTransformer (X) +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: false + +(58) VeloxResizeBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X, X + +(59) ColumnarExchange +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(60) ShuffleQueryStage +Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(61) InputAdapter +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(62) InputIteratorTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(63) RegularHashAggregateExecTransformer +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(64) ProjectExecTransformer +Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(65) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(66) TakeOrderedAndProjectExecTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 + +(67) VeloxColumnarToRow +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(68) Scan parquet +Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(69) Filter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(70) Exchange +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(73) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(74) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(75) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(78) Project +Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] + +(79) Exchange +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(82) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) + +(83) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(84) Exchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(87) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(88) Exchange +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(91) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(92) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(95) Project +Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] + +(96) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] + +(97) Exchange +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(98) HashAggregate +Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(99) TakeOrderedAndProject +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(100) AdaptiveSparkPlan +Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/11.txt new file mode 100644 index 000000000000..99539cde1a75 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/11.txt @@ -0,0 +1,709 @@ +== Physical Plan == +AdaptiveSparkPlan (82) ++- == Final Plan == + VeloxColumnarToRow (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxResizeBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxResizeBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxResizeBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ ScanTransformer parquet (27) ++- == Initial Plan == + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) + + +(1) ScanTransformer parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(3) ProjectExecTransformer +Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(8) InputAdapter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(9) InputIteratorTransformer +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(10) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(18) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(25) InputAdapter +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(26) InputIteratorTransformer +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(27) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(29) ProjectExecTransformer +Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(30) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(34) InputAdapter +Input [1]: [n_nationkey#X] + +(35) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [2]: [ps_partkey#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(38) FlushableHashAggregateExecTransformer +Input [2]: [ps_partkey#X, _pre_X#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(39) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(40) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: false + +(41) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(42) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(43) ShuffleQueryStage +Output [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(44) InputAdapter +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(45) InputIteratorTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(46) RegularHashAggregateExecTransformer +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(47) FilterExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(48) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(49) VeloxResizeBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X, X + +(50) ColumnarExchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(51) ShuffleQueryStage +Output [2]: [ps_partkey#X, value#X] +Arguments: X + +(52) InputAdapter +Input [2]: [ps_partkey#X, value#X] + +(53) InputIteratorTransformer +Input [2]: [ps_partkey#X, value#X] + +(54) SortExecTransformer +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(55) WholeStageCodegenTransformer (X) +Input [2]: [ps_partkey#X, value#X] +Arguments: false + +(56) VeloxColumnarToRow +Input [2]: [ps_partkey#X, value#X] + +(57) Scan parquet +Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(58) Filter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(59) Exchange +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(62) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(63) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(66) Project +Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(67) Exchange +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(70) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(71) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(72) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(75) Project +Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(76) HashAggregate +Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [ps_partkey#X, sum#X, isEmpty#X] + +(77) Exchange +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(78) HashAggregate +Input [3]: [ps_partkey#X, sum#X, isEmpty#X] +Keys [1]: [ps_partkey#X] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] + +(79) Filter +Input [2]: [ps_partkey#X, value#X] +Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) + +(80) Exchange +Input [2]: [ps_partkey#X, value#X] +Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(81) Sort +Input [2]: [ps_partkey#X, value#X] +Arguments: [value#X DESC NULLS LAST], true, 0 + +(82) AdaptiveSparkPlan +Output [2]: [ps_partkey#X, value#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (135) ++- == Final Plan == + VeloxColumnarToRow (113) + +- ^ ProjectExecTransformer (111) + +- ^ RegularHashAggregateExecTransformer (110) + +- ^ ProjectExecTransformer (109) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) + :- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxResizeBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) + : :- ^ InputIteratorTransformer (91) + : : +- ShuffleQueryStage (89), Statistics(X) + : : +- ColumnarExchange (88) + : : +- VeloxResizeBatches (87) + : : +- ^ ProjectExecTransformer (85) + : : +- ^ FilterExecTransformer (84) + : : +- ^ ScanTransformer parquet (83) + : +- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ReusedExchange (92) + +- ^ InputIteratorTransformer (107) + +- ShuffleQueryStage (105), Statistics(X) + +- ReusedExchange (104) ++- == Initial Plan == + HashAggregate (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (117) + : : +- Exchange (116) + : : +- Filter (115) + : : +- Scan parquet (114) + : +- Sort (121) + : +- Exchange (120) + : +- Filter (119) + : +- Scan parquet (118) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) + + +(83) ScanTransformer parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(84) FilterExecTransformer +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: isnotnull(ps_suppkey#X) + +(85) ProjectExecTransformer +Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(86) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: false + +(87) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X, X + +(88) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(89) ShuffleQueryStage +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(90) InputAdapter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(91) InputIteratorTransformer +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(92) ReusedExchange [Reuses operator id: 15] +Output [2]: [s_suppkey#X, s_nationkey#X] + +(93) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(94) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(95) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(96) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(97) ProjectExecTransformer +Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(98) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: false + +(99) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X, X + +(100) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(102) InputAdapter +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(103) InputIteratorTransformer +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] + +(104) ReusedExchange [Reuses operator id: 32] +Output [1]: [n_nationkey#X] + +(105) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(106) InputAdapter +Input [1]: [n_nationkey#X] + +(107) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(108) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(109) ProjectExecTransformer +Output [1]: [(ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(110) RegularHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [sum(_pre_X#X)] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(111) ProjectExecTransformer +Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] + +(112) WholeStageCodegenTransformer (X) +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: false + +(113) VeloxColumnarToRow +Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(114) Scan parquet +Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(115) Filter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Condition : isnotnull(ps_suppkey#X) + +(116) Exchange +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(117) Sort +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(118) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(119) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(120) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(121) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(123) Project +Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] + +(124) Exchange +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(125) Sort +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(127) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) + +(128) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(129) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(132) Project +Output [2]: [ps_availqty#X, ps_supplycost#X] +Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] + +(133) HashAggregate +Input [2]: [ps_availqty#X, ps_supplycost#X] +Keys: [] +Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(134) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] +Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] + +(135) AdaptiveSparkPlan +Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/12.txt new file mode 100644 index 000000000000..9cb9c145f1bb --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/12.txt @@ -0,0 +1,289 @@ +== Physical Plan == +AdaptiveSparkPlan (55) ++- == Final Plan == + VeloxColumnarToRow (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) + + +(1) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: isnotnull(o_orderkey#X) + +(3) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: X + +(8) InputAdapter +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(9) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(10) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_shipmode#X] +Arguments: X + +(17) InputAdapter +Input [2]: [l_orderkey#X, l_shipmode#X] + +(18) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_shipmode#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [3]: [l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(21) FlushableHashAggregateExecTransformer +Input [3]: [l_shipmode#X, _pre_X#X, _pre_X#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(22) ProjectExecTransformer +Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(23) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: false + +(24) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X, X + +(25) ColumnarExchange +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: X + +(27) InputAdapter +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(28) InputIteratorTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] + +(29) RegularHashAggregateExecTransformer +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(34) InputAdapter +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(35) InputIteratorTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(36) SortExecTransformer +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(37) WholeStageCodegenTransformer (X) +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: false + +(38) VeloxColumnarToRow +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] + +(39) Scan parquet +Output [2]: [o_orderkey#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(40) Filter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Condition : isnotnull(o_orderkey#X) + +(41) Exchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet +Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(44) Filter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) + +(45) Project +Output [2]: [l_orderkey#X, l_shipmode#X] +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(46) Exchange +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(49) Project +Output [2]: [o_orderpriority#X, l_shipmode#X] +Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] + +(50) HashAggregate +Input [2]: [o_orderpriority#X, l_shipmode#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum#X, sum#X] +Results [3]: [l_shipmode#X, sum#X, sum#X] + +(51) Exchange +Input [3]: [l_shipmode#X, sum#X, sum#X] +Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(52) HashAggregate +Input [3]: [l_shipmode#X, sum#X, sum#X] +Keys [1]: [l_shipmode#X] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] +Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] + +(53) Exchange +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(54) Sort +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 + +(55) AdaptiveSparkPlan +Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/13.txt new file mode 100644 index 000000000000..59abfb682c42 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/13.txt @@ -0,0 +1,306 @@ +== Physical Plan == +AdaptiveSparkPlan (57) ++- == Final Plan == + VeloxColumnarToRow (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxResizeBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxResizeBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxResizeBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxResizeBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ ScanTransformer parquet (9) ++- == Initial Plan == + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin LeftOuter (48) + :- Sort (42) + : +- Exchange (41) + : +- Scan parquet (40) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) + + +(1) ScanTransformer parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(2) ProjectExecTransformer +Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] +Input [1]: [c_custkey#X] + +(3) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: false + +(4) VeloxResizeBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X, X + +(5) ColumnarExchange +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(6) ShuffleQueryStage +Output [1]: [c_custkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [c_custkey#X] + +(8) InputIteratorTransformer +Input [1]: [c_custkey#X] + +(9) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(10) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(11) ProjectExecTransformer +Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(12) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(13) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(14) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(15) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(16) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(17) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(19) ProjectExecTransformer +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(20) RegularHashAggregateExecTransformer +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(21) ProjectExecTransformer +Output [1]: [count(o_orderkey#X)#X AS c_count#X] +Input [2]: [c_custkey#X, count(o_orderkey#X)#X] + +(22) FlushableHashAggregateExecTransformer +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(23) ProjectExecTransformer +Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] +Input [2]: [c_count#X, count#X] + +(24) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: false + +(25) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X, X + +(26) ColumnarExchange +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(27) ShuffleQueryStage +Output [2]: [c_count#X, count#X] +Arguments: X + +(28) InputAdapter +Input [2]: [c_count#X, count#X] + +(29) InputIteratorTransformer +Input [2]: [c_count#X, count#X] + +(30) RegularHashAggregateExecTransformer +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(31) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(32) VeloxResizeBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X, X + +(33) ColumnarExchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(34) ShuffleQueryStage +Output [2]: [c_count#X, custdist#X] +Arguments: X + +(35) InputAdapter +Input [2]: [c_count#X, custdist#X] + +(36) InputIteratorTransformer +Input [2]: [c_count#X, custdist#X] + +(37) SortExecTransformer +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(38) WholeStageCodegenTransformer (X) +Input [2]: [c_count#X, custdist#X] +Arguments: false + +(39) VeloxColumnarToRow +Input [2]: [c_count#X, custdist#X] + +(40) Scan parquet +Output [1]: [c_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(41) Exchange +Input [1]: [c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(42) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(44) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) + +(45) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] + +(46) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(47) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftOuter +Join condition: None + +(49) Project +Output [2]: [c_custkey#X, o_orderkey#X] +Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] + +(50) HashAggregate +Input [2]: [c_custkey#X, o_orderkey#X] +Keys [1]: [c_custkey#X] +Functions [1]: [partial_count(o_orderkey#X)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_custkey#X, count#X] + +(51) HashAggregate +Input [2]: [c_custkey#X, count#X] +Keys [1]: [c_custkey#X] +Functions [1]: [count(o_orderkey#X)] +Aggregate Attributes [1]: [count(o_orderkey#X)#X] +Results [1]: [count(o_orderkey#X)#X AS c_count#X] + +(52) HashAggregate +Input [1]: [c_count#X] +Keys [1]: [c_count#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [c_count#X, count#X] + +(53) Exchange +Input [2]: [c_count#X, count#X] +Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(54) HashAggregate +Input [2]: [c_count#X, count#X] +Keys [1]: [c_count#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [c_count#X, count(1)#X AS custdist#X] + +(55) Exchange +Input [2]: [c_count#X, custdist#X] +Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) Sort +Input [2]: [c_count#X, custdist#X] +Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 + +(57) AdaptiveSparkPlan +Output [2]: [c_count#X, custdist#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/14.txt new file mode 100644 index 000000000000..058ec94ab093 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/14.txt @@ -0,0 +1,209 @@ +== Physical Plan == +AdaptiveSparkPlan (38) ++- == Final Plan == + VeloxColumnarToRow (24) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) + + +(1) ScanTransformer parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(3) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(8) InputAdapter +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] + +(9) InputIteratorTransformer +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] + +(10) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: isnotnull(p_partkey#X) + +(12) ProjectExecTransformer +Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] +Input [2]: [p_partkey#X, p_type#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [p_partkey#X, p_type#X] +Arguments: X + +(17) InputAdapter +Input [2]: [p_partkey#X, p_type#X] + +(18) InputIteratorTransformer +Input [2]: [p_partkey#X, p_type#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [2]: [CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(21) RegularHashAggregateExecTransformer +Input [2]: [_pre_X#X, _pre_X#X] +Keys: [] +Functions [2]: [sum(_pre_X#X), sum(_pre_X#X)] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(22) ProjectExecTransformer +Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] +Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(23) WholeStageCodegenTransformer (X) +Input [1]: [promo_revenue#X] +Arguments: false + +(24) VeloxColumnarToRow +Input [1]: [promo_revenue#X] + +(25) Scan parquet +Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(26) Filter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) + +(27) Project +Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(28) Exchange +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(29) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(31) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : isnotnull(p_partkey#X) + +(32) Exchange +Input [2]: [p_partkey#X, p_type#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(33) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(35) Project +Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] + +(36) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] +Keys: [] +Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] + +(37) HashAggregate +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] + +(38) AdaptiveSparkPlan +Output [1]: [promo_revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/15.txt new file mode 100644 index 000000000000..e0c0204c0b87 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/15.txt @@ -0,0 +1,410 @@ +== Physical Plan == +AdaptiveSparkPlan (47) ++- == Final Plan == + VeloxColumnarToRow (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxResizeBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxResizeBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (46) + +- Exchange (45) + +- Project (44) + +- SortMergeJoin Inner (43) + :- Sort (34) + : +- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Sort (42) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: isnotnull(s_suppkey#X) + +(3) ProjectExecTransformer +Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(8) InputAdapter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(9) InputIteratorTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(10) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(12) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(13) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(14) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(15) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(16) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(17) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(18) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(19) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(20) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(21) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(22) FilterExecTransformer +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(23) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(24) ProjectExecTransformer +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(25) WholeStageCodegenTransformer (X) +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: false + +(26) VeloxResizeBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X, X + +(27) ColumnarExchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(28) ShuffleQueryStage +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(29) AQEShuffleRead +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: local + +(30) VeloxColumnarToRow +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] + +(31) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(32) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Condition : isnotnull(s_suppkey#X) + +(33) Exchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(34) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(35) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(36) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) + +(37) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(38) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(39) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(40) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(41) Filter +Input [2]: [supplier_no#X, total_revenue#X] +Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) + +(42) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(43) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [supplier_no#X] +Join type: Inner +Join condition: None + +(44) Project +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] + +(45) Exchange +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(46) Sort +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 + +(47) AdaptiveSparkPlan +Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (72) ++- == Final Plan == + VeloxColumnarToRow (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ ProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxResizeBatches (54) + +- ^ ProjectExecTransformer (52) + +- ^ FlushableHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ FilterExecTransformer (49) + +- ^ ScanTransformer parquet (48) ++- == Initial Plan == + HashAggregate (71) + +- HashAggregate (70) + +- HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) + + +(48) ScanTransformer parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(49) FilterExecTransformer +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(50) ProjectExecTransformer +Output [2]: [l_suppkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(51) FlushableHashAggregateExecTransformer +Input [2]: [l_suppkey#X, _pre_X#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(52) ProjectExecTransformer +Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(53) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: false + +(54) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(55) ColumnarExchange +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(56) ShuffleQueryStage +Output [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(57) InputAdapter +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(58) InputIteratorTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(59) RegularHashAggregateExecTransformer +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(60) ProjectExecTransformer +Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] +Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(61) RegularHashAggregateExecTransformer +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(62) WholeStageCodegenTransformer (X) +Input [1]: [max(total_revenue)#X] +Arguments: false + +(63) VeloxColumnarToRow +Input [1]: [max(total_revenue)#X] + +(64) Scan parquet +Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(65) Filter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) + +(66) Project +Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(67) HashAggregate +Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_suppkey#X, sum#X, isEmpty#X] + +(68) Exchange +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(69) HashAggregate +Input [3]: [l_suppkey#X, sum#X, isEmpty#X] +Keys [1]: [l_suppkey#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] + +(70) HashAggregate +Input [1]: [total_revenue#X] +Keys: [] +Functions [1]: [partial_max(total_revenue#X)] +Aggregate Attributes [1]: [max#X] +Results [1]: [max#X] + +(71) HashAggregate +Input [1]: [max#X] +Keys: [] +Functions [1]: [max(total_revenue#X)] +Aggregate Attributes [1]: [max(total_revenue#X)#X] +Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] + +(72) AdaptiveSparkPlan +Output [1]: [max(total_revenue)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/16.txt new file mode 100644 index 000000000000..3441216f140f --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/16.txt @@ -0,0 +1,382 @@ +== Physical Plan == +AdaptiveSparkPlan (71) ++- == Final Plan == + VeloxColumnarToRow (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxResizeBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxResizeBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) + + +(1) ScanTransformer parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: isnotnull(ps_partkey#X) + +(3) ProjectExecTransformer +Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Input [2]: [ps_partkey#X, ps_suppkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [ps_partkey#X, ps_suppkey#X] + +(9) InputIteratorTransformer +Input [2]: [ps_partkey#X, ps_suppkey#X] + +(10) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(12) ProjectExecTransformer +Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(17) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(18) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(21) FlushableHashAggregateExecTransformer +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(22) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(23) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: false + +(24) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X, X + +(25) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(27) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(28) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(29) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(30) FlushableHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(31) ProjectExecTransformer +Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(32) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: false + +(33) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X, X + +(34) ColumnarExchange +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(35) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(36) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(37) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(38) RegularHashAggregateExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(39) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(40) VeloxResizeBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X, X + +(41) ColumnarExchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(42) ShuffleQueryStage +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(43) InputAdapter +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(44) InputIteratorTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(45) SortExecTransformer +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(46) WholeStageCodegenTransformer (X) +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: false + +(47) VeloxColumnarToRow +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] + +(48) Scan parquet +Output [2]: [ps_partkey#X, ps_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(49) Filter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Condition : isnotnull(ps_partkey#X) + +(50) Scan parquet +Output [2]: [s_suppkey#X, s_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct + +(51) Filter +Input [2]: [s_suppkey#X, s_comment#X] +Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) + +(52) Project +Output [1]: [s_suppkey#X] +Input [2]: [s_suppkey#X, s_comment#X] + +(53) BroadcastExchange +Input [1]: [s_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] + +(54) BroadcastHashJoin +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: LeftAnti +Join condition: None + +(55) Exchange +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(58) Filter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) + +(59) Exchange +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(62) Project +Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(63) HashAggregate +Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(64) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(65) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] + +(66) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [partial_count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] + +(67) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(68) HashAggregate +Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] +Keys [3]: [p_brand#X, p_type#X, p_size#X] +Functions [1]: [count(distinct ps_suppkey#X)] +Aggregate Attributes [1]: [count(ps_suppkey#X)#X] +Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] + +(69) Exchange +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(70) Sort +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 + +(71) AdaptiveSparkPlan +Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/17.txt new file mode 100644 index 000000000000..c1b43b7fc421 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/17.txt @@ -0,0 +1,347 @@ +== Physical Plan == +AdaptiveSparkPlan (62) ++- == Final Plan == + VeloxColumnarToRow (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxResizeBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ ScanTransformer parquet (21) ++- == Initial Plan == + HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- SortMergeJoin Inner (58) + :- Project (50) + : +- SortMergeJoin Inner (49) + : :- Sort (43) + : : +- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Sort (48) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Sort (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Filter (52) + +- Scan parquet (51) + + +(1) ScanTransformer parquet +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) + +(3) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(8) InputAdapter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(9) InputIteratorTransformer +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(10) ScanTransformer parquet +Output [3]: [p_partkey#X, p_brand#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) + +(12) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [3]: [p_partkey#X, p_brand#X, p_container#X] + +(13) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(17) InputAdapter +Input [1]: [p_partkey#X] + +(18) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] + +(21) ScanTransformer parquet +Output [2]: [l_partkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(22) FilterExecTransformer +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: isnotnull(l_partkey#X) + +(23) FlushableHashAggregateExecTransformer +Input [2]: [l_partkey#X, l_quantity#X] +Keys [1]: [l_partkey#X] +Functions [1]: [partial_avg(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [3]: [l_partkey#X, sum#X, count#X] + +(24) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] +Input [3]: [l_partkey#X, sum#X, count#X] + +(25) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: false + +(26) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X, X + +(27) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(28) ShuffleQueryStage +Output [3]: [l_partkey#X, sum#X, count#X] +Arguments: X + +(29) InputAdapter +Input [3]: [l_partkey#X, sum#X, count#X] + +(30) InputIteratorTransformer +Input [3]: [l_partkey#X, sum#X, count#X] + +(31) RegularHashAggregateExecTransformer +Input [3]: [l_partkey#X, sum#X, count#X] +Keys [1]: [l_partkey#X] +Functions [1]: [avg(l_quantity#X)] +Aggregate Attributes [1]: [avg(l_quantity#X)#X] +Results [2]: [l_partkey#X, avg(l_quantity#X)#X] + +(32) ProjectExecTransformer +Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] +Input [2]: [l_partkey#X, avg(l_quantity#X)#X] + +(33) FilterExecTransformer +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: isnotnull((0.2 * avg(l_quantity))#X) + +(34) ShuffledHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) + +(35) ProjectExecTransformer +Output [1]: [l_extendedprice#X] +Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] + +(36) RegularHashAggregateExecTransformer +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [sum(l_extendedprice#X)#X] + +(37) ProjectExecTransformer +Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] +Input [1]: [sum(l_extendedprice#X)#X] + +(38) WholeStageCodegenTransformer (X) +Input [1]: [avg_yearly#X] +Arguments: false + +(39) VeloxColumnarToRow +Input [1]: [avg_yearly#X] + +(40) Scan parquet +Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(41) Filter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) + +(42) Exchange +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet +Output [3]: [p_partkey#X, p_brand#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +ReadSchema: struct + +(45) Filter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) + +(46) Project +Output [1]: [p_partkey#X] +Input [3]: [p_partkey#X, p_brand#X, p_container#X] + +(47) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(48) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: None + +(50) Project +Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] + +(51) Scan parquet +Output [2]: [l_partkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(52) Filter +Input [2]: [l_partkey#X, l_quantity#X] +Condition : isnotnull(l_partkey#X) + +(53) HashAggregate +Input [2]: [l_partkey#X, l_quantity#X] +Keys [1]: [l_partkey#X] +Functions [1]: [partial_avg(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [3]: [l_partkey#X, sum#X, count#X] + +(54) Exchange +Input [3]: [l_partkey#X, sum#X, count#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(55) HashAggregate +Input [3]: [l_partkey#X, sum#X, count#X] +Keys [1]: [l_partkey#X] +Functions [1]: [avg(l_quantity#X)] +Aggregate Attributes [1]: [avg(l_quantity#X)#X] +Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] + +(56) Filter +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Condition : isnotnull((0.2 * avg(l_quantity))#X) + +(57) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(58) SortMergeJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) + +(59) Project +Output [1]: [l_extendedprice#X] +Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] + +(60) HashAggregate +Input [1]: [l_extendedprice#X] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(61) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum(l_extendedprice#X)] +Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] +Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] + +(62) AdaptiveSparkPlan +Output [1]: [avg_yearly#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/18.txt new file mode 100644 index 000000000000..c1a6b181bb34 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/18.txt @@ -0,0 +1,589 @@ +== Physical Plan == +AdaptiveSparkPlan (109) ++- == Final Plan == + VeloxColumnarToRow (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxResizeBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxResizeBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxResizeBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ ScanTransformer parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxResizeBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ ScanTransformer parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxResizeBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ScanTransformer parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) ++- == Initial Plan == + TakeOrderedAndProject (108) + +- HashAggregate (107) + +- HashAggregate (106) + +- Project (105) + +- SortMergeJoin Inner (104) + :- Sort (91) + : +- Exchange (90) + : +- Project (89) + : +- SortMergeJoin Inner (88) + : :- Sort (73) + : : +- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Sort (87) + : +- Exchange (86) + : +- SortMergeJoin LeftSemi (85) + : :- Sort (77) + : : +- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Sort (84) + : +- Project (83) + : +- Filter (82) + : +- HashAggregate (81) + : +- Exchange (80) + : +- HashAggregate (79) + : +- Scan parquet (78) + +- SortMergeJoin LeftSemi (103) + :- Sort (95) + : +- Exchange (94) + : +- Filter (93) + : +- Scan parquet (92) + +- Sort (102) + +- Project (101) + +- Filter (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Scan parquet (96) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_name#X] +Arguments: isnotnull(c_custkey#X) + +(3) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] +Input [2]: [c_custkey#X, c_name#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [c_custkey#X, c_name#X] +Arguments: X + +(8) InputAdapter +Input [2]: [c_custkey#X, c_name#X] + +(9) InputIteratorTransformer +Input [2]: [c_custkey#X, c_name#X] + +(10) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(17) InputAdapter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(18) InputIteratorTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(19) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(20) FlushableHashAggregateExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(21) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(22) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: false + +(23) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X, X + +(24) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) InputAdapter +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(27) InputIteratorTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(28) RegularHashAggregateExecTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(29) FilterExecTransformer +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(30) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(31) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(32) ProjectExecTransformer +Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: false + +(34) VeloxResizeBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X, X + +(35) ColumnarExchange +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(36) ShuffleQueryStage +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(37) InputAdapter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(38) InputIteratorTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(39) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(40) ProjectExecTransformer +Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(41) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: false + +(42) VeloxResizeBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X, X + +(43) ColumnarExchange +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(44) ShuffleQueryStage +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(45) InputAdapter +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] + +(46) InputIteratorTransformer +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] + +(47) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(48) FilterExecTransformer +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: isnotnull(l_orderkey#X) + +(49) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] +Input [2]: [l_orderkey#X, l_quantity#X] + +(50) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: false + +(51) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X, X + +(52) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [shuffle_writer_type=hash] + +(53) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_quantity#X] +Arguments: X + +(54) InputAdapter +Input [2]: [l_orderkey#X, l_quantity#X] + +(55) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_quantity#X] + +(56) ReusedExchange [Reuses operator id: 24] +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(57) ShuffleQueryStage +Output [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(58) InputAdapter +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(59) InputIteratorTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(60) RegularHashAggregateExecTransformer +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(61) FilterExecTransformer +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(62) ProjectExecTransformer +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(63) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(64) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(65) ProjectExecTransformer +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(66) RegularHashAggregateExecTransformer +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(67) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(68) TakeOrderedAndProjectExecTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 + +(69) VeloxColumnarToRow +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(70) Scan parquet +Output [2]: [c_custkey#X, c_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(71) Filter +Input [2]: [c_custkey#X, c_name#X] +Condition : isnotnull(c_custkey#X) + +(72) Exchange +Input [2]: [c_custkey#X, c_name#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(73) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(74) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(75) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) + +(76) Exchange +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(77) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(78) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(79) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(80) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(81) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(82) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(83) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(84) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(85) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(86) Exchange +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(87) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(88) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(89) Project +Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(90) Exchange +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(91) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(92) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(93) Filter +Input [2]: [l_orderkey#X, l_quantity#X] +Condition : isnotnull(l_orderkey#X) + +(94) Exchange +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(95) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet +Output [2]: [l_orderkey#X, l_quantity#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(97) HashAggregate +Input [2]: [l_orderkey#X, l_quantity#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [l_orderkey#X, sum#X, isEmpty#X] + +(98) Exchange +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(99) HashAggregate +Input [3]: [l_orderkey#X, sum#X, isEmpty#X] +Keys [1]: [l_orderkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] + +(100) Filter +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] +Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) + +(101) Project +Output [1]: [l_orderkey#X] +Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] + +(102) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(103) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(104) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(105) Project +Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] + +(106) HashAggregate +Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] + +(107) HashAggregate +Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] + +(108) TakeOrderedAndProject +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(109) AdaptiveSparkPlan +Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/19.txt new file mode 100644 index 000000000000..420c980f3273 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/19.txt @@ -0,0 +1,204 @@ +== Physical Plan == +AdaptiveSparkPlan (37) ++- == Final Plan == + VeloxColumnarToRow (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- SortMergeJoin Inner (33) + :- Sort (28) + : +- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Sort (32) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) + + +(1) ScanTransformer parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(2) FilterExecTransformer +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(3) ProjectExecTransformer +Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(8) InputAdapter +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(9) InputIteratorTransformer +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(10) ScanTransformer parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(12) ProjectExecTransformer +Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(17) InputAdapter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(18) InputIteratorTransformer +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(20) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(21) RegularHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [sum(_pre_X#X)] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(22) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(23) VeloxColumnarToRow +Input [1]: [revenue#X] + +(24) Scan parquet +Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(25) Filter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) + +(26) Project +Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(27) Exchange +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(28) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(29) Scan parquet +Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(30) Filter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) + +(31) Exchange +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(32) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(33) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: Inner +Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) + +(34) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(35) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(36) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(37) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/20.txt new file mode 100644 index 000000000000..4d8eee1f1fb7 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/20.txt @@ -0,0 +1,734 @@ +== Physical Plan == +AdaptiveSparkPlan (142) ++- == Final Plan == + VeloxColumnarToRow (92) + +- AQEShuffleRead (91) + +- ShuffleQueryStage (90), Statistics(X) + +- ColumnarExchange (89) + +- VeloxResizeBatches (88) + +- ^ ProjectExecTransformer (86) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) + :- ^ InputIteratorTransformer (75) + : +- ShuffleQueryStage (73), Statistics(X) + : +- ColumnarExchange (72) + : +- VeloxResizeBatches (71) + : +- ^ ProjectExecTransformer (69) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (68) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (67) + : +- ShuffleQueryStage (65), Statistics(X) + : +- ColumnarExchange (64) + : +- VeloxResizeBatches (63) + : +- ^ ProjectExecTransformer (61) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (60) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxResizeBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxResizeBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ ScanTransformer parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxResizeBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ ScanTransformer parquet (19) + : +- ^ InputIteratorTransformer (59) + : +- ShuffleQueryStage (57), Statistics(X) + : +- ColumnarExchange (56) + : +- VeloxResizeBatches (55) + : +- ^ ProjectExecTransformer (53) + : +- ^ FilterExecTransformer (52) + : +- ^ ProjectExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxResizeBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ ScanTransformer parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (84) + +- ShuffleQueryStage (82), Statistics(X) + +- ColumnarExchange (81) + +- VeloxResizeBatches (80) + +- ^ ProjectExecTransformer (78) + +- ^ FilterExecTransformer (77) + +- ^ ScanTransformer parquet (76) ++- == Initial Plan == + Sort (141) + +- Exchange (140) + +- Project (139) + +- SortMergeJoin Inner (138) + :- Sort (132) + : +- Exchange (131) + : +- Project (130) + : +- SortMergeJoin LeftSemi (129) + : :- Sort (96) + : : +- Exchange (95) + : : +- Filter (94) + : : +- Scan parquet (93) + : +- Sort (128) + : +- Exchange (127) + : +- Project (126) + : +- SortMergeJoin Inner (125) + : :- Sort (108) + : : +- Exchange (107) + : : +- SortMergeJoin LeftSemi (106) + : : :- Sort (100) + : : : +- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Sort (105) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Sort (124) + : +- Exchange (123) + : +- Filter (122) + : +- HashAggregate (121) + : +- HashAggregate (120) + : +- SortMergeJoin LeftSemi (119) + : :- Sort (113) + : : +- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Sort (118) + : +- Exchange (117) + : +- Project (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Sort (137) + +- Exchange (136) + +- Project (135) + +- Filter (134) + +- Scan parquet (133) + + +(1) ScanTransformer parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: isnotnull(s_nationkey#X) + +(3) ProjectExecTransformer +Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(10) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(12) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(13) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: false + +(14) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X, X + +(15) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(17) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(18) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(19) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(20) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(21) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(22) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(23) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(24) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(26) InputAdapter +Input [1]: [p_partkey#X] + +(27) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(28) ShuffledHashJoinExecTransformer +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(29) ProjectExecTransformer +Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(34) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(35) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(36) ScanTransformer parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(37) FilterExecTransformer +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(38) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(39) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: false + +(40) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X, X + +(41) ColumnarExchange +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [shuffle_writer_type=hash] + +(42) ShuffleQueryStage +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(43) InputAdapter +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] + +(44) InputIteratorTransformer +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] + +(45) ReusedExchange [Reuses operator id: 24] +Output [1]: [p_partkey#X] + +(46) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(47) InputAdapter +Input [1]: [p_partkey#X] + +(48) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(49) ShuffledHashJoinExecTransformer +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(50) RegularHashAggregateExecTransformer +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(51) ProjectExecTransformer +Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] + +(52) FilterExecTransformer +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: isnotnull((0.5 * sum(l_quantity))#X) + +(53) ProjectExecTransformer +Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(54) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: false + +(55) VeloxResizeBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X, X + +(56) ColumnarExchange +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(57) ShuffleQueryStage +Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(58) InputAdapter +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(59) InputIteratorTransformer +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(60) ShuffledHashJoinExecTransformer +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(61) ProjectExecTransformer +Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(62) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: false + +(63) VeloxResizeBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X, X + +(64) ColumnarExchange +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(65) ShuffleQueryStage +Output [1]: [ps_suppkey#X] +Arguments: X + +(66) InputAdapter +Input [1]: [ps_suppkey#X] + +(67) InputIteratorTransformer +Input [1]: [ps_suppkey#X] + +(68) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(69) ProjectExecTransformer +Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(70) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: false + +(71) VeloxResizeBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X, X + +(72) ColumnarExchange +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(73) ShuffleQueryStage +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(74) InputAdapter +Input [3]: [s_name#X, s_address#X, s_nationkey#X] + +(75) InputIteratorTransformer +Input [3]: [s_name#X, s_address#X, s_nationkey#X] + +(76) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(77) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(78) ProjectExecTransformer +Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(79) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: false + +(80) VeloxResizeBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X, X + +(81) ColumnarExchange +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(82) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(83) InputAdapter +Input [1]: [n_nationkey#X] + +(84) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(85) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(86) ProjectExecTransformer +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(87) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, s_address#X] +Arguments: false + +(88) VeloxResizeBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X, X + +(89) ColumnarExchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(90) ShuffleQueryStage +Output [2]: [s_name#X, s_address#X] +Arguments: X + +(91) AQEShuffleRead +Input [2]: [s_name#X, s_address#X] +Arguments: local + +(92) VeloxColumnarToRow +Input [2]: [s_name#X, s_address#X] + +(93) Scan parquet +Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(94) Filter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Condition : isnotnull(s_nationkey#X) + +(95) Exchange +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(96) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(98) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) + +(99) Exchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(100) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(102) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(103) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(104) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(105) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(106) SortMergeJoin +Left keys [1]: [ps_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(107) Exchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(108) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(109) Scan parquet +Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(110) Filter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) + +(111) Project +Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(112) Exchange +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(113) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(115) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) + +(116) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(117) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(118) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(119) SortMergeJoin +Left keys [1]: [l_partkey#X] +Right keys [1]: [p_partkey#X] +Join type: LeftSemi +Join condition: None + +(120) HashAggregate +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [partial_sum(l_quantity#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] + +(121) HashAggregate +Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Keys [2]: [l_partkey#X, l_suppkey#X] +Functions [1]: [sum(l_quantity#X)] +Aggregate Attributes [1]: [sum(l_quantity#X)#X] +Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(122) Filter +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Condition : isnotnull((0.5 * sum(l_quantity))#X) + +(123) Exchange +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(124) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(125) SortMergeJoin +Left keys [2]: [ps_partkey#X, ps_suppkey#X] +Right keys [2]: [l_partkey#X, l_suppkey#X] +Join type: Inner +Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) + +(126) Project +Output [1]: [ps_suppkey#X] +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] + +(127) Exchange +Input [1]: [ps_suppkey#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(128) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(129) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [ps_suppkey#X] +Join type: LeftSemi +Join condition: None + +(130) Project +Output [3]: [s_name#X, s_address#X, s_nationkey#X] +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(131) Exchange +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(132) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(133) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(134) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) + +(135) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(136) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(137) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(138) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(139) Project +Output [2]: [s_name#X, s_address#X] +Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] + +(140) Exchange +Input [2]: [s_name#X, s_address#X] +Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(141) Sort +Input [2]: [s_name#X, s_address#X] +Arguments: [s_name#X ASC NULLS FIRST], true, 0 + +(142) AdaptiveSparkPlan +Output [2]: [s_name#X, s_address#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/21.txt new file mode 100644 index 000000000000..96750b2b3ecc --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/21.txt @@ -0,0 +1,713 @@ +== Physical Plan == +AdaptiveSparkPlan (137) ++- == Final Plan == + VeloxColumnarToRow (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxResizeBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxResizeBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ ScanTransformer parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxResizeBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxResizeBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ ScanTransformer parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxResizeBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ ScanTransformer parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxResizeBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ ScanTransformer parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxResizeBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ ScanTransformer parquet (70) ++- == Initial Plan == + TakeOrderedAndProject (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (116) + : : +- Exchange (115) + : : +- Project (114) + : : +- SortMergeJoin Inner (113) + : : :- Sort (95) + : : : +- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftAnti (110) + : : :- SortMergeJoin LeftSemi (104) + : : : :- Sort (100) + : : : : +- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Sort (103) + : : : +- Exchange (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) + + +(1) ScanTransformer parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) ProjectExecTransformer +Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] + +(10) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(18) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(19) ScanTransformer parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(20) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [2]: [l_orderkey#X, l_suppkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(25) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(26) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(27) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(28) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(29) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(30) ProjectExecTransformer +Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(31) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(32) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(33) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(34) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(35) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(36) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(37) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(38) ProjectExecTransformer +Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Input [2]: [l_orderkey#X, l_suppkey#X] + +(39) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: false + +(40) VeloxResizeBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X, X + +(41) ColumnarExchange +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(42) ShuffleQueryStage +Output [2]: [l_orderkey#X, l_suppkey#X] +Arguments: X + +(43) InputAdapter +Input [2]: [l_orderkey#X, l_suppkey#X] + +(44) InputIteratorTransformer +Input [2]: [l_orderkey#X, l_suppkey#X] + +(45) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(46) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(47) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: false + +(48) VeloxResizeBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X, X + +(49) ColumnarExchange +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(51) InputAdapter +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] + +(52) InputIteratorTransformer +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] + +(53) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(54) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(55) ProjectExecTransformer +Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(56) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: false + +(57) VeloxResizeBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X, X + +(58) ColumnarExchange +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(59) ShuffleQueryStage +Output [1]: [o_orderkey#X] +Arguments: X + +(60) InputAdapter +Input [1]: [o_orderkey#X] + +(61) InputIteratorTransformer +Input [1]: [o_orderkey#X] + +(62) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(63) ProjectExecTransformer +Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [s_name#X, s_nationkey#X] +Arguments: X + +(68) InputAdapter +Input [2]: [s_name#X, s_nationkey#X] + +(69) InputIteratorTransformer +Input [2]: [s_name#X, s_nationkey#X] + +(70) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(71) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(72) ProjectExecTransformer +Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(73) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: false + +(74) VeloxResizeBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X, X + +(75) ColumnarExchange +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(76) ShuffleQueryStage +Output [1]: [n_nationkey#X] +Arguments: X + +(77) InputAdapter +Input [1]: [n_nationkey#X] + +(78) InputIteratorTransformer +Input [1]: [n_nationkey#X] + +(79) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(80) ProjectExecTransformer +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(81) FlushableHashAggregateExecTransformer +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(82) ProjectExecTransformer +Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] +Input [2]: [s_name#X, count#X] + +(83) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: false + +(84) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X, X + +(85) ColumnarExchange +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(86) ShuffleQueryStage +Output [2]: [s_name#X, count#X] +Arguments: X + +(87) InputAdapter +Input [2]: [s_name#X, count#X] + +(88) InputIteratorTransformer +Input [2]: [s_name#X, count#X] + +(89) RegularHashAggregateExecTransformer +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(90) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, numwait#X] +Arguments: false + +(91) VeloxColumnarToRow +Input [2]: [s_name#X, numwait#X] + +(92) Scan parquet +Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(93) Filter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(94) Exchange +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(95) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(97) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(98) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(99) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(100) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet +Output [2]: [l_orderkey#X, l_suppkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(102) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(103) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(105) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(106) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) + +(107) Project +Output [2]: [l_orderkey#X, l_suppkey#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(108) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(109) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftAnti +Join condition: NOT (l_suppkey#X = l_suppkey#X) + +(111) Exchange +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(112) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(113) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(114) Project +Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] + +(115) Exchange +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(116) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(117) Scan parquet +Output [2]: [o_orderkey#X, o_orderstatus#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(118) Filter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) + +(119) Project +Output [1]: [o_orderkey#X] +Input [2]: [o_orderkey#X, o_orderstatus#X] + +(120) Exchange +Input [1]: [o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(121) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(123) Project +Output [2]: [s_name#X, s_nationkey#X] +Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] + +(124) Exchange +Input [2]: [s_name#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(125) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(127) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) + +(128) Project +Output [1]: [n_nationkey#X] +Input [2]: [n_nationkey#X, n_name#X] + +(129) Exchange +Input [1]: [n_nationkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(132) Project +Output [1]: [s_name#X] +Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] + +(133) HashAggregate +Input [1]: [s_name#X] +Keys [1]: [s_name#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [s_name#X, count#X] + +(134) Exchange +Input [2]: [s_name#X, count#X] +Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(135) HashAggregate +Input [2]: [s_name#X, count#X] +Keys [1]: [s_name#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [s_name#X, count(1)#X AS numwait#X] + +(136) TakeOrderedAndProject +Input [2]: [s_name#X, numwait#X] +Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] + +(137) AdaptiveSparkPlan +Output [2]: [s_name#X, numwait#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/22.txt new file mode 100644 index 000000000000..c6e147da04d8 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/22.txt @@ -0,0 +1,412 @@ +== Physical Plan == +AdaptiveSparkPlan (52) ++- == Final Plan == + VeloxColumnarToRow (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxResizeBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxResizeBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxResizeBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) + + +(1) ScanTransformer parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(3) ProjectExecTransformer +Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(4) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: false + +(5) VeloxResizeBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X, X + +(6) ColumnarExchange +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(8) InputAdapter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(9) InputIteratorTransformer +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(10) ScanTransformer parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(11) ProjectExecTransformer +Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] +Input [1]: [o_custkey#X] + +(12) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: false + +(13) VeloxResizeBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X, X + +(14) ColumnarExchange +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(15) ShuffleQueryStage +Output [1]: [o_custkey#X] +Arguments: X + +(16) InputAdapter +Input [1]: [o_custkey#X] + +(17) InputIteratorTransformer +Input [1]: [o_custkey#X] + +(18) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(19) ProjectExecTransformer +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(20) FlushableHashAggregateExecTransformer +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(21) ProjectExecTransformer +Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(22) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: false + +(23) VeloxResizeBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X, X + +(24) ColumnarExchange +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(25) ShuffleQueryStage +Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(26) InputAdapter +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(27) InputIteratorTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(28) RegularHashAggregateExecTransformer +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(29) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(30) VeloxResizeBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X, X + +(31) ColumnarExchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(32) ShuffleQueryStage +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(33) InputAdapter +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(34) InputIteratorTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(35) SortExecTransformer +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(36) WholeStageCodegenTransformer (X) +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: false + +(37) VeloxColumnarToRow +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] + +(38) Scan parquet +Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(39) Filter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) + +(40) Exchange +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet +Output [1]: [o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +ReadSchema: struct + +(43) Exchange +Input [1]: [o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: LeftAnti +Join condition: None + +(46) Project +Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] + +(47) HashAggregate +Input [2]: [cntrycode#X, c_acctbal#X] +Keys [1]: [cntrycode#X] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] +Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] +Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] + +(48) Exchange +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(49) HashAggregate +Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] +Keys [1]: [cntrycode#X] +Functions [2]: [count(1), sum(c_acctbal#X)] +Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] +Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] + +(50) Exchange +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(51) Sort +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 + +(52) AdaptiveSparkPlan +Output [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (72) ++- == Final Plan == + VeloxColumnarToRow (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ ScanTransformer parquet (53) ++- == Initial Plan == + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) + + +(53) ScanTransformer parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(54) FilterExecTransformer +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(55) ProjectExecTransformer +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(56) FlushableHashAggregateExecTransformer +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(57) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, count#X] +Arguments: false + +(58) VeloxResizeBatches +Input [2]: [sum#X, count#X] +Arguments: X, X + +(59) ColumnarExchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(60) ShuffleQueryStage +Output [2]: [sum#X, count#X] +Arguments: X + +(61) InputAdapter +Input [2]: [sum#X, count#X] + +(62) InputIteratorTransformer +Input [2]: [sum#X, count#X] + +(63) RegularHashAggregateExecTransformer +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(64) WholeStageCodegenTransformer (X) +Input [1]: [avg(c_acctbal)#X] +Arguments: false + +(65) VeloxColumnarToRow +Input [1]: [avg(c_acctbal)#X] + +(66) Scan parquet +Output [2]: [c_phone#X, c_acctbal#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(67) Filter +Input [2]: [c_phone#X, c_acctbal#X] +Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) + +(68) Project +Output [1]: [c_acctbal#X] +Input [2]: [c_phone#X, c_acctbal#X] + +(69) HashAggregate +Input [1]: [c_acctbal#X] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#X)] +Aggregate Attributes [2]: [sum#X, count#X] +Results [2]: [sum#X, count#X] + +(70) Exchange +Input [2]: [sum#X, count#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(71) HashAggregate +Input [2]: [sum#X, count#X] +Keys: [] +Functions [1]: [avg(c_acctbal#X)] +Aggregate Attributes [1]: [avg(c_acctbal#X)#X] +Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] + +(72) AdaptiveSparkPlan +Output [1]: [avg(c_acctbal)#X] +Arguments: isFinalPlan=true + +Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (72) ++- == Final Plan == + VeloxColumnarToRow (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxResizeBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ ScanTransformer parquet (53) ++- == Initial Plan == + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/3.txt new file mode 100644 index 000000000000..d6573f7483ba --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/3.txt @@ -0,0 +1,351 @@ +== Physical Plan == +AdaptiveSparkPlan (66) ++- == Final Plan == + VeloxColumnarToRow (42) + +- TakeOrderedAndProjectExecTransformer (41) + +- ^ ProjectExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxResizeBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxResizeBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ ScanTransformer parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxResizeBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ ScanTransformer parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ ScanTransformer parquet (27) ++- == Initial Plan == + TakeOrderedAndProject (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (55) + : +- Exchange (54) + : +- Project (53) + : +- SortMergeJoin Inner (52) + : :- Sort (47) + : : +- Exchange (46) + : : +- Project (45) + : : +- Filter (44) + : : +- Scan parquet (43) + : +- Sort (51) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Sort (60) + +- Exchange (59) + +- Project (58) + +- Filter (57) + +- Scan parquet (56) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(3) ProjectExecTransformer +Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(4) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [1]: [c_custkey#X] +Arguments: X + +(8) InputAdapter +Input [1]: [c_custkey#X] + +(9) InputIteratorTransformer +Input [1]: [c_custkey#X] + +(10) ScanTransformer parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(13) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(14) VeloxResizeBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X, X + +(15) ColumnarExchange +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(17) InputAdapter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(18) InputIteratorTransformer +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(21) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(22) VeloxResizeBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X, X + +(23) ColumnarExchange +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(25) InputAdapter +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] + +(26) InputIteratorTransformer +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] + +(27) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(29) ProjectExecTransformer +Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(30) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(31) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(32) ColumnarExchange +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(34) InputAdapter +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(35) InputIteratorTransformer +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(38) RegularHashAggregateExecTransformer +Input [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, _pre_X#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum(_pre_X#X)] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(39) ProjectExecTransformer +Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] +Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] + +(40) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(41) TakeOrderedAndProjectExecTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 + +(42) VeloxColumnarToRow +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(43) Scan parquet +Output [2]: [c_custkey#X, c_mktsegment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(44) Filter +Input [2]: [c_custkey#X, c_mktsegment#X] +Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) + +(45) Project +Output [1]: [c_custkey#X] +Input [2]: [c_custkey#X, c_mktsegment#X] + +(46) Exchange +Input [1]: [c_custkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(47) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(48) Scan parquet +Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(49) Filter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(50) Exchange +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(51) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(52) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(53) Project +Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(54) Exchange +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(55) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(56) Scan parquet +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(57) Filter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) + +(58) Project +Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(59) Exchange +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(60) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(62) Project +Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] + +(63) HashAggregate +Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] + +(64) HashAggregate +Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] + +(65) TakeOrderedAndProject +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(66) AdaptiveSparkPlan +Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/4.txt new file mode 100644 index 000000000000..71a8f6974ed1 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/4.txt @@ -0,0 +1,294 @@ +== Physical Plan == +AdaptiveSparkPlan (56) ++- == Final Plan == + VeloxColumnarToRow (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxResizeBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxResizeBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxResizeBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ ScanTransformer parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxResizeBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ ScanTransformer parquet (10) ++- == Initial Plan == + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) + + +(1) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(3) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: X + +(8) InputAdapter +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(9) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(10) ScanTransformer parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(12) ProjectExecTransformer +Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [1]: [l_orderkey#X] +Arguments: X + +(17) InputAdapter +Input [1]: [l_orderkey#X] + +(18) InputIteratorTransformer +Input [1]: [l_orderkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(20) ProjectExecTransformer +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(21) FlushableHashAggregateExecTransformer +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(22) ProjectExecTransformer +Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] +Input [2]: [o_orderpriority#X, count#X] + +(23) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: false + +(24) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X, X + +(25) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [shuffle_writer_type=hash] + +(26) ShuffleQueryStage +Output [2]: [o_orderpriority#X, count#X] +Arguments: X + +(27) InputAdapter +Input [2]: [o_orderpriority#X, count#X] + +(28) InputIteratorTransformer +Input [2]: [o_orderpriority#X, count#X] + +(29) RegularHashAggregateExecTransformer +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(30) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(31) VeloxResizeBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X, X + +(32) ColumnarExchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(34) InputAdapter +Input [2]: [o_orderpriority#X, order_count#X] + +(35) InputIteratorTransformer +Input [2]: [o_orderpriority#X, order_count#X] + +(36) SortExecTransformer +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(37) WholeStageCodegenTransformer (X) +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: false + +(38) VeloxColumnarToRow +Input [2]: [o_orderpriority#X, order_count#X] + +(39) Scan parquet +Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(40) Filter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) + +(41) Project +Output [2]: [o_orderkey#X, o_orderpriority#X] +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(42) Exchange +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet +Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(45) Filter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) + +(46) Project +Output [1]: [l_orderkey#X] +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(47) Exchange +Input [1]: [l_orderkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: LeftSemi +Join condition: None + +(50) Project +Output [1]: [o_orderpriority#X] +Input [2]: [o_orderkey#X, o_orderpriority#X] + +(51) HashAggregate +Input [1]: [o_orderpriority#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#X] +Results [2]: [o_orderpriority#X, count#X] + +(52) Exchange +Input [2]: [o_orderpriority#X, count#X] +Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(53) HashAggregate +Input [2]: [o_orderpriority#X, count#X] +Keys [1]: [o_orderpriority#X] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#X] +Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] + +(54) Exchange +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(55) Sort +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 + +(56) AdaptiveSparkPlan +Output [2]: [o_orderpriority#X, order_count#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/5.txt new file mode 100644 index 000000000000..a78e477ace2c --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/5.txt @@ -0,0 +1,802 @@ +== Physical Plan == +AdaptiveSparkPlan (156) ++- == Final Plan == + VeloxColumnarToRow (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxResizeBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxResizeBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxResizeBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxResizeBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxResizeBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxResizeBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxResizeBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxResizeBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ ScanTransformer parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ ScanTransformer parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ ScanTransformer parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxResizeBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ ScanTransformer parquet (78) ++- == Initial Plan == + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) + + +(1) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(3) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] +Input [2]: [c_custkey#X, c_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(9) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(10) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(12) ProjectExecTransformer +Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(14) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(15) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(17) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(18) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(21) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: false + +(22) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X, X + +(23) ColumnarExchange +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [2]: [c_nationkey#X, o_orderkey#X] +Arguments: X + +(25) InputAdapter +Input [2]: [c_nationkey#X, o_orderkey#X] + +(26) InputIteratorTransformer +Input [2]: [c_nationkey#X, o_orderkey#X] + +(27) ScanTransformer parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(29) ProjectExecTransformer +Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(30) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(31) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(32) ColumnarExchange +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(34) InputAdapter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(35) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(38) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(39) VeloxResizeBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(40) ColumnarExchange +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(42) InputAdapter +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(43) InputIteratorTransformer +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(44) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(46) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(47) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(48) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(49) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(51) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(52) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(55) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: false + +(56) VeloxResizeBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X, X + +(57) ColumnarExchange +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(59) InputAdapter +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] + +(60) InputIteratorTransformer +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] + +(61) ScanTransformer parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(63) ProjectExecTransformer +Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(64) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: false + +(65) VeloxResizeBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X, X + +(66) ColumnarExchange +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(68) InputAdapter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(69) InputIteratorTransformer +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(72) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: false + +(73) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X, X + +(74) ColumnarExchange +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(76) InputAdapter +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] + +(77) InputIteratorTransformer +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] + +(78) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(79) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(80) ProjectExecTransformer +Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(81) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: false + +(82) VeloxResizeBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X, X + +(83) ColumnarExchange +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(84) ShuffleQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(85) InputAdapter +Input [1]: [r_regionkey#X] + +(86) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(87) ShuffledHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(88) ProjectExecTransformer +Output [2]: [n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(89) FlushableHashAggregateExecTransformer +Input [2]: [n_name#X, _pre_X#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(90) ProjectExecTransformer +Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(91) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: false + +(92) VeloxResizeBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X, X + +(93) ColumnarExchange +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(94) ShuffleQueryStage +Output [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: X + +(95) InputAdapter +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(96) InputIteratorTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] + +(97) RegularHashAggregateExecTransformer +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(98) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(99) VeloxResizeBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X, X + +(100) ColumnarExchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [2]: [n_name#X, revenue#X] +Arguments: X + +(102) InputAdapter +Input [2]: [n_name#X, revenue#X] + +(103) InputIteratorTransformer +Input [2]: [n_name#X, revenue#X] + +(104) SortExecTransformer +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(105) WholeStageCodegenTransformer (X) +Input [2]: [n_name#X, revenue#X] +Arguments: false + +(106) VeloxColumnarToRow +Input [2]: [n_name#X, revenue#X] + +(107) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(108) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(109) Exchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(112) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) + +(113) Project +Output [2]: [o_orderkey#X, o_custkey#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(114) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin +Left keys [1]: [c_custkey#X] +Right keys [1]: [o_custkey#X] +Join type: Inner +Join condition: None + +(117) Project +Output [2]: [c_nationkey#X, o_orderkey#X] +Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] + +(118) Exchange +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(121) Filter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) + +(122) Exchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin +Left keys [1]: [o_orderkey#X] +Right keys [1]: [l_orderkey#X] +Join type: Inner +Join condition: None + +(125) Project +Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(126) Exchange +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(129) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(130) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin +Left keys [2]: [l_suppkey#X, c_nationkey#X] +Right keys [2]: [s_suppkey#X, s_nationkey#X] +Join type: Inner +Join condition: None + +(133) Project +Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(134) Exchange +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet +Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(137) Filter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(138) Exchange +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(141) Project +Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] + +(142) Exchange +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(145) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) + +(146) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(147) Exchange +Input [1]: [r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(150) Project +Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] + +(151) HashAggregate +Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] +Keys [1]: [n_name#X] +Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [3]: [n_name#X, sum#X, isEmpty#X] + +(152) Exchange +Input [3]: [n_name#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(153) HashAggregate +Input [3]: [n_name#X, sum#X, isEmpty#X] +Keys [1]: [n_name#X] +Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] +Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] + +(154) Exchange +Input [2]: [n_name#X, revenue#X] +Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(155) Sort +Input [2]: [n_name#X, revenue#X] +Arguments: [revenue#X DESC NULLS LAST], true, 0 + +(156) AdaptiveSparkPlan +Output [2]: [n_name#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/6.txt new file mode 100644 index 000000000000..b85e63e10655 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/6.txt @@ -0,0 +1,116 @@ +== Physical Plan == +AdaptiveSparkPlan (20) ++- == Final Plan == + VeloxColumnarToRow (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxResizeBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ ScanTransformer parquet (1) ++- == Initial Plan == + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) + + +(1) ScanTransformer parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(3) ProjectExecTransformer +Output [1]: [(l_extendedprice#X * l_discount#X) AS _pre_X#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(4) FlushableHashAggregateExecTransformer +Input [1]: [_pre_X#X] +Keys: [] +Functions [1]: [partial_sum(_pre_X#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(5) WholeStageCodegenTransformer (X) +Input [2]: [sum#X, isEmpty#X] +Arguments: false + +(6) VeloxResizeBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X, X + +(7) ColumnarExchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(8) ShuffleQueryStage +Output [2]: [sum#X, isEmpty#X] +Arguments: X + +(9) InputAdapter +Input [2]: [sum#X, isEmpty#X] + +(10) InputIteratorTransformer +Input [2]: [sum#X, isEmpty#X] + +(11) RegularHashAggregateExecTransformer +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(12) WholeStageCodegenTransformer (X) +Input [1]: [revenue#X] +Arguments: false + +(13) VeloxColumnarToRow +Input [1]: [revenue#X] + +(14) Scan parquet +Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(15) Filter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) + +(16) Project +Output [2]: [l_extendedprice#X, l_discount#X] +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(17) HashAggregate +Input [2]: [l_extendedprice#X, l_discount#X] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [2]: [sum#X, isEmpty#X] + +(18) Exchange +Input [2]: [sum#X, isEmpty#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] + +(19) HashAggregate +Input [2]: [sum#X, isEmpty#X] +Keys: [] +Functions [1]: [sum((l_extendedprice#X * l_discount#X))] +Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] +Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] + +(20) AdaptiveSparkPlan +Output [1]: [revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/7.txt new file mode 100644 index 000000000000..a0894489f0a5 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/7.txt @@ -0,0 +1,764 @@ +== Physical Plan == +AdaptiveSparkPlan (149) ++- == Final Plan == + VeloxColumnarToRow (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxResizeBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxResizeBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxResizeBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxResizeBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxResizeBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxResizeBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxResizeBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxResizeBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ ScanTransformer parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ ScanTransformer parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ ScanTransformer parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) ++- == Initial Plan == + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) + + +(1) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(3) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(4) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(8) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(9) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(10) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(13) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: false + +(14) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X, X + +(15) ColumnarExchange +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(17) InputAdapter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(18) InputIteratorTransformer +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(21) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: false + +(22) VeloxResizeBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X, X + +(23) ColumnarExchange +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(25) InputAdapter +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) InputIteratorTransformer +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(27) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(29) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] +Input [2]: [o_orderkey#X, o_custkey#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_custkey#X] +Arguments: X + +(34) InputAdapter +Input [2]: [o_orderkey#X, o_custkey#X] + +(35) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_custkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(38) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: false + +(39) VeloxResizeBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X, X + +(40) ColumnarExchange +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(42) InputAdapter +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] + +(43) InputIteratorTransformer +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] + +(44) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(46) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] +Input [2]: [c_custkey#X, c_nationkey#X] + +(47) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: false + +(48) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X, X + +(49) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(51) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(52) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(55) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: false + +(56) VeloxResizeBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X, X + +(57) ColumnarExchange +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(59) InputAdapter +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] + +(60) InputIteratorTransformer +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] + +(61) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(62) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(63) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(68) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(69) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(72) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: false + +(73) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X, X + +(74) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(76) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] + +(77) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] + +(78) ReusedExchange [Reuses operator id: 66] +Output [2]: [n_nationkey#X, n_name#X] + +(79) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(80) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(81) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(82) ShuffledHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(83) ProjectExecTransformer +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(84) FlushableHashAggregateExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(85) ProjectExecTransformer +Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(86) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: false + +(87) VeloxResizeBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(88) ColumnarExchange +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(89) ShuffleQueryStage +Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(90) InputAdapter +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(91) InputIteratorTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(92) RegularHashAggregateExecTransformer +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(93) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(94) VeloxResizeBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X, X + +(95) ColumnarExchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(96) ShuffleQueryStage +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(97) InputAdapter +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(98) InputIteratorTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(99) SortExecTransformer +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(100) WholeStageCodegenTransformer (X) +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: false + +(101) VeloxColumnarToRow +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] + +(102) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(103) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(104) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet +Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(107) Filter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(108) Exchange +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin +Left keys [1]: [s_suppkey#X] +Right keys [1]: [l_suppkey#X] +Join type: Inner +Join condition: None + +(111) Project +Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(112) Exchange +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet +Output [2]: [o_orderkey#X, o_custkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(115) Filter +Input [2]: [o_orderkey#X, o_custkey#X] +Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) + +(116) Exchange +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(119) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] + +(120) Exchange +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(123) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(124) Exchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(127) Project +Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] + +(128) Exchange +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(131) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) + +(132) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(135) Project +Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] + +(136) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] +ReadSchema: struct + +(139) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) + +(140) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) + +(143) Project +Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] +Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] + +(144) HashAggregate +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [partial_sum(volume#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] + +(145) Exchange +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(146) HashAggregate +Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] +Functions [1]: [sum(volume#X)] +Aggregate Attributes [1]: [sum(volume#X)#X] +Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] + +(147) Exchange +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(148) Sort +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 + +(149) AdaptiveSparkPlan +Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/8.txt new file mode 100644 index 000000000000..88a6a79e074e --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/8.txt @@ -0,0 +1,1061 @@ +== Physical Plan == +AdaptiveSparkPlan (207) ++- == Final Plan == + VeloxColumnarToRow (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxResizeBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxResizeBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxResizeBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxResizeBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxResizeBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxResizeBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxResizeBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxResizeBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxResizeBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ ScanTransformer parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxResizeBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ ScanTransformer parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxResizeBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ ScanTransformer parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxResizeBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ ScanTransformer parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxResizeBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ ScanTransformer parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxResizeBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ ScanTransformer parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxResizeBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ ScanTransformer parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxResizeBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ ScanTransformer parquet (112) ++- == Initial Plan == + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_type#X] +Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(4) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(8) InputAdapter +Input [1]: [p_partkey#X] + +(9) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(10) ScanTransformer parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(13) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(14) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(15) ColumnarExchange +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(17) InputAdapter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(18) InputIteratorTransformer +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(21) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(22) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(23) ColumnarExchange +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(25) InputAdapter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(27) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(29) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(34) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(35) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(38) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: false + +(39) VeloxResizeBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X, X + +(40) ColumnarExchange +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(42) InputAdapter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(43) InputIteratorTransformer +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(44) ScanTransformer parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(46) ProjectExecTransformer +Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(47) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: false + +(48) VeloxResizeBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X, X + +(49) ColumnarExchange +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(51) InputAdapter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(52) InputIteratorTransformer +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(55) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: false + +(56) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X, X + +(57) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(59) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] + +(60) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] + +(61) ScanTransformer parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(63) ProjectExecTransformer +Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] +Input [2]: [c_custkey#X, c_nationkey#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [c_custkey#X, c_nationkey#X] +Arguments: X + +(68) InputAdapter +Input [2]: [c_custkey#X, c_nationkey#X] + +(69) InputIteratorTransformer +Input [2]: [c_custkey#X, c_nationkey#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(72) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: false + +(73) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X, X + +(74) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(76) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] + +(77) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] + +(78) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(79) FilterExecTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(80) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Input [2]: [n_nationkey#X, n_regionkey#X] + +(81) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: false + +(82) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X, X + +(83) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(84) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_regionkey#X] +Arguments: X + +(85) InputAdapter +Input [2]: [n_nationkey#X, n_regionkey#X] + +(86) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_regionkey#X] + +(87) ShuffledHashJoinExecTransformer +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(88) ProjectExecTransformer +Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(89) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: false + +(90) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X, X + +(91) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(92) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(93) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] + +(94) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] + +(95) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(96) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(97) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(98) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(99) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(100) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(102) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(103) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(104) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(105) ProjectExecTransformer +Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(106) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: false + +(107) VeloxResizeBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X, X + +(108) ColumnarExchange +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(109) ShuffleQueryStage +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(110) InputAdapter +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] + +(111) InputIteratorTransformer +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] + +(112) ScanTransformer parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(113) FilterExecTransformer +Input [2]: [r_regionkey#X, r_name#X] +Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(114) ProjectExecTransformer +Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(115) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: false + +(116) VeloxResizeBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X, X + +(117) ColumnarExchange +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(118) ShuffleQueryStage +Output [1]: [r_regionkey#X] +Arguments: X + +(119) InputAdapter +Input [1]: [r_regionkey#X] + +(120) InputIteratorTransformer +Input [1]: [r_regionkey#X] + +(121) ShuffledHashJoinExecTransformer +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(122) ProjectExecTransformer +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(123) FlushableHashAggregateExecTransformer +Input [3]: [o_year#X, volume#X, _pre_X#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(124) ProjectExecTransformer +Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(125) WholeStageCodegenTransformer (X) +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: false + +(126) VeloxResizeBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X, X + +(127) ColumnarExchange +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(128) ShuffleQueryStage +Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(129) InputAdapter +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(130) InputIteratorTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(131) RegularHashAggregateExecTransformer +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(132) ProjectExecTransformer +Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] +Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] + +(133) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(134) VeloxResizeBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X, X + +(135) ColumnarExchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(136) ShuffleQueryStage +Output [2]: [o_year#X, mkt_share#X] +Arguments: X + +(137) InputAdapter +Input [2]: [o_year#X, mkt_share#X] + +(138) InputIteratorTransformer +Input [2]: [o_year#X, mkt_share#X] + +(139) SortExecTransformer +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(140) WholeStageCodegenTransformer (X) +Input [2]: [o_year#X, mkt_share#X] +Arguments: false + +(141) VeloxColumnarToRow +Input [2]: [o_year#X, mkt_share#X] + +(142) Scan parquet +Output [2]: [p_partkey#X, p_type#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(143) Filter +Input [2]: [p_partkey#X, p_type#X] +Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) + +(144) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_type#X] + +(145) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet +Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(148) Filter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(149) Exchange +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(152) Project +Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(153) Exchange +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(156) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(157) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(160) Project +Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(161) Exchange +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet +Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(164) Filter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) + +(165) Exchange +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(168) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] + +(169) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet +Output [2]: [c_custkey#X, c_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(172) Filter +Input [2]: [c_custkey#X, c_nationkey#X] +Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) + +(173) Exchange +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin +Left keys [1]: [o_custkey#X] +Right keys [1]: [c_custkey#X] +Join type: Inner +Join condition: None + +(176) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] + +(177) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet +Output [2]: [n_nationkey#X, n_regionkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(180) Filter +Input [2]: [n_nationkey#X, n_regionkey#X] +Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) + +(181) Exchange +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin +Left keys [1]: [c_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(184) Project +Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] + +(185) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(188) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(189) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(192) Project +Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] + +(193) Exchange +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet +Output [2]: [r_regionkey#X, r_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(196) Filter +Input [2]: [r_regionkey#X, r_name#X] +Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) + +(197) Project +Output [1]: [r_regionkey#X] +Input [2]: [r_regionkey#X, r_name#X] + +(198) Exchange +Input [1]: [r_regionkey#X] +Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin +Left keys [1]: [n_regionkey#X] +Right keys [1]: [r_regionkey#X] +Join type: Inner +Join condition: None + +(201) Project +Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] +Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] + +(202) HashAggregate +Input [3]: [o_year#X, volume#X, nation#X] +Keys [1]: [o_year#X] +Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] +Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] + +(203) Exchange +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(204) HashAggregate +Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Keys [1]: [o_year#X] +Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] +Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] + +(205) Exchange +Input [2]: [o_year#X, mkt_share#X] +Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(206) Sort +Input [2]: [o_year#X, mkt_share#X] +Arguments: [o_year#X ASC NULLS FIRST], true, 0 + +(207) AdaptiveSparkPlan +Output [2]: [o_year#X, mkt_share#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/9.txt new file mode 100644 index 000000000000..4f537e91e132 --- /dev/null +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark35/9.txt @@ -0,0 +1,797 @@ +== Physical Plan == +AdaptiveSparkPlan (155) ++- == Final Plan == + VeloxColumnarToRow (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxResizeBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxResizeBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxResizeBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxResizeBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxResizeBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxResizeBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxResizeBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ ScanTransformer parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxResizeBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ ScanTransformer parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxResizeBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ ScanTransformer parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxResizeBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ ScanTransformer parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxResizeBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ ScanTransformer parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxResizeBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ ScanTransformer parquet (78) ++- == Initial Plan == + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) + + +(1) ScanTransformer parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) FilterExecTransformer +Input [2]: [p_partkey#X, p_name#X] +Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(3) ProjectExecTransformer +Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(4) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: false + +(5) VeloxResizeBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X, X + +(6) ColumnarExchange +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(7) ShuffleQueryStage +Output [1]: [p_partkey#X] +Arguments: X + +(8) InputAdapter +Input [1]: [p_partkey#X] + +(9) InputIteratorTransformer +Input [1]: [p_partkey#X] + +(10) ScanTransformer parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) FilterExecTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(12) ProjectExecTransformer +Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(13) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(14) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(15) ColumnarExchange +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(16) ShuffleQueryStage +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(17) InputAdapter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(18) InputIteratorTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(19) ShuffledHashJoinExecTransformer +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(20) ProjectExecTransformer +Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(21) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: false + +(22) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X, X + +(23) ColumnarExchange +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] + +(24) ShuffleQueryStage +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(25) InputAdapter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(26) InputIteratorTransformer +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(27) ScanTransformer parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(28) FilterExecTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(29) ProjectExecTransformer +Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Input [2]: [s_suppkey#X, s_nationkey#X] + +(30) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: false + +(31) VeloxResizeBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X, X + +(32) ColumnarExchange +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(33) ShuffleQueryStage +Output [2]: [s_suppkey#X, s_nationkey#X] +Arguments: X + +(34) InputAdapter +Input [2]: [s_suppkey#X, s_nationkey#X] + +(35) InputIteratorTransformer +Input [2]: [s_suppkey#X, s_nationkey#X] + +(36) ShuffledHashJoinExecTransformer +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(37) ProjectExecTransformer +Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(38) WholeStageCodegenTransformer (X) +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: false + +(39) VeloxResizeBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X, X + +(40) ColumnarExchange +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] + +(41) ShuffleQueryStage +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(42) InputAdapter +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(43) InputIteratorTransformer +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] + +(44) ScanTransformer parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(45) FilterExecTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(46) ProjectExecTransformer +Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(47) WholeStageCodegenTransformer (X) +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: false + +(48) VeloxResizeBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X, X + +(49) ColumnarExchange +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(50) ShuffleQueryStage +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(51) InputAdapter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(52) InputIteratorTransformer +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(53) ShuffledHashJoinExecTransformer +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(54) ProjectExecTransformer +Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(55) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: false + +(56) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X, X + +(57) ColumnarExchange +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] + +(58) ShuffleQueryStage +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(59) InputAdapter +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] + +(60) InputIteratorTransformer +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] + +(61) ScanTransformer parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(62) FilterExecTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: isnotnull(o_orderkey#X) + +(63) ProjectExecTransformer +Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Input [2]: [o_orderkey#X, o_orderdate#X] + +(64) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: false + +(65) VeloxResizeBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X, X + +(66) ColumnarExchange +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(67) ShuffleQueryStage +Output [2]: [o_orderkey#X, o_orderdate#X] +Arguments: X + +(68) InputAdapter +Input [2]: [o_orderkey#X, o_orderdate#X] + +(69) InputIteratorTransformer +Input [2]: [o_orderkey#X, o_orderdate#X] + +(70) ShuffledHashJoinExecTransformer +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(71) ProjectExecTransformer +Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(72) WholeStageCodegenTransformer (X) +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: false + +(73) VeloxResizeBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X, X + +(74) ColumnarExchange +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] + +(75) ShuffleQueryStage +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(76) InputAdapter +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] + +(77) InputIteratorTransformer +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] + +(78) ScanTransformer parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(79) FilterExecTransformer +Input [2]: [n_nationkey#X, n_name#X] +Arguments: isnotnull(n_nationkey#X) + +(80) ProjectExecTransformer +Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] +Input [2]: [n_nationkey#X, n_name#X] + +(81) WholeStageCodegenTransformer (X) +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: false + +(82) VeloxResizeBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X, X + +(83) ColumnarExchange +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] + +(84) ShuffleQueryStage +Output [2]: [n_nationkey#X, n_name#X] +Arguments: X + +(85) InputAdapter +Input [2]: [n_nationkey#X, n_name#X] + +(86) InputIteratorTransformer +Input [2]: [n_nationkey#X, n_name#X] + +(87) ShuffledHashJoinExecTransformer +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(88) ProjectExecTransformer +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(89) FlushableHashAggregateExecTransformer +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(90) ProjectExecTransformer +Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(91) WholeStageCodegenTransformer (X) +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: false + +(92) VeloxResizeBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X, X + +(93) ColumnarExchange +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] + +(94) ShuffleQueryStage +Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(95) InputAdapter +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(96) InputIteratorTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(97) RegularHashAggregateExecTransformer +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(98) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(99) VeloxResizeBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X, X + +(100) ColumnarExchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] + +(101) ShuffleQueryStage +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(102) InputAdapter +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(103) InputIteratorTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(104) SortExecTransformer +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(105) WholeStageCodegenTransformer (X) +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: false + +(106) VeloxColumnarToRow +Input [3]: [nation#X, o_year#X, sum_profit#X] + +(107) Scan parquet +Output [2]: [p_partkey#X, p_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(108) Filter +Input [2]: [p_partkey#X, p_name#X] +Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) + +(109) Project +Output [1]: [p_partkey#X] +Input [2]: [p_partkey#X, p_name#X] + +(110) Exchange +Input [1]: [p_partkey#X] +Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(113) Filter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) + +(114) Exchange +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin +Left keys [1]: [p_partkey#X] +Right keys [1]: [l_partkey#X] +Join type: Inner +Join condition: None + +(117) Project +Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(118) Exchange +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet +Output [2]: [s_suppkey#X, s_nationkey#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(121) Filter +Input [2]: [s_suppkey#X, s_nationkey#X] +Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) + +(122) Exchange +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin +Left keys [1]: [l_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: Inner +Join condition: None + +(125) Project +Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] + +(126) Exchange +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet +Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(129) Filter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) + +(130) Exchange +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin +Left keys [2]: [l_suppkey#X, l_partkey#X] +Right keys [2]: [ps_suppkey#X, ps_partkey#X] +Join type: Inner +Join condition: None + +(133) Project +Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(134) Exchange +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet +Output [2]: [o_orderkey#X, o_orderdate#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(137) Filter +Input [2]: [o_orderkey#X, o_orderdate#X] +Condition : isnotnull(o_orderkey#X) + +(138) Exchange +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin +Left keys [1]: [l_orderkey#X] +Right keys [1]: [o_orderkey#X] +Join type: Inner +Join condition: None + +(141) Project +Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] + +(142) Exchange +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet +Output [2]: [n_nationkey#X, n_name#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(145) Filter +Input [2]: [n_nationkey#X, n_name#X] +Condition : isnotnull(n_nationkey#X) + +(146) Exchange +Input [2]: [n_nationkey#X, n_name#X] +Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin +Left keys [1]: [s_nationkey#X] +Right keys [1]: [n_nationkey#X] +Join type: Inner +Join condition: None + +(149) Project +Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] +Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] + +(150) HashAggregate +Input [3]: [nation#X, o_year#X, amount#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [partial_sum(amount#X)] +Aggregate Attributes [2]: [sum#X, isEmpty#X] +Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] + +(151) Exchange +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(152) HashAggregate +Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] +Keys [2]: [nation#X, o_year#X] +Functions [1]: [sum(amount#X)] +Aggregate Attributes [1]: [sum(amount#X)#X] +Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] + +(153) Exchange +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] + +(154) Sort +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 + +(155) AdaptiveSparkPlan +Output [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala index 9fd24e6cd166..1fbb3644c473 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala @@ -92,6 +92,7 @@ abstract class VeloxTPCHSuite extends VeloxTPCHTableSupport { case "32" => true case "33" => true case "34" => true + case "35" => true case _ => false } )