From 982af124174d7c465d2a834725e52636d55896a4 Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Wed, 5 May 2021 20:57:43 -0700 Subject: [PATCH 1/6] Add codegen for left/right outer sort merge join Co-authored-by: Chen Li --- .../execution/joins/SortMergeJoinExec.scala | 155 ++++++++++++++---- .../execution/WholeStageCodegenSuite.scala | 32 ++++ .../sql/execution/joins/OuterJoinSuite.scala | 2 +- 3 files changed, 154 insertions(+), 35 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala index 74371f2005d9d..b237a50f73db0 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala @@ -354,7 +354,8 @@ case class SortMergeJoinExec( } private lazy val ((streamedPlan, streamedKeys), (bufferedPlan, bufferedKeys)) = joinType match { - case _: InnerLike => ((left, leftKeys), (right, rightKeys)) + case _: InnerLike | LeftOuter => ((left, leftKeys), (right, rightKeys)) + case RightOuter => ((right, rightKeys), (left, leftKeys)) case x => throw new IllegalArgumentException( s"SortMergeJoin.streamedPlan/bufferedPlan should not take $x as the JoinType") @@ -363,8 +364,9 @@ case class SortMergeJoinExec( private lazy val streamedOutput = streamedPlan.output private lazy val bufferedOutput = bufferedPlan.output - override def supportCodegen: Boolean = { - joinType.isInstanceOf[InnerLike] + override def supportCodegen: Boolean = joinType match { + case _: InnerLike | LeftOuter | RightOuter => true + case _ => false } override def inputRDDs(): Seq[RDD[InternalRow]] = { @@ -431,6 +433,41 @@ case class SortMergeJoinExec( // Copy the streamed keys as class members so they could be used in next function call. val matchedKeyVars = copyKeys(ctx, streamedKeyVars) + // Handle the case when streamed rows has any NULL keys. + val handleStreamedAnyNull = joinType match { + case _: InnerLike => + // Skip streamed row. + s""" + |$streamedRow = null; + |continue; + """.stripMargin + case LeftOuter | RightOuter => + // Eagerly return streamed row. Only call `matches.clear()` when `matches.isEmpty()` is + // false, to reduce unnecessary computation. + s""" + |if (!$matches.isEmpty()) { + | $matches.clear(); + |} + |return false; + """.stripMargin + case x => + throw new IllegalArgumentException( + s"SortMergeJoin.genScanner should not take $x as the JoinType") + } + + // Handle the case when streamed keys less than buffered keys. + val handleStreamedLessThanBuffered = joinType match { + case _: InnerLike => + // Skip streamed row. + s"$streamedRow = null;" + case LeftOuter | RightOuter => + // Eagerly return with streamed row. + "return false;" + case x => + throw new IllegalArgumentException( + s"SortMergeJoin.genScanner should not take $x as the JoinType") + } + ctx.addNewFunction("findNextJoinRows", s""" |private boolean findNextJoinRows( @@ -443,8 +480,7 @@ case class SortMergeJoinExec( | $streamedRow = (InternalRow) streamedIter.next(); | ${streamedKeyVars.map(_.code).mkString("\n")} | if ($streamedAnyNull) { - | $streamedRow = null; - | continue; + | $handleStreamedAnyNull | } | if (!$matches.isEmpty()) { | ${genComparison(ctx, streamedKeyVars, matchedKeyVars)} @@ -475,15 +511,18 @@ case class SortMergeJoinExec( | if (!$matches.isEmpty()) { | ${matchedKeyVars.map(_.code).mkString("\n")} | return true; + | } else { + | $handleStreamedLessThanBuffered | } - | $streamedRow = null; | } else { | $matches.add((UnsafeRow) $bufferedRow); | $bufferedRow = null; | } | } while ($streamedRow != null); | } - | return false; // unreachable + | + | throw new IllegalStateException("Executed unreachable code path in findNextJoinRows"); + | return false; |} """.stripMargin, inlineToOuterClass = true) @@ -501,7 +540,7 @@ case class SortMergeJoinExec( ctx: CodegenContext, streamedRow: String): (Seq[ExprCode], Seq[String]) = { ctx.INPUT_ROW = streamedRow - left.output.zipWithIndex.map { case (a, i) => + streamedPlan.output.zipWithIndex.map { case (a, i) => val value = ctx.freshName("value") val valueCode = CodeGenerator.getValue(streamedRow, a.dataType, i.toString) val javaType = CodeGenerator.javaType(a.dataType) @@ -569,7 +608,15 @@ case class SortMergeJoinExec( val iterator = ctx.freshName("iterator") val numOutput = metricTerm(ctx, "numOutputRows") - val resultVars = streamedVars ++ bufferedVars + val resultVars = joinType match { + case _: InnerLike | LeftOuter => + streamedVars ++ bufferedVars + case RightOuter => + bufferedVars ++ streamedVars + case x => + throw new IllegalArgumentException( + s"SortMergeJoin.doProduce should not take $x as the JoinType") + } val (beforeLoop, condCheck) = if (condition.isDefined) { // Split the code of creating variables based on whether it's used by condition or not. @@ -580,21 +627,27 @@ case class SortMergeJoinExec( ctx.currentVars = resultVars val cond = BindReferences.bindReference(condition.get, output).genCode(ctx) // evaluate the columns those used by condition before loop - val before = s""" + val before = + s""" |boolean $loaded = false; |$streamedBefore """.stripMargin - val checking = s""" - |$bufferedBefore - |${cond.code} - |if (${cond.isNull} || !${cond.value}) continue; - |if (!$loaded) { - | $loaded = true; - | $streamedAfter - |} - |$bufferedAfter - """.stripMargin + val checking = + s""" + |$bufferedBefore + |if ($bufferedRow != null) { + | ${cond.code} + | if (${cond.isNull} || !${cond.value}) { + | continue; + | } + |} + |if (!$loaded) { + | $loaded = true; + | $streamedAfter + |} + |$bufferedAfter + """.stripMargin (before, checking) } else { (evaluateVariables(streamedVars), "") @@ -603,21 +656,55 @@ case class SortMergeJoinExec( val thisPlan = ctx.addReferenceObj("plan", this) val eagerCleanup = s"$thisPlan.cleanupResources();" - s""" - |while (findNextJoinRows($streamedInput, $bufferedInput)) { - | ${streamedVarDecl.mkString("\n")} - | ${beforeLoop.trim} - | scala.collection.Iterator $iterator = $matches.generateIterator(); - | while ($iterator.hasNext()) { - | InternalRow $bufferedRow = (InternalRow) $iterator.next(); - | ${condCheck.trim} - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - | if (shouldStop()) return; - |} - |$eagerCleanup + lazy val innerJoin = + s""" + |while (findNextJoinRows($streamedInput, $bufferedInput)) { + | ${streamedVarDecl.mkString("\n")} + | ${beforeLoop.trim} + | scala.collection.Iterator $iterator = $matches.generateIterator(); + | while ($iterator.hasNext()) { + | InternalRow $bufferedRow = (InternalRow) $iterator.next(); + | ${condCheck.trim} + | $numOutput.add(1); + | ${consume(ctx, resultVars)} + | } + | if (shouldStop()) return; + |} + |$eagerCleanup """.stripMargin + + lazy val outerJoin = { + val foundMatch = ctx.freshName("foundMatch") + s""" + |while ($streamedInput.hasNext()) { + | findNextJoinRows($streamedInput, $bufferedInput); + | ${streamedVarDecl.mkString("\n")} + | ${beforeLoop.trim} + | scala.collection.Iterator $iterator = $matches.generateIterator(); + | boolean $foundMatch = false; + | + | // the last iteration of this loop is to emit an empty row if there is no matched rows. + | while ($iterator.hasNext() || !$foundMatch) { + | InternalRow $bufferedRow = $iterator.hasNext() ? + | (InternalRow) $iterator.next() : null; + | ${condCheck.trim} + | $foundMatch = true; + | $numOutput.add(1); + | ${consume(ctx, resultVars)} + | } + | if (shouldStop()) return; + |} + |$eagerCleanup + """.stripMargin + } + + joinType match { + case _: InnerLike => innerJoin + case LeftOuter | RightOuter => outerJoin + case x => + throw new IllegalArgumentException( + s"SortMergeJoin.doProduce should not take $x as the JoinType") + } } override protected def withNewChildrenInternal( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala index b66308c4f880f..9199a5e51e669 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala @@ -171,6 +171,38 @@ class WholeStageCodegenSuite extends QueryTest with SharedSparkSession Seq(Row(0, 0, 0), Row(1, 1, 1), Row(2, 2, 2), Row(3, 3, 3), Row(4, 4, 4))) } + test("Left/Right Outer SortMergeJoin should be included in WholeStageCodegen") { + val df1 = spark.range(10).select($"id".as("k1")) + val df2 = spark.range(4).select($"id".as("k2")) + val df3 = spark.range(6).select($"id".as("k3")) + + // test one left outer sort merge join + val oneLeftOuterJoinDF = df1.join(df2.hint("SHUFFLE_MERGE"), $"k1" === $"k2", "left_outer") + assert(oneLeftOuterJoinDF.queryExecution.executedPlan.collect { + case WholeStageCodegenExec(_ : SortMergeJoinExec) => true + }.size === 1) + checkAnswer(oneLeftOuterJoinDF, Seq(Row(0, 0), Row(1, 1), Row(2, 2), Row(3, 3), Row(4, null), + Row(5, null), Row(6, null), Row(7, null), Row(8, null), Row(9, null))) + + // test one right outer sort merge join + val oneRightOuterJoinDF = df2.join(df3.hint("SHUFFLE_MERGE"), $"k2" === $"k3", "right_outer") + assert(oneRightOuterJoinDF.queryExecution.executedPlan.collect { + case WholeStageCodegenExec(_ : SortMergeJoinExec) => true + }.size === 1) + checkAnswer(oneRightOuterJoinDF, Seq(Row(0, 0), Row(1, 1), Row(2, 2), Row(3, 3), Row(null, 4), + Row(null, 5))) + + // test two sort merge joins + val twoJoinsDF = df3.join(df2.hint("SHUFFLE_MERGE"), $"k3" === $"k2", "left_outer") + .join(df1.hint("SHUFFLE_MERGE"), $"k3" === $"k1", "right_outer") + assert(twoJoinsDF.queryExecution.executedPlan.collect { + case WholeStageCodegenExec(_ : SortMergeJoinExec) => true + }.size === 2) + checkAnswer(twoJoinsDF, + Seq(Row(0, 0, 0), Row(1, 1, 1), Row(2, 2, 2), Row(3, 3, 3), Row(4, null, 4), Row(5, null, 5), + Row(null, null, 6), Row(null, null, 7), Row(null, null, 8), Row(null, null, 9))) + } + test("Inner/Cross BroadcastNestedLoopJoinExec should be included in WholeStageCodegen") { val df1 = spark.range(4).select($"id".as("k1")) val df2 = spark.range(3).select($"id".as("k2")) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala index 810eeea5b9a60..744ee1ca1ad23 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala @@ -137,7 +137,7 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession { } } - test(s"$testName using SortMergeJoin") { + testWithWholeStageCodegenOnAndOff(s"$testName using SortMergeJoin") { _ => extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) => withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) => From 4a57664de66e9fb4c1c2fbc39593f6b94f98ed12 Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Sat, 8 May 2021 16:12:17 -0700 Subject: [PATCH 2/6] Fix unit test failures --- .../approved-plans-v1_4/q40/explain.txt | 4 +-- .../approved-plans-v1_4/q40/simplified.txt | 5 ++-- .../approved-plans-v1_4/q72/explain.txt | 4 +-- .../approved-plans-v1_4/q72/simplified.txt | 5 ++-- .../approved-plans-v1_4/q75/explain.txt | 24 +++++++-------- .../approved-plans-v1_4/q75/simplified.txt | 30 +++++++++++-------- .../approved-plans-v1_4/q78/explain.txt | 12 ++++---- .../approved-plans-v1_4/q78/simplified.txt | 15 ++++++---- .../approved-plans-v1_4/q80/explain.txt | 12 ++++---- .../approved-plans-v1_4/q80/simplified.txt | 15 ++++++---- .../approved-plans-v2_7/q72/explain.txt | 4 +-- .../approved-plans-v2_7/q72/simplified.txt | 5 ++-- .../approved-plans-v2_7/q75/explain.txt | 24 +++++++-------- .../approved-plans-v2_7/q75/simplified.txt | 30 +++++++++++-------- .../approved-plans-v2_7/q78/explain.txt | 12 ++++---- .../approved-plans-v2_7/q78/simplified.txt | 15 ++++++---- .../approved-plans-v2_7/q80a/explain.txt | 12 ++++---- .../approved-plans-v2_7/q80a/simplified.txt | 15 ++++++---- .../execution/metric/SQLMetricsSuite.scala | 6 ++-- 19 files changed, 138 insertions(+), 111 deletions(-) diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt index 05a6bb2928ec6..367d15e3b52f5 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt @@ -10,7 +10,7 @@ TakeOrderedAndProject (36) : :- * Project (19) : : +- * BroadcastHashJoin Inner BuildRight (18) : : :- * Project (13) - : : : +- SortMergeJoin LeftOuter (12) + : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) : : : : +- * Filter (3) @@ -86,7 +86,7 @@ Arguments: hashpartitioning(cr_order_number#9, cr_item_sk#8, 5), ENSURE_REQUIREM Input [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10] Arguments: [cr_order_number#9 ASC NULLS FIRST, cr_item_sk#8 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 8] Left keys [2]: [cs_order_number#3, cs_item_sk#2] Right keys [2]: [cr_order_number#9, cr_item_sk#8] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt index 2c15c5c38dda3..9f8f7b0c5fe45 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt @@ -12,8 +12,8 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] Project [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] Project [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (2) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -25,6 +25,7 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] Scan parquet default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #3 + InputAdapter WholeStageCodegen (4) Sort [cr_order_number,cr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt index 177b8e681608a..75a96ae5d2cdd 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt @@ -4,7 +4,7 @@ TakeOrderedAndProject (74) +- Exchange (72) +- * HashAggregate (71) +- * Project (70) - +- SortMergeJoin LeftOuter (69) + +- * SortMergeJoin LeftOuter (69) :- * Sort (62) : +- Exchange (61) : +- * Project (60) @@ -380,7 +380,7 @@ Arguments: hashpartitioning(cr_item_sk#41, cr_order_number#42, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#41, cr_order_number#42] Arguments: [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST], false, 0 -(69) SortMergeJoin +(69) SortMergeJoin [codegen id : 14] Left keys [2]: [cs_item_sk#4, cs_order_number#6] Right keys [2]: [cr_item_sk#41, cr_order_number#42] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt index ce71f65b75d1a..92800d31b82f1 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt @@ -6,8 +6,8 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom WholeStageCodegen (14) HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] Project [w_warehouse_name,i_item_desc,d_week_seq] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (11) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -103,6 +103,7 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom ColumnarToRow InputAdapter Scan parquet default.promotion [p_promo_sk] + InputAdapter WholeStageCodegen (13) Sort [cr_item_sk,cr_order_number] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index 0c5773a89e580..d2f110a58788f 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (133) : +- * HashAggregate (66) : +- Union (65) : :- * Project (26) - : : +- SortMergeJoin LeftOuter (25) + : : +- * SortMergeJoin LeftOuter (25) : : :- * Sort (18) : : : +- Exchange (17) : : : +- * Project (16) @@ -38,7 +38,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (20) : : +- Scan parquet default.catalog_returns (19) : :- * Project (45) - : : +- SortMergeJoin LeftOuter (44) + : : +- * SortMergeJoin LeftOuter (44) : : :- * Sort (37) : : : +- Exchange (36) : : : +- * Project (35) @@ -57,7 +57,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (39) : : +- Scan parquet default.store_returns (38) : +- * Project (64) - : +- SortMergeJoin LeftOuter (63) + : +- * SortMergeJoin LeftOuter (63) : :- * Sort (56) : : +- Exchange (55) : : +- * Project (54) @@ -85,7 +85,7 @@ TakeOrderedAndProject (133) +- * HashAggregate (123) +- Union (122) :- * Project (91) - : +- SortMergeJoin LeftOuter (90) + : +- * SortMergeJoin LeftOuter (90) : :- * Sort (87) : : +- Exchange (86) : : +- * Project (85) @@ -103,7 +103,7 @@ TakeOrderedAndProject (133) : +- * Sort (89) : +- ReusedExchange (88) :- * Project (106) - : +- SortMergeJoin LeftOuter (105) + : +- * SortMergeJoin LeftOuter (105) : :- * Sort (102) : : +- Exchange (101) : : +- * Project (100) @@ -118,7 +118,7 @@ TakeOrderedAndProject (133) : +- * Sort (104) : +- ReusedExchange (103) +- * Project (121) - +- SortMergeJoin LeftOuter (120) + +- * SortMergeJoin LeftOuter (120) :- * Sort (117) : +- Exchange (116) : +- * Project (115) @@ -241,7 +241,7 @@ Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21] Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0 -(25) SortMergeJoin +(25) SortMergeJoin [codegen id : 7] Left keys [2]: [cs_order_number#2, cs_item_sk#1] Right keys [2]: [cr_order_number#19, cr_item_sk#18] Join condition: None @@ -323,7 +323,7 @@ Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUI Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42] Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0 -(44) SortMergeJoin +(44) SortMergeJoin [codegen id : 14] Left keys [2]: [ss_ticket_number#27, ss_item_sk#26] Right keys [2]: [sr_ticket_number#40, sr_item_sk#39] Join condition: None @@ -405,7 +405,7 @@ Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63] Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0 -(63) SortMergeJoin +(63) SortMergeJoin [codegen id : 21] Left keys [2]: [ws_order_number#48, ws_item_sk#47] Right keys [2]: [wr_order_number#61, wr_item_sk#60] Join condition: None @@ -529,7 +529,7 @@ Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97] Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0 -(90) SortMergeJoin +(90) SortMergeJoin [codegen id : 32] Left keys [2]: [cs_order_number#80, cs_item_sk#79] Right keys [2]: [cr_order_number#95, cr_item_sk#94] Join condition: None @@ -592,7 +592,7 @@ Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_re Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114] Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(105) SortMergeJoin [codegen id : 39] Left keys [2]: [ss_ticket_number#99, ss_item_sk#98] Right keys [2]: [sr_ticket_number#112, sr_item_sk#111] Join condition: None @@ -655,7 +655,7 @@ Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_ret Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131] Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0 -(120) SortMergeJoin +(120) SortMergeJoin [codegen id : 46] Left keys [2]: [ws_order_number#116, ws_item_sk#115] Right keys [2]: [wr_order_number#129, wr_item_sk#128] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt index 168d11ffe65b4..170dad0b5dadd 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt @@ -22,8 +22,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Union WholeStageCodegen (7) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (4) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -54,6 +54,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (6) Sort [cr_order_number,cr_item_sk] InputAdapter @@ -66,8 +67,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (14) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (11) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -86,6 +87,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (13) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (21) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (18) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -118,6 +120,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (20) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -148,8 +151,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Union WholeStageCodegen (32) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (29) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -174,14 +177,15 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (31) Sort [cr_order_number,cr_item_sk] InputAdapter ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 WholeStageCodegen (39) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (36) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -200,14 +204,15 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (38) Sort [sr_ticket_number,sr_item_sk] InputAdapter ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 WholeStageCodegen (46) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (43) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -226,6 +231,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (45) Sort [wr_order_number,wr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt index 8195049644816..5ed24863c0e1d 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (73) : : +- * BroadcastHashJoin Inner BuildRight (19) : : :- * Project (14) : : : +- * Filter (13) - : : : +- SortMergeJoin LeftOuter (12) + : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) : : : : +- * Filter (3) @@ -37,7 +37,7 @@ TakeOrderedAndProject (73) : +- * BroadcastHashJoin Inner BuildRight (40) : :- * Project (38) : : +- * Filter (37) - : : +- SortMergeJoin LeftOuter (36) + : : +- * SortMergeJoin LeftOuter (36) : : :- * Sort (29) : : : +- Exchange (28) : : : +- * Filter (27) @@ -59,7 +59,7 @@ TakeOrderedAndProject (73) +- * BroadcastHashJoin Inner BuildRight (64) :- * Project (62) : +- * Filter (61) - : +- SortMergeJoin LeftOuter (60) + : +- * SortMergeJoin LeftOuter (60) : :- * Sort (53) : : +- Exchange (52) : : +- * Filter (51) @@ -123,7 +123,7 @@ Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUI Input [2]: [sr_item_sk#10, sr_ticket_number#11] Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 6] Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] Right keys [2]: [sr_ticket_number#11, sr_item_sk#10] Join condition: None @@ -234,7 +234,7 @@ Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIR Input [2]: [wr_item_sk#39, wr_order_number#40] Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0 -(36) SortMergeJoin +(36) SortMergeJoin [codegen id : 13] Left keys [2]: [ws_order_number#33, ws_item_sk#31] Right keys [2]: [wr_order_number#40, wr_item_sk#39] Join condition: None @@ -343,7 +343,7 @@ Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#68, cr_order_number#69] Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0 -(60) SortMergeJoin +(60) SortMergeJoin [codegen id : 21] Left keys [2]: [cs_order_number#62, cs_item_sk#61] Right keys [2]: [cr_order_number#69, cr_item_sk#68] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt index 39b0eb295e1ea..0aa69ab4645e4 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt @@ -18,8 +18,8 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] Filter [sr_ticket_number] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (2) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -31,6 +31,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #3 + InputAdapter WholeStageCodegen (4) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -61,8 +62,8 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] Filter [wr_order_number] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (9) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -73,6 +74,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (11) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] Filter [cr_order_number] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (17) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -110,6 +112,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (19) Sort [cr_order_number,cr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt index 8a413745875c9..5d0c3b9ddbb31 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt @@ -17,7 +17,7 @@ TakeOrderedAndProject (111) : : : :- * Project (20) : : : : +- * BroadcastHashJoin Inner BuildRight (19) : : : : :- * Project (13) - : : : : : +- SortMergeJoin LeftOuter (12) + : : : : : +- * SortMergeJoin LeftOuter (12) : : : : : :- * Sort (5) : : : : : : +- Exchange (4) : : : : : : +- * Filter (3) @@ -60,7 +60,7 @@ TakeOrderedAndProject (111) : : : :- * Project (59) : : : : +- * BroadcastHashJoin Inner BuildRight (58) : : : : :- * Project (56) - : : : : : +- SortMergeJoin LeftOuter (55) + : : : : : +- * SortMergeJoin LeftOuter (55) : : : : : :- * Sort (48) : : : : : : +- Exchange (47) : : : : : : +- * Filter (46) @@ -91,7 +91,7 @@ TakeOrderedAndProject (111) : : :- * Project (90) : : : +- * BroadcastHashJoin Inner BuildRight (89) : : : :- * Project (87) - : : : : +- SortMergeJoin LeftOuter (86) + : : : : +- * SortMergeJoin LeftOuter (86) : : : : :- * Sort (79) : : : : : +- Exchange (78) : : : : : +- * Filter (77) @@ -161,7 +161,7 @@ Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUI Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13] Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 9] Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] Right keys [2]: [sr_item_sk#10, sr_ticket_number#11] Join condition: None @@ -357,7 +357,7 @@ Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58] Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0 -(55) SortMergeJoin +(55) SortMergeJoin [codegen id : 19] Left keys [2]: [cs_item_sk#48, cs_order_number#50] Right keys [2]: [cr_item_sk#55, cr_order_number#56] Join condition: None @@ -496,7 +496,7 @@ Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97] Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0 -(86) SortMergeJoin +(86) SortMergeJoin [codegen id : 29] Left keys [2]: [ws_item_sk#86, ws_order_number#89] Right keys [2]: [wr_item_sk#94, wr_order_number#95] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt index d07a6ac61b418..5c8bd611e8dbc 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt @@ -23,8 +23,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - InputAdapter - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + InputAdapter WholeStageCodegen (2) Sort [ss_item_sk,ss_ticket_number] InputAdapter @@ -36,6 +36,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk] #4 + InputAdapter WholeStageCodegen (4) Sort [sr_item_sk,sr_ticket_number] InputAdapter @@ -92,8 +93,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (12) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -104,6 +105,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (14) Sort [cr_item_sk,cr_order_number] InputAdapter @@ -142,8 +144,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - InputAdapter - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + InputAdapter WholeStageCodegen (22) Sort [ws_item_sk,ws_order_number] InputAdapter @@ -154,6 +156,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (24) Sort [wr_item_sk,wr_order_number] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt index 0e67565e856ad..ee861169d77a4 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt @@ -4,7 +4,7 @@ TakeOrderedAndProject (74) +- Exchange (72) +- * HashAggregate (71) +- * Project (70) - +- SortMergeJoin LeftOuter (69) + +- * SortMergeJoin LeftOuter (69) :- * Sort (62) : +- Exchange (61) : +- * Project (60) @@ -380,7 +380,7 @@ Arguments: hashpartitioning(cr_item_sk#41, cr_order_number#42, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#41, cr_order_number#42] Arguments: [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST], false, 0 -(69) SortMergeJoin +(69) SortMergeJoin [codegen id : 14] Left keys [2]: [cs_item_sk#4, cs_order_number#6] Right keys [2]: [cr_item_sk#41, cr_order_number#42] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt index ce71f65b75d1a..92800d31b82f1 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt @@ -6,8 +6,8 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom WholeStageCodegen (14) HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] Project [w_warehouse_name,i_item_desc,d_week_seq] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (11) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -103,6 +103,7 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom ColumnarToRow InputAdapter Scan parquet default.promotion [p_promo_sk] + InputAdapter WholeStageCodegen (13) Sort [cr_item_sk,cr_order_number] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index 420144cfe4536..eb5287bfa8850 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (133) : +- * HashAggregate (66) : +- Union (65) : :- * Project (26) - : : +- SortMergeJoin LeftOuter (25) + : : +- * SortMergeJoin LeftOuter (25) : : :- * Sort (18) : : : +- Exchange (17) : : : +- * Project (16) @@ -38,7 +38,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (20) : : +- Scan parquet default.catalog_returns (19) : :- * Project (45) - : : +- SortMergeJoin LeftOuter (44) + : : +- * SortMergeJoin LeftOuter (44) : : :- * Sort (37) : : : +- Exchange (36) : : : +- * Project (35) @@ -57,7 +57,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (39) : : +- Scan parquet default.store_returns (38) : +- * Project (64) - : +- SortMergeJoin LeftOuter (63) + : +- * SortMergeJoin LeftOuter (63) : :- * Sort (56) : : +- Exchange (55) : : +- * Project (54) @@ -85,7 +85,7 @@ TakeOrderedAndProject (133) +- * HashAggregate (123) +- Union (122) :- * Project (91) - : +- SortMergeJoin LeftOuter (90) + : +- * SortMergeJoin LeftOuter (90) : :- * Sort (87) : : +- Exchange (86) : : +- * Project (85) @@ -103,7 +103,7 @@ TakeOrderedAndProject (133) : +- * Sort (89) : +- ReusedExchange (88) :- * Project (106) - : +- SortMergeJoin LeftOuter (105) + : +- * SortMergeJoin LeftOuter (105) : :- * Sort (102) : : +- Exchange (101) : : +- * Project (100) @@ -118,7 +118,7 @@ TakeOrderedAndProject (133) : +- * Sort (104) : +- ReusedExchange (103) +- * Project (121) - +- SortMergeJoin LeftOuter (120) + +- * SortMergeJoin LeftOuter (120) :- * Sort (117) : +- Exchange (116) : +- * Project (115) @@ -241,7 +241,7 @@ Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21] Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0 -(25) SortMergeJoin +(25) SortMergeJoin [codegen id : 7] Left keys [2]: [cs_order_number#2, cs_item_sk#1] Right keys [2]: [cr_order_number#19, cr_item_sk#18] Join condition: None @@ -323,7 +323,7 @@ Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUI Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42] Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0 -(44) SortMergeJoin +(44) SortMergeJoin [codegen id : 14] Left keys [2]: [ss_ticket_number#27, ss_item_sk#26] Right keys [2]: [sr_ticket_number#40, sr_item_sk#39] Join condition: None @@ -405,7 +405,7 @@ Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63] Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0 -(63) SortMergeJoin +(63) SortMergeJoin [codegen id : 21] Left keys [2]: [ws_order_number#48, ws_item_sk#47] Right keys [2]: [wr_order_number#61, wr_item_sk#60] Join condition: None @@ -529,7 +529,7 @@ Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97] Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0 -(90) SortMergeJoin +(90) SortMergeJoin [codegen id : 32] Left keys [2]: [cs_order_number#80, cs_item_sk#79] Right keys [2]: [cr_order_number#95, cr_item_sk#94] Join condition: None @@ -592,7 +592,7 @@ Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_re Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114] Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(105) SortMergeJoin [codegen id : 39] Left keys [2]: [ss_ticket_number#99, ss_item_sk#98] Right keys [2]: [sr_ticket_number#112, sr_item_sk#111] Join condition: None @@ -655,7 +655,7 @@ Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_ret Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131] Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0 -(120) SortMergeJoin +(120) SortMergeJoin [codegen id : 46] Left keys [2]: [ws_order_number#116, ws_item_sk#115] Right keys [2]: [wr_order_number#129, wr_item_sk#128] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt index e2c2d625eec59..1e5555b76b4e5 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt @@ -22,8 +22,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Union WholeStageCodegen (7) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (4) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -54,6 +54,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (6) Sort [cr_order_number,cr_item_sk] InputAdapter @@ -66,8 +67,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (14) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (11) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -86,6 +87,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (13) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (21) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (18) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -118,6 +120,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (20) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -148,8 +151,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Union WholeStageCodegen (32) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (29) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -174,14 +177,15 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (31) Sort [cr_order_number,cr_item_sk] InputAdapter ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 WholeStageCodegen (39) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (36) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -200,14 +204,15 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (38) Sort [sr_ticket_number,sr_item_sk] InputAdapter ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 WholeStageCodegen (46) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (43) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -226,6 +231,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (45) Sort [wr_order_number,wr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt index 68cf417e81905..5694b16800d8e 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (73) : : +- * BroadcastHashJoin Inner BuildRight (19) : : :- * Project (14) : : : +- * Filter (13) - : : : +- SortMergeJoin LeftOuter (12) + : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) : : : : +- * Filter (3) @@ -37,7 +37,7 @@ TakeOrderedAndProject (73) : +- * BroadcastHashJoin Inner BuildRight (40) : :- * Project (38) : : +- * Filter (37) - : : +- SortMergeJoin LeftOuter (36) + : : +- * SortMergeJoin LeftOuter (36) : : :- * Sort (29) : : : +- Exchange (28) : : : +- * Filter (27) @@ -59,7 +59,7 @@ TakeOrderedAndProject (73) +- * BroadcastHashJoin Inner BuildRight (64) :- * Project (62) : +- * Filter (61) - : +- SortMergeJoin LeftOuter (60) + : +- * SortMergeJoin LeftOuter (60) : :- * Sort (53) : : +- Exchange (52) : : +- * Filter (51) @@ -123,7 +123,7 @@ Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUI Input [2]: [sr_item_sk#10, sr_ticket_number#11] Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 6] Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] Right keys [2]: [sr_ticket_number#11, sr_item_sk#10] Join condition: None @@ -234,7 +234,7 @@ Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIR Input [2]: [wr_item_sk#39, wr_order_number#40] Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0 -(36) SortMergeJoin +(36) SortMergeJoin [codegen id : 13] Left keys [2]: [ws_order_number#33, ws_item_sk#31] Right keys [2]: [wr_order_number#40, wr_item_sk#39] Join condition: None @@ -343,7 +343,7 @@ Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#68, cr_order_number#69] Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0 -(60) SortMergeJoin +(60) SortMergeJoin [codegen id : 21] Left keys [2]: [cs_order_number#62, cs_item_sk#61] Right keys [2]: [cr_order_number#69, cr_item_sk#68] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt index 17421ac128698..cb9ca4f3d3d32 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt @@ -18,8 +18,8 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] Filter [sr_ticket_number] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (2) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -31,6 +31,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #3 + InputAdapter WholeStageCodegen (4) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -61,8 +62,8 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] Filter [wr_order_number] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (9) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -73,6 +74,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (11) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] Filter [cr_order_number] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (17) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -110,6 +112,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (19) Sort [cr_order_number,cr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt index a9efbf56ef743..845d05e05b9bf 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject (124) : : : : :- * Project (20) : : : : : +- * BroadcastHashJoin Inner BuildRight (19) : : : : : :- * Project (13) - : : : : : : +- SortMergeJoin LeftOuter (12) + : : : : : : +- * SortMergeJoin LeftOuter (12) : : : : : : :- * Sort (5) : : : : : : : +- Exchange (4) : : : : : : : +- * Filter (3) @@ -63,7 +63,7 @@ TakeOrderedAndProject (124) : : : : :- * Project (59) : : : : : +- * BroadcastHashJoin Inner BuildRight (58) : : : : : :- * Project (56) - : : : : : : +- SortMergeJoin LeftOuter (55) + : : : : : : +- * SortMergeJoin LeftOuter (55) : : : : : : :- * Sort (48) : : : : : : : +- Exchange (47) : : : : : : : +- * Filter (46) @@ -94,7 +94,7 @@ TakeOrderedAndProject (124) : : : :- * Project (90) : : : : +- * BroadcastHashJoin Inner BuildRight (89) : : : : :- * Project (87) - : : : : : +- SortMergeJoin LeftOuter (86) + : : : : : +- * SortMergeJoin LeftOuter (86) : : : : : :- * Sort (79) : : : : : : +- Exchange (78) : : : : : : +- * Filter (77) @@ -174,7 +174,7 @@ Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUI Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13] Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 9] Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] Right keys [2]: [sr_item_sk#10, sr_ticket_number#11] Join condition: None @@ -370,7 +370,7 @@ Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58] Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0 -(55) SortMergeJoin +(55) SortMergeJoin [codegen id : 19] Left keys [2]: [cs_item_sk#48, cs_order_number#50] Right keys [2]: [cr_item_sk#55, cr_order_number#56] Join condition: None @@ -509,7 +509,7 @@ Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97] Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0 -(86) SortMergeJoin +(86) SortMergeJoin [codegen id : 29] Left keys [2]: [ws_item_sk#86, ws_order_number#89] Right keys [2]: [wr_item_sk#94, wr_order_number#95] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt index ea5cd01ab8889..30b609e21c467 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt @@ -30,8 +30,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - InputAdapter - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + InputAdapter WholeStageCodegen (2) Sort [ss_item_sk,ss_ticket_number] InputAdapter @@ -43,6 +43,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk] #5 + InputAdapter WholeStageCodegen (4) Sort [sr_item_sk,sr_ticket_number] InputAdapter @@ -99,8 +100,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (12) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -111,6 +112,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (14) Sort [cr_item_sk,cr_order_number] InputAdapter @@ -149,8 +151,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - InputAdapter - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + InputAdapter WholeStageCodegen (22) Sort [ws_item_sk,ws_order_number] InputAdapter @@ -161,6 +163,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (24) Sort [wr_item_sk,wr_order_number] InputAdapter diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala index bbf58c61a1e12..1e7032ab07425 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala @@ -309,11 +309,11 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils val rightJoinQuery = "SELECT * FROM testDataForJoin right JOIN testData2 ON " + "testData2.a = testDataForJoin.a" - Seq((leftJoinQuery, false), (leftJoinQuery, true), (rightJoinQuery, false), - (rightJoinQuery, true)).foreach { case (query, enableWholeStage) => + Seq((leftJoinQuery, 0L, false), (leftJoinQuery, 1L, true), (rightJoinQuery, 0L, false), + (rightJoinQuery, 1L, true)).foreach { case (query, nodeId, enableWholeStage) => val df = spark.sql(query) testSparkPlanMetrics(df, 1, Map( - 0L -> (("SortMergeJoin", Map( + nodeId -> (("SortMergeJoin", Map( // It's 8 because we read 6 rows in the left and 2 row in the right one "number of output rows" -> 8L)))), enableWholeStage From 44b210fff8dd04e0ed76924fc9bca4b3907af41c Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Sat, 8 May 2021 20:24:36 -0700 Subject: [PATCH 3/6] Fix more unit test failure --- .../execution/joins/SortMergeJoinExec.scala | 4 +-- .../approved-plans-v1_4/q40.sf100/explain.txt | 4 +-- .../q40.sf100/simplified.txt | 5 ++-- .../approved-plans-v1_4/q72.sf100/explain.txt | 4 +-- .../q72.sf100/simplified.txt | 5 ++-- .../approved-plans-v1_4/q75.sf100/explain.txt | 24 +++++++-------- .../q75.sf100/simplified.txt | 30 +++++++++++-------- .../approved-plans-v1_4/q78.sf100/explain.txt | 12 ++++---- .../q78.sf100/simplified.txt | 15 ++++++---- .../approved-plans-v1_4/q80.sf100/explain.txt | 12 ++++---- .../q80.sf100/simplified.txt | 15 ++++++---- .../approved-plans-v2_7/q72.sf100/explain.txt | 4 +-- .../q72.sf100/simplified.txt | 5 ++-- .../approved-plans-v2_7/q75.sf100/explain.txt | 24 +++++++-------- .../q75.sf100/simplified.txt | 30 +++++++++++-------- .../approved-plans-v2_7/q78.sf100/explain.txt | 12 ++++---- .../q78.sf100/simplified.txt | 15 ++++++---- .../q80a.sf100/explain.txt | 12 ++++---- .../q80a.sf100/simplified.txt | 15 ++++++---- 19 files changed, 136 insertions(+), 111 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala index b237a50f73db0..3a8f262e388e8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala @@ -520,9 +520,7 @@ case class SortMergeJoinExec( | } | } while ($streamedRow != null); | } - | - | throw new IllegalStateException("Executed unreachable code path in findNextJoinRows"); - | return false; + | return false; // unreachable |} """.stripMargin, inlineToOuterClass = true) diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt index c53874d2288e5..fcd1b82886242 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt @@ -10,7 +10,7 @@ TakeOrderedAndProject (36) : :- * Project (20) : : +- * BroadcastHashJoin Inner BuildRight (19) : : :- * Project (13) - : : : +- SortMergeJoin LeftOuter (12) + : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) : : : : +- * Filter (3) @@ -86,7 +86,7 @@ Arguments: hashpartitioning(cr_order_number#9, cr_item_sk#8, 5), ENSURE_REQUIREM Input [3]: [cr_item_sk#8, cr_order_number#9, cr_refunded_cash#10] Arguments: [cr_order_number#9 ASC NULLS FIRST, cr_item_sk#8 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 8] Left keys [2]: [cs_order_number#3, cs_item_sk#2] Right keys [2]: [cr_order_number#9, cr_item_sk#8] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt index 20e94c9baba71..53eafedcaa812 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt @@ -12,8 +12,8 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] Project [cs_warehouse_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,i_item_id] BroadcastHashJoin [cs_item_sk,i_item_sk] Project [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (2) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -25,6 +25,7 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] Scan parquet default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #3 + InputAdapter WholeStageCodegen (4) Sort [cr_order_number,cr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt index 95092a8bbcc51..ac66894467451 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt @@ -4,7 +4,7 @@ TakeOrderedAndProject (80) +- Exchange (78) +- * HashAggregate (77) +- * Project (76) - +- SortMergeJoin LeftOuter (75) + +- * SortMergeJoin LeftOuter (75) :- * Sort (68) : +- Exchange (67) : +- * Project (66) @@ -410,7 +410,7 @@ Arguments: hashpartitioning(cr_item_sk#43, cr_order_number#44, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#43, cr_order_number#44] Arguments: [cr_item_sk#43 ASC NULLS FIRST, cr_order_number#44 ASC NULLS FIRST], false, 0 -(75) SortMergeJoin +(75) SortMergeJoin [codegen id : 20] Left keys [2]: [cs_item_sk#4, cs_order_number#6] Right keys [2]: [cr_item_sk#43, cr_order_number#44] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt index c98d0fd9e1fc5..da6b79ca3c79d 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt @@ -6,8 +6,8 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom WholeStageCodegen (20) HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] Project [w_warehouse_name,i_item_desc,d_week_seq] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (17) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -121,6 +121,7 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom ColumnarToRow InputAdapter Scan parquet default.promotion [p_promo_sk] + InputAdapter WholeStageCodegen (19) Sort [cr_item_sk,cr_order_number] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt index 0c5773a89e580..d2f110a58788f 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (133) : +- * HashAggregate (66) : +- Union (65) : :- * Project (26) - : : +- SortMergeJoin LeftOuter (25) + : : +- * SortMergeJoin LeftOuter (25) : : :- * Sort (18) : : : +- Exchange (17) : : : +- * Project (16) @@ -38,7 +38,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (20) : : +- Scan parquet default.catalog_returns (19) : :- * Project (45) - : : +- SortMergeJoin LeftOuter (44) + : : +- * SortMergeJoin LeftOuter (44) : : :- * Sort (37) : : : +- Exchange (36) : : : +- * Project (35) @@ -57,7 +57,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (39) : : +- Scan parquet default.store_returns (38) : +- * Project (64) - : +- SortMergeJoin LeftOuter (63) + : +- * SortMergeJoin LeftOuter (63) : :- * Sort (56) : : +- Exchange (55) : : +- * Project (54) @@ -85,7 +85,7 @@ TakeOrderedAndProject (133) +- * HashAggregate (123) +- Union (122) :- * Project (91) - : +- SortMergeJoin LeftOuter (90) + : +- * SortMergeJoin LeftOuter (90) : :- * Sort (87) : : +- Exchange (86) : : +- * Project (85) @@ -103,7 +103,7 @@ TakeOrderedAndProject (133) : +- * Sort (89) : +- ReusedExchange (88) :- * Project (106) - : +- SortMergeJoin LeftOuter (105) + : +- * SortMergeJoin LeftOuter (105) : :- * Sort (102) : : +- Exchange (101) : : +- * Project (100) @@ -118,7 +118,7 @@ TakeOrderedAndProject (133) : +- * Sort (104) : +- ReusedExchange (103) +- * Project (121) - +- SortMergeJoin LeftOuter (120) + +- * SortMergeJoin LeftOuter (120) :- * Sort (117) : +- Exchange (116) : +- * Project (115) @@ -241,7 +241,7 @@ Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21] Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0 -(25) SortMergeJoin +(25) SortMergeJoin [codegen id : 7] Left keys [2]: [cs_order_number#2, cs_item_sk#1] Right keys [2]: [cr_order_number#19, cr_item_sk#18] Join condition: None @@ -323,7 +323,7 @@ Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUI Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42] Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0 -(44) SortMergeJoin +(44) SortMergeJoin [codegen id : 14] Left keys [2]: [ss_ticket_number#27, ss_item_sk#26] Right keys [2]: [sr_ticket_number#40, sr_item_sk#39] Join condition: None @@ -405,7 +405,7 @@ Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63] Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0 -(63) SortMergeJoin +(63) SortMergeJoin [codegen id : 21] Left keys [2]: [ws_order_number#48, ws_item_sk#47] Right keys [2]: [wr_order_number#61, wr_item_sk#60] Join condition: None @@ -529,7 +529,7 @@ Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97] Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0 -(90) SortMergeJoin +(90) SortMergeJoin [codegen id : 32] Left keys [2]: [cs_order_number#80, cs_item_sk#79] Right keys [2]: [cr_order_number#95, cr_item_sk#94] Join condition: None @@ -592,7 +592,7 @@ Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_re Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114] Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(105) SortMergeJoin [codegen id : 39] Left keys [2]: [ss_ticket_number#99, ss_item_sk#98] Right keys [2]: [sr_ticket_number#112, sr_item_sk#111] Join condition: None @@ -655,7 +655,7 @@ Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_ret Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131] Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0 -(120) SortMergeJoin +(120) SortMergeJoin [codegen id : 46] Left keys [2]: [ws_order_number#116, ws_item_sk#115] Right keys [2]: [wr_order_number#129, wr_item_sk#128] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt index 168d11ffe65b4..170dad0b5dadd 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt @@ -22,8 +22,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Union WholeStageCodegen (7) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (4) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -54,6 +54,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (6) Sort [cr_order_number,cr_item_sk] InputAdapter @@ -66,8 +67,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (14) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (11) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -86,6 +87,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (13) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (21) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (18) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -118,6 +120,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (20) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -148,8 +151,8 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat Union WholeStageCodegen (32) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (29) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -174,14 +177,15 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (31) Sort [cr_order_number,cr_item_sk] InputAdapter ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 WholeStageCodegen (39) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (36) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -200,14 +204,15 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (38) Sort [sr_ticket_number,sr_item_sk] InputAdapter ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 WholeStageCodegen (46) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (43) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -226,6 +231,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (45) Sort [wr_order_number,wr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt index 8195049644816..5ed24863c0e1d 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (73) : : +- * BroadcastHashJoin Inner BuildRight (19) : : :- * Project (14) : : : +- * Filter (13) - : : : +- SortMergeJoin LeftOuter (12) + : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) : : : : +- * Filter (3) @@ -37,7 +37,7 @@ TakeOrderedAndProject (73) : +- * BroadcastHashJoin Inner BuildRight (40) : :- * Project (38) : : +- * Filter (37) - : : +- SortMergeJoin LeftOuter (36) + : : +- * SortMergeJoin LeftOuter (36) : : :- * Sort (29) : : : +- Exchange (28) : : : +- * Filter (27) @@ -59,7 +59,7 @@ TakeOrderedAndProject (73) +- * BroadcastHashJoin Inner BuildRight (64) :- * Project (62) : +- * Filter (61) - : +- SortMergeJoin LeftOuter (60) + : +- * SortMergeJoin LeftOuter (60) : :- * Sort (53) : : +- Exchange (52) : : +- * Filter (51) @@ -123,7 +123,7 @@ Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUI Input [2]: [sr_item_sk#10, sr_ticket_number#11] Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 6] Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] Right keys [2]: [sr_ticket_number#11, sr_item_sk#10] Join condition: None @@ -234,7 +234,7 @@ Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIR Input [2]: [wr_item_sk#39, wr_order_number#40] Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0 -(36) SortMergeJoin +(36) SortMergeJoin [codegen id : 13] Left keys [2]: [ws_order_number#33, ws_item_sk#31] Right keys [2]: [wr_order_number#40, wr_item_sk#39] Join condition: None @@ -343,7 +343,7 @@ Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#68, cr_order_number#69] Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0 -(60) SortMergeJoin +(60) SortMergeJoin [codegen id : 21] Left keys [2]: [cs_order_number#62, cs_item_sk#61] Right keys [2]: [cr_order_number#69, cr_item_sk#68] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt index 39b0eb295e1ea..0aa69ab4645e4 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt @@ -18,8 +18,8 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] Filter [sr_ticket_number] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (2) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -31,6 +31,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #3 + InputAdapter WholeStageCodegen (4) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -61,8 +62,8 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] Filter [wr_order_number] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (9) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -73,6 +74,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (11) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] Filter [cr_order_number] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (17) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -110,6 +112,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes InputAdapter Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (19) Sort [cr_order_number,cr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt index 2c625a8f4fd41..d9794c33a767d 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt @@ -17,7 +17,7 @@ TakeOrderedAndProject (111) : : : :- * Project (20) : : : : +- * BroadcastHashJoin Inner BuildRight (19) : : : : :- * Project (13) - : : : : : +- SortMergeJoin LeftOuter (12) + : : : : : +- * SortMergeJoin LeftOuter (12) : : : : : :- * Sort (5) : : : : : : +- Exchange (4) : : : : : : +- * Filter (3) @@ -60,7 +60,7 @@ TakeOrderedAndProject (111) : : : :- * Project (59) : : : : +- * BroadcastHashJoin Inner BuildRight (58) : : : : :- * Project (56) - : : : : : +- SortMergeJoin LeftOuter (55) + : : : : : +- * SortMergeJoin LeftOuter (55) : : : : : :- * Sort (48) : : : : : : +- Exchange (47) : : : : : : +- * Filter (46) @@ -91,7 +91,7 @@ TakeOrderedAndProject (111) : : :- * Project (90) : : : +- * BroadcastHashJoin Inner BuildRight (89) : : : :- * Project (87) - : : : : +- SortMergeJoin LeftOuter (86) + : : : : +- * SortMergeJoin LeftOuter (86) : : : : :- * Sort (79) : : : : : +- Exchange (78) : : : : : +- * Filter (77) @@ -161,7 +161,7 @@ Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUI Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13] Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 9] Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] Right keys [2]: [sr_item_sk#10, sr_ticket_number#11] Join condition: None @@ -357,7 +357,7 @@ Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58] Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0 -(55) SortMergeJoin +(55) SortMergeJoin [codegen id : 19] Left keys [2]: [cs_item_sk#48, cs_order_number#50] Right keys [2]: [cr_item_sk#55, cr_order_number#56] Join condition: None @@ -496,7 +496,7 @@ Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97] Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0 -(86) SortMergeJoin +(86) SortMergeJoin [codegen id : 29] Left keys [2]: [ws_item_sk#86, ws_order_number#89] Right keys [2]: [wr_item_sk#94, wr_order_number#95] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt index ccba19ac63c1a..4155b5176397e 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt @@ -23,8 +23,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - InputAdapter - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + InputAdapter WholeStageCodegen (2) Sort [ss_item_sk,ss_ticket_number] InputAdapter @@ -36,6 +36,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk] #4 + InputAdapter WholeStageCodegen (4) Sort [sr_item_sk,sr_ticket_number] InputAdapter @@ -92,8 +93,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [cs_catalog_page_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] BroadcastHashJoin [cs_item_sk,i_item_sk] Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (12) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -104,6 +105,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (14) Sort [cr_item_sk,cr_order_number] InputAdapter @@ -142,8 +144,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] BroadcastHashJoin [ws_item_sk,i_item_sk] Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - InputAdapter - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + InputAdapter WholeStageCodegen (22) Sort [ws_item_sk,ws_order_number] InputAdapter @@ -154,6 +156,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (24) Sort [wr_item_sk,wr_order_number] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt index 949f9c51b3d22..c22dec073af41 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt @@ -4,7 +4,7 @@ TakeOrderedAndProject (80) +- Exchange (78) +- * HashAggregate (77) +- * Project (76) - +- SortMergeJoin LeftOuter (75) + +- * SortMergeJoin LeftOuter (75) :- * Sort (68) : +- Exchange (67) : +- * Project (66) @@ -410,7 +410,7 @@ Arguments: hashpartitioning(cr_item_sk#43, cr_order_number#44, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#43, cr_order_number#44] Arguments: [cr_item_sk#43 ASC NULLS FIRST, cr_order_number#44 ASC NULLS FIRST], false, 0 -(75) SortMergeJoin +(75) SortMergeJoin [codegen id : 20] Left keys [2]: [cs_item_sk#4, cs_order_number#6] Right keys [2]: [cr_item_sk#43, cr_order_number#44] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt index c98d0fd9e1fc5..da6b79ca3c79d 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt @@ -6,8 +6,8 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom WholeStageCodegen (20) HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] Project [w_warehouse_name,i_item_desc,d_week_seq] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (17) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -121,6 +121,7 @@ TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_prom ColumnarToRow InputAdapter Scan parquet default.promotion [p_promo_sk] + InputAdapter WholeStageCodegen (19) Sort [cr_item_sk,cr_order_number] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt index 420144cfe4536..eb5287bfa8850 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (133) : +- * HashAggregate (66) : +- Union (65) : :- * Project (26) - : : +- SortMergeJoin LeftOuter (25) + : : +- * SortMergeJoin LeftOuter (25) : : :- * Sort (18) : : : +- Exchange (17) : : : +- * Project (16) @@ -38,7 +38,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (20) : : +- Scan parquet default.catalog_returns (19) : :- * Project (45) - : : +- SortMergeJoin LeftOuter (44) + : : +- * SortMergeJoin LeftOuter (44) : : :- * Sort (37) : : : +- Exchange (36) : : : +- * Project (35) @@ -57,7 +57,7 @@ TakeOrderedAndProject (133) : : +- * ColumnarToRow (39) : : +- Scan parquet default.store_returns (38) : +- * Project (64) - : +- SortMergeJoin LeftOuter (63) + : +- * SortMergeJoin LeftOuter (63) : :- * Sort (56) : : +- Exchange (55) : : +- * Project (54) @@ -85,7 +85,7 @@ TakeOrderedAndProject (133) +- * HashAggregate (123) +- Union (122) :- * Project (91) - : +- SortMergeJoin LeftOuter (90) + : +- * SortMergeJoin LeftOuter (90) : :- * Sort (87) : : +- Exchange (86) : : +- * Project (85) @@ -103,7 +103,7 @@ TakeOrderedAndProject (133) : +- * Sort (89) : +- ReusedExchange (88) :- * Project (106) - : +- SortMergeJoin LeftOuter (105) + : +- * SortMergeJoin LeftOuter (105) : :- * Sort (102) : : +- Exchange (101) : : +- * Project (100) @@ -118,7 +118,7 @@ TakeOrderedAndProject (133) : +- * Sort (104) : +- ReusedExchange (103) +- * Project (121) - +- SortMergeJoin LeftOuter (120) + +- * SortMergeJoin LeftOuter (120) :- * Sort (117) : +- Exchange (116) : +- * Project (115) @@ -241,7 +241,7 @@ Arguments: hashpartitioning(cr_order_number#19, cr_item_sk#18, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#18, cr_order_number#19, cr_return_quantity#20, cr_return_amount#21] Arguments: [cr_order_number#19 ASC NULLS FIRST, cr_item_sk#18 ASC NULLS FIRST], false, 0 -(25) SortMergeJoin +(25) SortMergeJoin [codegen id : 7] Left keys [2]: [cs_order_number#2, cs_item_sk#1] Right keys [2]: [cr_order_number#19, cr_item_sk#18] Join condition: None @@ -323,7 +323,7 @@ Arguments: hashpartitioning(sr_ticket_number#40, sr_item_sk#39, 5), ENSURE_REQUI Input [4]: [sr_item_sk#39, sr_ticket_number#40, sr_return_quantity#41, sr_return_amt#42] Arguments: [sr_ticket_number#40 ASC NULLS FIRST, sr_item_sk#39 ASC NULLS FIRST], false, 0 -(44) SortMergeJoin +(44) SortMergeJoin [codegen id : 14] Left keys [2]: [ss_ticket_number#27, ss_item_sk#26] Right keys [2]: [sr_ticket_number#40, sr_item_sk#39] Join condition: None @@ -405,7 +405,7 @@ Arguments: hashpartitioning(wr_order_number#61, wr_item_sk#60, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#60, wr_order_number#61, wr_return_quantity#62, wr_return_amt#63] Arguments: [wr_order_number#61 ASC NULLS FIRST, wr_item_sk#60 ASC NULLS FIRST], false, 0 -(63) SortMergeJoin +(63) SortMergeJoin [codegen id : 21] Left keys [2]: [ws_order_number#48, ws_item_sk#47] Right keys [2]: [wr_order_number#61, wr_item_sk#60] Join condition: None @@ -529,7 +529,7 @@ Output [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return Input [4]: [cr_item_sk#94, cr_order_number#95, cr_return_quantity#96, cr_return_amount#97] Arguments: [cr_order_number#95 ASC NULLS FIRST, cr_item_sk#94 ASC NULLS FIRST], false, 0 -(90) SortMergeJoin +(90) SortMergeJoin [codegen id : 32] Left keys [2]: [cs_order_number#80, cs_item_sk#79] Right keys [2]: [cr_order_number#95, cr_item_sk#94] Join condition: None @@ -592,7 +592,7 @@ Output [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_re Input [4]: [sr_item_sk#111, sr_ticket_number#112, sr_return_quantity#113, sr_return_amt#114] Arguments: [sr_ticket_number#112 ASC NULLS FIRST, sr_item_sk#111 ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(105) SortMergeJoin [codegen id : 39] Left keys [2]: [ss_ticket_number#99, ss_item_sk#98] Right keys [2]: [sr_ticket_number#112, sr_item_sk#111] Join condition: None @@ -655,7 +655,7 @@ Output [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_ret Input [4]: [wr_item_sk#128, wr_order_number#129, wr_return_quantity#130, wr_return_amt#131] Arguments: [wr_order_number#129 ASC NULLS FIRST, wr_item_sk#128 ASC NULLS FIRST], false, 0 -(120) SortMergeJoin +(120) SortMergeJoin [codegen id : 46] Left keys [2]: [ws_order_number#116, ws_item_sk#115] Right keys [2]: [wr_order_number#129, wr_item_sk#128] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt index e2c2d625eec59..1e5555b76b4e5 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt @@ -22,8 +22,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Union WholeStageCodegen (7) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (4) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -54,6 +54,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (6) Sort [cr_order_number,cr_item_sk] InputAdapter @@ -66,8 +67,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Scan parquet default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (14) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (11) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -86,6 +87,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (13) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Scan parquet default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (21) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (18) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -118,6 +120,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #5 + InputAdapter WholeStageCodegen (20) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -148,8 +151,8 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i Union WholeStageCodegen (32) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (29) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -174,14 +177,15 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter Scan parquet default.date_dim [d_date_sk,d_year] + InputAdapter WholeStageCodegen (31) Sort [cr_order_number,cr_item_sk] InputAdapter ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 WholeStageCodegen (39) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (36) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -200,14 +204,15 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (38) Sort [sr_ticket_number,sr_item_sk] InputAdapter ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 WholeStageCodegen (46) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (43) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -226,6 +231,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #16 + InputAdapter WholeStageCodegen (45) Sort [wr_order_number,wr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt index 68cf417e81905..5694b16800d8e 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt @@ -12,7 +12,7 @@ TakeOrderedAndProject (73) : : +- * BroadcastHashJoin Inner BuildRight (19) : : :- * Project (14) : : : +- * Filter (13) - : : : +- SortMergeJoin LeftOuter (12) + : : : +- * SortMergeJoin LeftOuter (12) : : : :- * Sort (5) : : : : +- Exchange (4) : : : : +- * Filter (3) @@ -37,7 +37,7 @@ TakeOrderedAndProject (73) : +- * BroadcastHashJoin Inner BuildRight (40) : :- * Project (38) : : +- * Filter (37) - : : +- SortMergeJoin LeftOuter (36) + : : +- * SortMergeJoin LeftOuter (36) : : :- * Sort (29) : : : +- Exchange (28) : : : +- * Filter (27) @@ -59,7 +59,7 @@ TakeOrderedAndProject (73) +- * BroadcastHashJoin Inner BuildRight (64) :- * Project (62) : +- * Filter (61) - : +- SortMergeJoin LeftOuter (60) + : +- * SortMergeJoin LeftOuter (60) : :- * Sort (53) : : +- Exchange (52) : : +- * Filter (51) @@ -123,7 +123,7 @@ Arguments: hashpartitioning(sr_ticket_number#11, sr_item_sk#10, 5), ENSURE_REQUI Input [2]: [sr_item_sk#10, sr_ticket_number#11] Arguments: [sr_ticket_number#11 ASC NULLS FIRST, sr_item_sk#10 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 6] Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] Right keys [2]: [sr_ticket_number#11, sr_item_sk#10] Join condition: None @@ -234,7 +234,7 @@ Arguments: hashpartitioning(wr_order_number#40, wr_item_sk#39, 5), ENSURE_REQUIR Input [2]: [wr_item_sk#39, wr_order_number#40] Arguments: [wr_order_number#40 ASC NULLS FIRST, wr_item_sk#39 ASC NULLS FIRST], false, 0 -(36) SortMergeJoin +(36) SortMergeJoin [codegen id : 13] Left keys [2]: [ws_order_number#33, ws_item_sk#31] Right keys [2]: [wr_order_number#40, wr_item_sk#39] Join condition: None @@ -343,7 +343,7 @@ Arguments: hashpartitioning(cr_order_number#69, cr_item_sk#68, 5), ENSURE_REQUIR Input [2]: [cr_item_sk#68, cr_order_number#69] Arguments: [cr_order_number#69 ASC NULLS FIRST, cr_item_sk#68 ASC NULLS FIRST], false, 0 -(60) SortMergeJoin +(60) SortMergeJoin [codegen id : 21] Left keys [2]: [cs_order_number#62, cs_item_sk#61] Right keys [2]: [cr_order_number#69, cr_item_sk#68] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt index 17421ac128698..cb9ca4f3d3d32 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt @@ -18,8 +18,8 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp BroadcastHashJoin [ss_sold_date_sk,d_date_sk] Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] Filter [sr_ticket_number] - InputAdapter - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] + InputAdapter WholeStageCodegen (2) Sort [ss_ticket_number,ss_item_sk] InputAdapter @@ -31,6 +31,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp Scan parquet default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #3 + InputAdapter WholeStageCodegen (4) Sort [sr_ticket_number,sr_item_sk] InputAdapter @@ -61,8 +62,8 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp BroadcastHashJoin [ws_sold_date_sk,d_date_sk] Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] Filter [wr_order_number] - InputAdapter - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] + InputAdapter WholeStageCodegen (9) Sort [ws_order_number,ws_item_sk] InputAdapter @@ -73,6 +74,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (11) Sort [wr_order_number,wr_item_sk] InputAdapter @@ -98,8 +100,8 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp BroadcastHashJoin [cs_sold_date_sk,d_date_sk] Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] Filter [cr_order_number] - InputAdapter - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] + InputAdapter WholeStageCodegen (17) Sort [cs_order_number,cs_item_sk] InputAdapter @@ -110,6 +112,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp InputAdapter Scan parquet default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (19) Sort [cr_order_number,cr_item_sk] InputAdapter diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt index cd8f5d5c6ae61..61cae9d5bf5be 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject (124) : : : : :- * Project (20) : : : : : +- * BroadcastHashJoin Inner BuildRight (19) : : : : : :- * Project (13) - : : : : : : +- SortMergeJoin LeftOuter (12) + : : : : : : +- * SortMergeJoin LeftOuter (12) : : : : : : :- * Sort (5) : : : : : : : +- Exchange (4) : : : : : : : +- * Filter (3) @@ -63,7 +63,7 @@ TakeOrderedAndProject (124) : : : : :- * Project (59) : : : : : +- * BroadcastHashJoin Inner BuildRight (58) : : : : : :- * Project (56) - : : : : : : +- SortMergeJoin LeftOuter (55) + : : : : : : +- * SortMergeJoin LeftOuter (55) : : : : : : :- * Sort (48) : : : : : : : +- Exchange (47) : : : : : : : +- * Filter (46) @@ -94,7 +94,7 @@ TakeOrderedAndProject (124) : : : :- * Project (90) : : : : +- * BroadcastHashJoin Inner BuildRight (89) : : : : :- * Project (87) - : : : : : +- SortMergeJoin LeftOuter (86) + : : : : : +- * SortMergeJoin LeftOuter (86) : : : : : :- * Sort (79) : : : : : : +- Exchange (78) : : : : : : +- * Filter (77) @@ -174,7 +174,7 @@ Arguments: hashpartitioning(sr_item_sk#10, sr_ticket_number#11, 5), ENSURE_REQUI Input [4]: [sr_item_sk#10, sr_ticket_number#11, sr_return_amt#12, sr_net_loss#13] Arguments: [sr_item_sk#10 ASC NULLS FIRST, sr_ticket_number#11 ASC NULLS FIRST], false, 0 -(12) SortMergeJoin +(12) SortMergeJoin [codegen id : 9] Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] Right keys [2]: [sr_item_sk#10, sr_ticket_number#11] Join condition: None @@ -370,7 +370,7 @@ Arguments: hashpartitioning(cr_item_sk#55, cr_order_number#56, 5), ENSURE_REQUIR Input [4]: [cr_item_sk#55, cr_order_number#56, cr_return_amount#57, cr_net_loss#58] Arguments: [cr_item_sk#55 ASC NULLS FIRST, cr_order_number#56 ASC NULLS FIRST], false, 0 -(55) SortMergeJoin +(55) SortMergeJoin [codegen id : 19] Left keys [2]: [cs_item_sk#48, cs_order_number#50] Right keys [2]: [cr_item_sk#55, cr_order_number#56] Join condition: None @@ -509,7 +509,7 @@ Arguments: hashpartitioning(wr_item_sk#94, wr_order_number#95, 5), ENSURE_REQUIR Input [4]: [wr_item_sk#94, wr_order_number#95, wr_return_amt#96, wr_net_loss#97] Arguments: [wr_item_sk#94 ASC NULLS FIRST, wr_order_number#95 ASC NULLS FIRST], false, 0 -(86) SortMergeJoin +(86) SortMergeJoin [codegen id : 29] Left keys [2]: [ws_item_sk#86, ws_order_number#89] Right keys [2]: [wr_item_sk#94, wr_order_number#95] Join condition: None diff --git a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt index 7e82ecbf47429..e73825fd5ad7c 100644 --- a/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt +++ b/sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt @@ -30,8 +30,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - InputAdapter - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + InputAdapter WholeStageCodegen (2) Sort [ss_item_sk,ss_ticket_number] InputAdapter @@ -43,6 +43,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Scan parquet default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 ReusedExchange [d_date_sk] #5 + InputAdapter WholeStageCodegen (4) Sort [sr_item_sk,sr_ticket_number] InputAdapter @@ -99,8 +100,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [cs_catalog_page_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] BroadcastHashJoin [cs_item_sk,i_item_sk] Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - InputAdapter - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter WholeStageCodegen (12) Sort [cs_item_sk,cs_order_number] InputAdapter @@ -111,6 +112,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (14) Sort [cr_item_sk,cr_order_number] InputAdapter @@ -149,8 +151,8 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] Project [ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] BroadcastHashJoin [ws_item_sk,i_item_sk] Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - InputAdapter - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + InputAdapter WholeStageCodegen (22) Sort [ws_item_sk,ws_order_number] InputAdapter @@ -161,6 +163,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter Scan parquet default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 + InputAdapter WholeStageCodegen (24) Sort [wr_item_sk,wr_order_number] InputAdapter From 765b2471ee05c64c41bd450081a4c27a40857384 Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Tue, 11 May 2021 02:16:52 -0700 Subject: [PATCH 4/6] Add comment for findNextJoinRows --- .../execution/joins/SortMergeJoinExec.scala | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala index 3a8f262e388e8..eb78d0a623e37 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala @@ -455,8 +455,8 @@ case class SortMergeJoinExec( s"SortMergeJoin.genScanner should not take $x as the JoinType") } - // Handle the case when streamed keys less than buffered keys. - val handleStreamedLessThanBuffered = joinType match { + // Handle the case when streamed keys has no match with buffered side. + val handleStreamedWithoutMatch = joinType match { case _: InnerLike => // Skip streamed row. s"$streamedRow = null;" @@ -468,6 +468,31 @@ case class SortMergeJoinExec( s"SortMergeJoin.genScanner should not take $x as the JoinType") } + // Generate a function to scan both streamed and buffered sides to find a match. + // Return whether a match is found. + // + // `streamedIter`: the iterator for streamed side. + // `bufferedIter`: the iterator for buffered side. + // `streamedRow`: the current row from streamed side. + // When `streamedIter` is empty, `streamedRow` is null. + // `matches`: the rows from buffered side already matched with `streamedRow`. + // `matches` is buffered and reused for all `streamedRow`s having same join keys. + // If there is no match with `streamedRow`, `matches` is empty. + // `bufferedRow`: the current matched row from buffered side. + // + // The function has the following step: + // - Step 1: Find the next `streamedRow` with non-null join keys. + // For `streamedRow` with null join keys (`handleStreamedAnyNull`): + // 1. Inner join: skip the row. + // 2. Left/Right Outer join: clear the previous `matches` if needed, keep the row, + // and return false. + // - Step 2: Find the `matches` from buffered side having same join keys with `streamedRow`. + // If previous `matches` is not empty, check the join keys and clear the `matches` + // if the keys are not matched. Use `bufferedRow` to iterate buffered side to put + // all matched rows into `matches`. Return true when getting all matched rows. + // For `streamedRow` without `matches` (`handleStreamedWithoutMatch`): + // 1. Inner join: skip the row. + // 2. Left/Right Outer join: keep the row and return false. ctx.addNewFunction("findNextJoinRows", s""" |private boolean findNextJoinRows( @@ -512,7 +537,7 @@ case class SortMergeJoinExec( | ${matchedKeyVars.map(_.code).mkString("\n")} | return true; | } else { - | $handleStreamedLessThanBuffered + | $handleStreamedWithoutMatch | } | } else { | $matches.add((UnsafeRow) $bufferedRow); From 617f89c5f3fd3c111826daab4d548271a910ae1b Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Tue, 11 May 2021 15:09:19 -0700 Subject: [PATCH 5/6] Address all comments --- .../spark/sql/execution/joins/SortMergeJoinExec.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala index eb78d0a623e37..66512f10b1a65 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala @@ -486,13 +486,15 @@ case class SortMergeJoinExec( // 1. Inner join: skip the row. // 2. Left/Right Outer join: clear the previous `matches` if needed, keep the row, // and return false. + // // - Step 2: Find the `matches` from buffered side having same join keys with `streamedRow`. - // If previous `matches` is not empty, check the join keys and clear the `matches` - // if the keys are not matched. Use `bufferedRow` to iterate buffered side to put - // all matched rows into `matches`. Return true when getting all matched rows. + // Clear `matches` if we hit a new `streamedRow`, as we need to find new matches. + // Use `bufferedRow` to iterate buffered side to put all matched rows into + // `matches`. Return true when getting all matched rows. // For `streamedRow` without `matches` (`handleStreamedWithoutMatch`): // 1. Inner join: skip the row. - // 2. Left/Right Outer join: keep the row and return false. + // 2. Left/Right Outer join: keep the row and return false (with `matches` being + // empty). ctx.addNewFunction("findNextJoinRows", s""" |private boolean findNextJoinRows( From 429edcc5b6ab11f600642fc61d5205da135ce083 Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Wed, 12 May 2021 00:17:22 -0700 Subject: [PATCH 6/6] Address all comments --- .../spark/sql/execution/joins/SortMergeJoinExec.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala index 66512f10b1a65..291ab29f1b3ba 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala @@ -483,7 +483,8 @@ case class SortMergeJoinExec( // The function has the following step: // - Step 1: Find the next `streamedRow` with non-null join keys. // For `streamedRow` with null join keys (`handleStreamedAnyNull`): - // 1. Inner join: skip the row. + // 1. Inner join: skip the row. `matches` will be cleared later when hitting the + // next `streamedRow` with non-null join keys. // 2. Left/Right Outer join: clear the previous `matches` if needed, keep the row, // and return false. // @@ -699,21 +700,21 @@ case class SortMergeJoinExec( """.stripMargin lazy val outerJoin = { - val foundMatch = ctx.freshName("foundMatch") + val hasOutputRow = ctx.freshName("hasOutputRow") s""" |while ($streamedInput.hasNext()) { | findNextJoinRows($streamedInput, $bufferedInput); | ${streamedVarDecl.mkString("\n")} | ${beforeLoop.trim} | scala.collection.Iterator $iterator = $matches.generateIterator(); - | boolean $foundMatch = false; + | boolean $hasOutputRow = false; | | // the last iteration of this loop is to emit an empty row if there is no matched rows. - | while ($iterator.hasNext() || !$foundMatch) { + | while ($iterator.hasNext() || !$hasOutputRow) { | InternalRow $bufferedRow = $iterator.hasNext() ? | (InternalRow) $iterator.next() : null; | ${condCheck.trim} - | $foundMatch = true; + | $hasOutputRow = true; | $numOutput.add(1); | ${consume(ctx, resultVars)} | }