Skip to content

Commit dfad4fc

Browse files
prakharjain09prakharjain09
committed
test fixes
1 parent e9a25d9 commit dfad4fc

File tree

6 files changed

+168
-179
lines changed

6 files changed

+168
-179
lines changed

‎sql/core/src/main/scala/org/apache/spark/sql/execution/CollapseAggregates.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ object CollapseAggregates extends Rule[SparkPlan] {
4343
if checkIfAggregatesCanBeCollapsed(parent, child) =>
4444
val completeAggregateExpressions = child.aggregateExpressions.map(_.copy(mode = Complete))
4545
HashAggregateExec(
46-
requiredChildDistributionExpressions = parent.requiredChildDistributionExpressions,
46+
requiredChildDistributionExpressions = Some(child.groupingExpressions),
4747
groupingExpressions = child.groupingExpressions,
4848
aggregateExpressions = completeAggregateExpressions,
4949
aggregateAttributes = completeAggregateExpressions.map(_.resultAttribute),
@@ -55,7 +55,7 @@ object CollapseAggregates extends Rule[SparkPlan] {
5555
if checkIfAggregatesCanBeCollapsed(parent, child) =>
5656
val completeAggregateExpressions = child.aggregateExpressions.map(_.copy(mode = Complete))
5757
SortAggregateExec(
58-
requiredChildDistributionExpressions = parent.requiredChildDistributionExpressions,
58+
requiredChildDistributionExpressions = Some(child.groupingExpressions),
5959
groupingExpressions = child.groupingExpressions,
6060
aggregateExpressions = completeAggregateExpressions,
6161
aggregateAttributes = completeAggregateExpressions.map(_.resultAttribute),
@@ -67,7 +67,7 @@ object CollapseAggregates extends Rule[SparkPlan] {
6767
if checkIfAggregatesCanBeCollapsed(parent, child) =>
6868
val completeAggregateExpressions = child.aggregateExpressions.map(_.copy(mode = Complete))
6969
ObjectHashAggregateExec(
70-
requiredChildDistributionExpressions = parent.requiredChildDistributionExpressions,
70+
requiredChildDistributionExpressions = Some(child.groupingExpressions),
7171
groupingExpressions = child.groupingExpressions,
7272
aggregateExpressions = completeAggregateExpressions,
7373
aggregateAttributes = completeAggregateExpressions.map(_.resultAttribute),

‎sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ case class AdaptiveSparkPlanExec(
9090
RemoveRedundantProjects,
9191
EnsureRequirements,
9292
RemoveRedundantSorts,
93+
CollapseAggregates,
9394
DisableUnnecessaryBucketedScan
9495
) ++ context.session.sessionState.queryStagePrepRules
9596

‎sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/explain.txt‎

Lines changed: 92 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,54 @@
11
== Physical Plan ==
2-
TakeOrderedAndProject (51)
3-
+- * Project (50)
4-
+- * SortMergeJoin Inner (49)
5-
:- * Sort (46)
6-
: +- Exchange (45)
7-
: +- * Project (44)
8-
: +- * SortMergeJoin Inner (43)
9-
: :- * Sort (37)
10-
: : +- Exchange (36)
11-
: : +- * HashAggregate (35)
12-
: : +- * HashAggregate (34)
13-
: : +- * Project (33)
14-
: : +- * SortMergeJoin Inner (32)
15-
: : :- * Sort (26)
16-
: : : +- Exchange (25)
17-
: : : +- * Project (24)
18-
: : : +- * BroadcastHashJoin Inner BuildRight (23)
19-
: : : :- * Project (17)
20-
: : : : +- * BroadcastHashJoin Inner BuildRight (16)
21-
: : : : :- * Project (10)
22-
: : : : : +- * BroadcastHashJoin Inner BuildRight (9)
23-
: : : : : :- * Filter (3)
24-
: : : : : : +- * ColumnarToRow (2)
25-
: : : : : : +- Scan parquet default.store_sales (1)
26-
: : : : : +- BroadcastExchange (8)
27-
: : : : : +- * Project (7)
28-
: : : : : +- * Filter (6)
29-
: : : : : +- * ColumnarToRow (5)
30-
: : : : : +- Scan parquet default.date_dim (4)
31-
: : : : +- BroadcastExchange (15)
32-
: : : : +- * Project (14)
33-
: : : : +- * Filter (13)
34-
: : : : +- * ColumnarToRow (12)
35-
: : : : +- Scan parquet default.store (11)
36-
: : : +- BroadcastExchange (22)
37-
: : : +- * Project (21)
38-
: : : +- * Filter (20)
39-
: : : +- * ColumnarToRow (19)
40-
: : : +- Scan parquet default.household_demographics (18)
41-
: : +- * Sort (31)
42-
: : +- Exchange (30)
43-
: : +- * Filter (29)
44-
: : +- * ColumnarToRow (28)
45-
: : +- Scan parquet default.customer_address (27)
46-
: +- * Sort (42)
47-
: +- Exchange (41)
48-
: +- * Filter (40)
49-
: +- * ColumnarToRow (39)
50-
: +- Scan parquet default.customer (38)
51-
+- * Sort (48)
52-
+- ReusedExchange (47)
2+
TakeOrderedAndProject (50)
3+
+- * Project (49)
4+
+- * SortMergeJoin Inner (48)
5+
:- * Sort (45)
6+
: +- Exchange (44)
7+
: +- * Project (43)
8+
: +- * SortMergeJoin Inner (42)
9+
: :- * Sort (36)
10+
: : +- Exchange (35)
11+
: : +- * HashAggregate (34)
12+
: : +- * Project (33)
13+
: : +- * SortMergeJoin Inner (32)
14+
: : :- * Sort (26)
15+
: : : +- Exchange (25)
16+
: : : +- * Project (24)
17+
: : : +- * BroadcastHashJoin Inner BuildRight (23)
18+
: : : :- * Project (17)
19+
: : : : +- * BroadcastHashJoin Inner BuildRight (16)
20+
: : : : :- * Project (10)
21+
: : : : : +- * BroadcastHashJoin Inner BuildRight (9)
22+
: : : : : :- * Filter (3)
23+
: : : : : : +- * ColumnarToRow (2)
24+
: : : : : : +- Scan parquet default.store_sales (1)
25+
: : : : : +- BroadcastExchange (8)
26+
: : : : : +- * Project (7)
27+
: : : : : +- * Filter (6)
28+
: : : : : +- * ColumnarToRow (5)
29+
: : : : : +- Scan parquet default.date_dim (4)
30+
: : : : +- BroadcastExchange (15)
31+
: : : : +- * Project (14)
32+
: : : : +- * Filter (13)
33+
: : : : +- * ColumnarToRow (12)
34+
: : : : +- Scan parquet default.store (11)
35+
: : : +- BroadcastExchange (22)
36+
: : : +- * Project (21)
37+
: : : +- * Filter (20)
38+
: : : +- * ColumnarToRow (19)
39+
: : : +- Scan parquet default.household_demographics (18)
40+
: : +- * Sort (31)
41+
: : +- Exchange (30)
42+
: : +- * Filter (29)
43+
: : +- * ColumnarToRow (28)
44+
: : +- Scan parquet default.customer_address (27)
45+
: +- * Sort (41)
46+
: +- Exchange (40)
47+
: +- * Filter (39)
48+
: +- * ColumnarToRow (38)
49+
: +- Scan parquet default.customer (37)
50+
+- * Sort (47)
51+
+- ReusedExchange (46)
5352

5453

5554
(1) Scan parquet default.store_sales
@@ -201,81 +200,74 @@ Input [7]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7,
201200
(34) HashAggregate [codegen id : 8]
202201
Input [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8, ca_city#22]
203202
Keys [4]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#22]
204-
Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#7)), partial_sum(UnscaledValue(ss_net_profit#8))]
205-
Aggregate Attributes [2]: [sum#24, sum#25]
206-
Results [6]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#22, sum#26, sum#27]
207-
208-
(35) HashAggregate [codegen id : 8]
209-
Input [6]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#22, sum#26, sum#27]
210-
Keys [4]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#22]
211203
Functions [2]: [sum(UnscaledValue(ss_coupon_amt#7)), sum(UnscaledValue(ss_net_profit#8))]
212-
Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#7))#28, sum(UnscaledValue(ss_net_profit#8))#29]
213-
Results [5]: [ss_ticket_number#6, ss_customer_sk#2, ca_city#22 AS bought_city#30, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#7))#28,17,2) AS amt#31, MakeDecimal(sum(UnscaledValue(ss_net_profit#8))#29,17,2) AS profit#32]
204+
Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#7))#24, sum(UnscaledValue(ss_net_profit#8))#25]
205+
Results [5]: [ss_ticket_number#6, ss_customer_sk#2, ca_city#22 AS bought_city#26, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#7))#24,17,2) AS amt#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#8))#25,17,2) AS profit#28]
214206

215-
(36) Exchange
216-
Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, amt#31, profit#32]
217-
Arguments: hashpartitioning(ss_customer_sk#2, 5), true, [id=#33]
207+
(35) Exchange
208+
Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit#28]
209+
Arguments: hashpartitioning(ss_customer_sk#2, 5), true, [id=#29]
218210

219-
(37) Sort [codegen id : 9]
220-
Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, amt#31, profit#32]
211+
(36) Sort [codegen id : 9]
212+
Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit#28]
221213
Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0
222214

223-
(38) Scan parquet default.customer
224-
Output [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
215+
(37) Scan parquet default.customer
216+
Output [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
225217
Batched: true
226218
Location [not included in comparison]/{warehouse_dir}/customer]
227219
PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)]
228220
ReadSchema: struct<c_customer_sk:int,c_current_addr_sk:int,c_first_name:string,c_last_name:string>
229221

230-
(39) ColumnarToRow [codegen id : 10]
231-
Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
222+
(38) ColumnarToRow [codegen id : 10]
223+
Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
232224

233-
(40) Filter [codegen id : 10]
234-
Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
235-
Condition : (isnotnull(c_customer_sk#34) AND isnotnull(c_current_addr_sk#35))
225+
(39) Filter [codegen id : 10]
226+
Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
227+
Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_current_addr_sk#31))
236228

237-
(41) Exchange
238-
Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
239-
Arguments: hashpartitioning(c_customer_sk#34, 5), true, [id=#38]
229+
(40) Exchange
230+
Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
231+
Arguments: hashpartitioning(c_customer_sk#30, 5), true, [id=#34]
240232

241-
(42) Sort [codegen id : 11]
242-
Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
243-
Arguments: [c_customer_sk#34 ASC NULLS FIRST], false, 0
233+
(41) Sort [codegen id : 11]
234+
Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
235+
Arguments: [c_customer_sk#30 ASC NULLS FIRST], false, 0
244236

245-
(43) SortMergeJoin [codegen id : 12]
237+
(42) SortMergeJoin [codegen id : 12]
246238
Left keys [1]: [ss_customer_sk#2]
247-
Right keys [1]: [c_customer_sk#34]
239+
Right keys [1]: [c_customer_sk#30]
248240
Join condition: None
249241

250-
(44) Project [codegen id : 12]
251-
Output [7]: [ss_ticket_number#6, bought_city#30, amt#31, profit#32, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
252-
Input [9]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, amt#31, profit#32, c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
242+
(43) Project [codegen id : 12]
243+
Output [7]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
244+
Input [9]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit#28, c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
253245

254-
(45) Exchange
255-
Input [7]: [ss_ticket_number#6, bought_city#30, amt#31, profit#32, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
256-
Arguments: hashpartitioning(c_current_addr_sk#35, 5), true, [id=#39]
246+
(44) Exchange
247+
Input [7]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
248+
Arguments: hashpartitioning(c_current_addr_sk#31, 5), true, [id=#35]
257249

258-
(46) Sort [codegen id : 13]
259-
Input [7]: [ss_ticket_number#6, bought_city#30, amt#31, profit#32, c_current_addr_sk#35, c_first_name#36, c_last_name#37]
260-
Arguments: [c_current_addr_sk#35 ASC NULLS FIRST], false, 0
250+
(45) Sort [codegen id : 13]
251+
Input [7]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#31, c_first_name#32, c_last_name#33]
252+
Arguments: [c_current_addr_sk#31 ASC NULLS FIRST], false, 0
261253

262-
(47) ReusedExchange [Reuses operator id: 30]
254+
(46) ReusedExchange [Reuses operator id: 30]
263255
Output [2]: [ca_address_sk#21, ca_city#22]
264256

265-
(48) Sort [codegen id : 15]
257+
(47) Sort [codegen id : 15]
266258
Input [2]: [ca_address_sk#21, ca_city#22]
267259
Arguments: [ca_address_sk#21 ASC NULLS FIRST], false, 0
268260

269-
(49) SortMergeJoin [codegen id : 16]
270-
Left keys [1]: [c_current_addr_sk#35]
261+
(48) SortMergeJoin [codegen id : 16]
262+
Left keys [1]: [c_current_addr_sk#31]
271263
Right keys [1]: [ca_address_sk#21]
272-
Join condition: NOT (ca_city#22 = bought_city#30)
264+
Join condition: NOT (ca_city#22 = bought_city#26)
273265

274-
(50) Project [codegen id : 16]
275-
Output [7]: [c_last_name#37, c_first_name#36, ca_city#22, bought_city#30, ss_ticket_number#6, amt#31, profit#32]
276-
Input [9]: [ss_ticket_number#6, bought_city#30, amt#31, profit#32, c_current_addr_sk#35, c_first_name#36, c_last_name#37, ca_address_sk#21, ca_city#22]
266+
(49) Project [codegen id : 16]
267+
Output [7]: [c_last_name#33, c_first_name#32, ca_city#22, bought_city#26, ss_ticket_number#6, amt#27, profit#28]
268+
Input [9]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#31, c_first_name#32, c_last_name#33, ca_address_sk#21, ca_city#22]
277269

278-
(51) TakeOrderedAndProject
279-
Input [7]: [c_last_name#37, c_first_name#36, ca_city#22, bought_city#30, ss_ticket_number#6, amt#31, profit#32]
280-
Arguments: 100, [c_last_name#37 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, ca_city#22 ASC NULLS FIRST, bought_city#30 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST], [c_last_name#37, c_first_name#36, ca_city#22, bought_city#30, ss_ticket_number#6, amt#31, profit#32]
270+
(50) TakeOrderedAndProject
271+
Input [7]: [c_last_name#33, c_first_name#32, ca_city#22, bought_city#26, ss_ticket_number#6, amt#27, profit#28]
272+
Arguments: 100, [c_last_name#33 ASC NULLS FIRST, c_first_name#32 ASC NULLS FIRST, ca_city#22 ASC NULLS FIRST, bought_city#26 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST], [c_last_name#33, c_first_name#32, ca_city#22, bought_city#26, ss_ticket_number#6, amt#27, profit#28]
281273

0 commit comments

Comments
 (0)