Skip to content

Commit 0a62098

Browse files
committed
Optimize cartesian order
1 parent adb33d3 commit 0a62098

File tree

1 file changed

+9
-1
lines changed

1 file changed

+9
-1
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProduct.scala

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,15 @@ case class CartesianProduct(left: SparkPlan, right: SparkPlan) extends BinaryNod
3434
val leftResults = left.execute().map(_.copy())
3535
val rightResults = right.execute().map(_.copy())
3636

37-
leftResults.cartesian(rightResults).mapPartitions { iter =>
37+
val cartesianRdd = if(leftResults.partitions.size > rightResults.partitions.size) {
38+
rightResults.cartesian(leftResults).mapPartitions{ iter =>
39+
iter.map(tuple => (tuple._2, tuple._1))
40+
}
41+
} else {
42+
leftResults.cartesian(rightResults)
43+
}
44+
45+
cartesianRdd.mapPartitions { iter =>
3846
val joinedRow = new JoinedRow
3947
iter.map(r => joinedRow(r._1, r._2))
4048
}

0 commit comments

Comments
 (0)