File tree Expand file tree Collapse file tree 4 files changed +17
-3
lines changed
core/src/main/java/org/apache/spark/unsafe/map
catalyst/src/main/scala/org/apache/spark/sql/internal
main/scala/org/apache/spark/sql/execution/joins
test/scala/org/apache/spark/sql Expand file tree Collapse file tree 4 files changed +17
-3
lines changed Original file line number Diff line number Diff line change @@ -429,7 +429,7 @@ public MapIterator destructiveIterator() {
429429 }
430430
431431 /**
432- * Iterator for the entries of this map. This is to first iterate over key index array
432+ * Iterator for the entries of this map. This is to first iterate over key indices in
433433 * `longArray` then accessing values in `dataPages`. NOTE: this is different from `MapIterator`
434434 * in the sense that key index is preserved here
435435 * (See `UnsafeHashedRelation` for example of usage).
Original file line number Diff line number Diff line change @@ -329,7 +329,9 @@ object SQLConf {
329329
330330 val PREFER_SORTMERGEJOIN = buildConf(" spark.sql.join.preferSortMergeJoin" )
331331 .internal()
332- .doc(" When true, prefer sort merge join over shuffle hash join." )
332+ .doc(" When true, prefer sort merge join over shuffled hash join. " +
333+ " Note that shuffled hash join supports all join types (e.g. full outer) " +
334+ " that sort merge join supports." )
333335 .version(" 2.0.0" )
334336 .booleanConf
335337 .createWithDefault(true )
Original file line number Diff line number Diff line change @@ -137,7 +137,7 @@ private[execution] object HashedRelation {
137137 0 )
138138 }
139139
140- if (! input.hasNext) {
140+ if (! input.hasNext && ! allowsNullKey ) {
141141 EmptyHashedRelation
142142 } else if (key.length == 1 && key.head.dataType == LongType && ! allowsNullKey) {
143143 // NOTE: LongHashedRelation does not support NULL keys.
Original file line number Diff line number Diff line change @@ -1199,6 +1199,18 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
11991199 (spark.range(10 ).selectExpr(" id % 5 as k1" ),
12001200 spark.range(30 ).selectExpr(" id % 5 as k2" ),
12011201 $" k1" === $" k2" ),
1202+ // Test empty build side
1203+ (spark.range(10 ).selectExpr(" id as k1" ).filter(" k1 < -1" ),
1204+ spark.range(30 ).selectExpr(" id as k2" ),
1205+ $" k1" === $" k2" ),
1206+ // Test empty stream side
1207+ (spark.range(10 ).selectExpr(" id as k1" ),
1208+ spark.range(30 ).selectExpr(" id as k2" ).filter(" k2 < -1" ),
1209+ $" k1" === $" k2" ),
1210+ // Test empty build and stream side
1211+ (spark.range(10 ).selectExpr(" id as k1" ).filter(" k1 < -1" ),
1212+ spark.range(30 ).selectExpr(" id as k2" ).filter(" k2 < -1" ),
1213+ $" k1" === $" k2" ),
12021214 // Test string join key
12031215 (spark.range(10 ).selectExpr(" cast(id * 3 as string) as k1" ),
12041216 spark.range(30 ).selectExpr(" cast(id as string) as k2" ),
You can’t perform that action at this time.
0 commit comments