File tree Expand file tree Collapse file tree 1 file changed +3
-1
lines changed Expand file tree Collapse file tree 1 file changed +3
-1
lines changed Original file line number Diff line number Diff line change 3333
3434from pyspark .resultiterable import ResultIterable
3535
36+
3637def _do_python_join (rdd , other , numPartitions , dispatch ):
3738 vs = rdd .map (lambda (k , v ): (k , (1 , v )))
3839 ws = other .map (lambda (k , v ): (k , (2 , v )))
39- return vs .union (ws ).groupByKey (numPartitions ).flatMapValues (lambda x : dispatch (x .__iter__ ()))
40+ return vs .union (ws ).groupByKey (numPartitions ).flatMapValues (lambda x : dispatch (x .__iter__ ()))
4041
4142
4243def python_join (rdd , other , numPartitions ):
@@ -85,6 +86,7 @@ def make_mapper(i):
8586 vrdds = [rdd .map (make_mapper (i )) for i , rdd in enumerate (rdds )]
8687 union_vrdds = reduce (lambda acc , other : acc .union (other ), vrdds )
8788 rdd_len = len (vrdds )
89+
8890 def dispatch (seq ):
8991 bufs = [[] for i in range (rdd_len )]
9092 for (n , v ) in seq :
You can’t perform that action at this time.
0 commit comments