Skip to content

Commit 00362da

Browse files
committed
[HOTFIX] [SPARK-3400] Revert 9b225ac "fix GraphX EdgeRDD zipPartitions"
9b225ac has been causing GraphX tests to fail nondeterministically, which is blocking development for others. Author: Ankur Dave <[email protected]> Closes apache#2271 from ankurdave/SPARK-3400 and squashes the following commits: 10c2a97 [Ankur Dave] [HOTFIX] [SPARK-3400] Revert 9b225ac "fix GraphX EdgeRDD zipPartitions"
1 parent 1bed0a3 commit 00362da

File tree

2 files changed

+2
-18
lines changed

2 files changed

+2
-18
lines changed

graphx/src/main/scala/org/apache/spark/graphx/EdgeRDD.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ package org.apache.spark.graphx
1919

2020
import scala.reflect.{classTag, ClassTag}
2121

22-
import org.apache.spark._
22+
import org.apache.spark.{OneToOneDependency, Partition, Partitioner, TaskContext}
2323
import org.apache.spark.rdd.RDD
2424
import org.apache.spark.storage.StorageLevel
2525

@@ -55,7 +55,7 @@ class EdgeRDD[@specialized ED: ClassTag, VD: ClassTag](
5555
* partitioner that allows co-partitioning with `partitionsRDD`.
5656
*/
5757
override val partitioner =
58-
partitionsRDD.partitioner.orElse(Some(new HashPartitioner(partitionsRDD.partitions.size)))
58+
partitionsRDD.partitioner.orElse(Some(Partitioner.defaultPartitioner(partitionsRDD)))
5959

6060
override def compute(part: Partition, context: TaskContext): Iterator[Edge[ED]] = {
6161
val p = firstParent[(PartitionID, EdgePartition[ED, VD])].iterator(part, context)

graphx/src/test/scala/org/apache/spark/graphx/GraphSuite.scala

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package org.apache.spark.graphx
1919

2020
import org.scalatest.FunSuite
2121

22-
import org.apache.spark.SparkConf
2322
import org.apache.spark.SparkContext
2423
import org.apache.spark.graphx.Graph._
2524
import org.apache.spark.graphx.PartitionStrategy._
@@ -351,19 +350,4 @@ class GraphSuite extends FunSuite with LocalSparkContext {
351350
}
352351
}
353352

354-
test("non-default number of edge partitions") {
355-
val n = 10
356-
val defaultParallelism = 3
357-
val numEdgePartitions = 4
358-
assert(defaultParallelism != numEdgePartitions)
359-
val conf = new SparkConf()
360-
.set("spark.default.parallelism", defaultParallelism.toString)
361-
val sc = new SparkContext("local", "test", conf)
362-
val edges = sc.parallelize((1 to n).map(x => (x: VertexId, 0: VertexId)),
363-
numEdgePartitions)
364-
val graph = Graph.fromEdgeTuples(edges, 1)
365-
val neighborAttrSums = graph.mapReduceTriplets[Int](
366-
et => Iterator((et.dstId, et.srcAttr)), _ + _)
367-
assert(neighborAttrSums.collect.toSet === Set((0: VertexId, n)))
368-
}
369353
}

0 commit comments

Comments
 (0)