From 5dd871ada65008510ec6171b81f09bb57731dc3e Mon Sep 17 00:00:00 2001 From: Ngone51 Date: Fri, 18 Jan 2019 15:46:37 +0800 Subject: [PATCH 1/2] use existing partitioner when defaultNumPartitions == MaxPartitioner.numPartitions --- core/src/main/scala/org/apache/spark/Partitioner.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala index 515237558fd8..6adfc0b882e0 100644 --- a/core/src/main/scala/org/apache/spark/Partitioner.scala +++ b/core/src/main/scala/org/apache/spark/Partitioner.scala @@ -81,7 +81,7 @@ object Partitioner { // If the existing max partitioner is an eligible one, or its partitions number is larger // than the default number of partitions, use the existing partitioner. if (hasMaxPartitioner.nonEmpty && (isEligiblePartitioner(hasMaxPartitioner.get, rdds) || - defaultNumPartitions < hasMaxPartitioner.get.getNumPartitions)) { + defaultNumPartitions <= hasMaxPartitioner.get.getNumPartitions)) { hasMaxPartitioner.get.partitioner.get } else { new HashPartitioner(defaultNumPartitions) From 44f632b4b734f58782463ebd6535dee97923b802 Mon Sep 17 00:00:00 2001 From: Ngone51 Date: Tue, 22 Jan 2019 11:02:46 +0800 Subject: [PATCH 2/2] update comment --- core/src/main/scala/org/apache/spark/Partitioner.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala index 6adfc0b882e0..a0cba8ab13fe 100644 --- a/core/src/main/scala/org/apache/spark/Partitioner.scala +++ b/core/src/main/scala/org/apache/spark/Partitioner.scala @@ -51,8 +51,8 @@ object Partitioner { * * When available, we choose the partitioner from rdds with maximum number of partitions. If this * partitioner is eligible (number of partitions within an order of maximum number of partitions - * in rdds), or has partition number higher than default partitions number - we use this - * partitioner. + * in rdds), or has partition number higher than or equal to default partitions number - we use + * this partitioner. * * Otherwise, we'll use a new HashPartitioner with the default partitions number. * @@ -79,7 +79,7 @@ object Partitioner { } // If the existing max partitioner is an eligible one, or its partitions number is larger - // than the default number of partitions, use the existing partitioner. + // than or equal to the default number of partitions, use the existing partitioner. if (hasMaxPartitioner.nonEmpty && (isEligiblePartitioner(hasMaxPartitioner.get, rdds) || defaultNumPartitions <= hasMaxPartitioner.get.getNumPartitions)) { hasMaxPartitioner.get.partitioner.get