From 70eb93eb6fa269c26f82e1125aeea69a589d0428 Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 10 Sep 2015 11:27:01 -0700 Subject: [PATCH 1/5] Check partitionId's range in ExternalSorter#spill() --- .../org/apache/spark/util/collection/ExternalSorter.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala index 19287edbaf166..9cfed1cc647f7 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala @@ -291,6 +291,12 @@ private[spark] class ExternalSorter[K, V, C]( val it = collection.destructiveSortedWritablePartitionedIterator(comparator) while (it.hasNext) { val partitionId = it.nextPartition() + if (partitionId < 0) { + throw new IllegalArgumentException("Encountered negative partition Id: " + partitionId) + } + if (partitionId >= numPartitions) { + throw new IllegalArgumentException("Encountered partition Id: " + partitionId + " which is >= " + numPartitions) + } it.writeNext(writer) elementsPerPartition(partitionId) += 1 objectsWritten += 1 From 63dfe1191ed9f04560cfd39fe01108fcef462673 Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 10 Sep 2015 11:29:11 -0700 Subject: [PATCH 2/5] Correct indentation --- .../org/apache/spark/util/collection/ExternalSorter.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala index 9cfed1cc647f7..5e9439f2f08e3 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala @@ -294,8 +294,8 @@ private[spark] class ExternalSorter[K, V, C]( if (partitionId < 0) { throw new IllegalArgumentException("Encountered negative partition Id: " + partitionId) } - if (partitionId >= numPartitions) { - throw new IllegalArgumentException("Encountered partition Id: " + partitionId + " which is >= " + numPartitions) + if (partitionId >= numPartitions) { + throw new IllegalArgumentException("Encountered partition Id: " + partitionId + " which is >= " + numPartitions) } it.writeNext(writer) elementsPerPartition(partitionId) += 1 From 04aa45e56aa79c402fdb4d51e3972f850724b05d Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 10 Sep 2015 12:15:02 -0700 Subject: [PATCH 3/5] Wrap long line --- .../org/apache/spark/util/collection/ExternalSorter.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala index 5e9439f2f08e3..feddc6190381d 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala @@ -292,10 +292,11 @@ private[spark] class ExternalSorter[K, V, C]( while (it.hasNext) { val partitionId = it.nextPartition() if (partitionId < 0) { - throw new IllegalArgumentException("Encountered negative partition Id: " + partitionId) + throw new IllegalArgumentException("Encountered negative partition Id: " + partitionId) } if (partitionId >= numPartitions) { - throw new IllegalArgumentException("Encountered partition Id: " + partitionId + " which is >= " + numPartitions) + throw new IllegalArgumentException("Encountered partition Id: " + partitionId + + " which is >= " + numPartitions) } it.writeNext(writer) elementsPerPartition(partitionId) += 1 From 0e86ae8f48f513d4f65a7f24e549006850945543 Mon Sep 17 00:00:00 2001 From: tedyu Date: Fri, 11 Sep 2015 04:04:32 -0700 Subject: [PATCH 4/5] Use require to check for legitimate range --- .../apache/spark/util/collection/ExternalSorter.scala | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala index feddc6190381d..56bd0a49f4707 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala @@ -291,13 +291,8 @@ private[spark] class ExternalSorter[K, V, C]( val it = collection.destructiveSortedWritablePartitionedIterator(comparator) while (it.hasNext) { val partitionId = it.nextPartition() - if (partitionId < 0) { - throw new IllegalArgumentException("Encountered negative partition Id: " + partitionId) - } - if (partitionId >= numPartitions) { - throw new IllegalArgumentException("Encountered partition Id: " + partitionId + - " which is >= " + numPartitions) - } + require(partitionId >= 0 && partitionId < numPartitions, + s"partition Id: ${partitionId} should be in the range (0, ${numPartitions})") it.writeNext(writer) elementsPerPartition(partitionId) += 1 objectsWritten += 1 From a293906a3cfd6ed9d1c4ce648cf0c6a1ccba140d Mon Sep 17 00:00:00 2001 From: tedyu Date: Fri, 11 Sep 2015 07:03:47 -0700 Subject: [PATCH 5/5] Use square bracket for left side of range --- .../scala/org/apache/spark/util/collection/ExternalSorter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala index 56bd0a49f4707..ae7e7dd7c8d35 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala @@ -292,7 +292,7 @@ private[spark] class ExternalSorter[K, V, C]( while (it.hasNext) { val partitionId = it.nextPartition() require(partitionId >= 0 && partitionId < numPartitions, - s"partition Id: ${partitionId} should be in the range (0, ${numPartitions})") + s"partition Id: ${partitionId} should be in the range [0, ${numPartitions})") it.writeNext(writer) elementsPerPartition(partitionId) += 1 objectsWritten += 1