From 4db388084de12d805ae905d9062db75e130a7b73 Mon Sep 17 00:00:00 2001 From: Sital Kedia Date: Thu, 17 Mar 2016 13:53:34 -0700 Subject: [PATCH] [SPARK-13958]Executor OOM due to unbounded growth of pointer array in Sorter --- .../spark/shuffle/sort/ShuffleExternalSorter.java | 13 ++++++++++++- .../unsafe/sort/UnsafeExternalSorter.java | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java index f97e76d7ed0d9..87c208e3de4b2 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java @@ -320,7 +320,18 @@ private void growPointerArrayIfNecessary() throws IOException { assert(inMemSorter != null); if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); - LongArray array = allocateArray(used / 8 * 2); + LongArray array; + try { + // could trigger spilling + array = allocateArray(used / 8 * 2); + } catch (OutOfMemoryError e) { + // should have trigger spilling + if (!inMemSorter.hasSpaceForAnotherRecord()) { + logger.error("Unable to grow the pointer array"); + throw e; + } + return; + } // check if spilling is triggered or not if (inMemSorter.hasSpaceForAnotherRecord()) { freeArray(array); diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java index 296bf722fc178..476974dcd2185 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java @@ -293,7 +293,18 @@ private void growPointerArrayIfNecessary() throws IOException { assert(inMemSorter != null); if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); - LongArray array = allocateArray(used / 8 * 2); + LongArray array; + try { + // could trigger spilling + array = allocateArray(used / 8 * 2); + } catch (OutOfMemoryError e) { + // should have trigger spilling + if (!inMemSorter.hasSpaceForAnotherRecord()) { + logger.error("Unable to grow the pointer array"); + throw e; + } + return; + } // check if spilling is triggered or not if (inMemSorter.hasSpaceForAnotherRecord()) { freeArray(array);