From 2f267a5d63e10d3d1e986a346a4385a93a27ce7c Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Tue, 1 Jan 2019 22:21:37 -0600 Subject: [PATCH 1/3] Use normal direct ByteBuffer allocation if Cleaner can't be set --- .../org/apache/spark/unsafe/Platform.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java index 076b693f81c8..87298862f6d5 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java @@ -209,21 +209,25 @@ public static long reallocateMemory(long address, long oldSize, long newSize) { } /** - * Uses internal JDK APIs to allocate a DirectByteBuffer while ignoring the JVM's - * MaxDirectMemorySize limit (the default limit is too low and we do not want to require users - * to increase it). + * Allocate a DirectByteBuffer, potentially bypassing the JVM's MaxDirectMemorySize limit. */ public static ByteBuffer allocateDirectBuffer(int size) { try { + if (CLEANER_CREATE_METHOD == null) { + // Can't set a Cleaner (see comments on field), so need to allocate via normal Java APIs + return ByteBuffer.allocateDirect(size); + } + // Otherwise, use internal JDK APIs to allocate a DirectByteBuffer while ignoring the JVM's + // MaxDirectMemorySize limit (the default limit is too low and we do not want to + // require users to increase it). long memory = allocateMemory(size); ByteBuffer buffer = (ByteBuffer) DBB_CONSTRUCTOR.newInstance(memory, size); - if (CLEANER_CREATE_METHOD != null) { - try { - DBB_CLEANER_FIELD.set(buffer, - CLEANER_CREATE_METHOD.invoke(null, buffer, (Runnable) () -> freeMemory(memory))); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new IllegalStateException(e); - } + try { + DBB_CLEANER_FIELD.set(buffer, + CLEANER_CREATE_METHOD.invoke(null, buffer, (Runnable) () -> freeMemory(memory))); + } catch (IllegalAccessException | InvocationTargetException e) { + freeMemory(memory); + throw new IllegalStateException(e); } return buffer; } catch (Exception e) { From ee0cd3c72856f586de153a44823051ff62fd1129 Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Thu, 3 Jan 2019 10:37:53 -0600 Subject: [PATCH 2/3] Add more info to OutOfMemoryError when allocating direct buffer --- .../src/main/java/org/apache/spark/unsafe/Platform.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java index 87298862f6d5..dd80eae7f8e5 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java @@ -215,7 +215,12 @@ public static ByteBuffer allocateDirectBuffer(int size) { try { if (CLEANER_CREATE_METHOD == null) { // Can't set a Cleaner (see comments on field), so need to allocate via normal Java APIs - return ByteBuffer.allocateDirect(size); + try { + return ByteBuffer.allocateDirect(size); + } catch (OutOfMemoryError oome) { + throw new OutOfMemoryError("Failed to allocate direct buffer (" + oome.getMessage() + + "); try increasing -XX:MaxDirectMemorySize=... to, for example, your heap size"); + } } // Otherwise, use internal JDK APIs to allocate a DirectByteBuffer while ignoring the JVM's // MaxDirectMemorySize limit (the default limit is too low and we do not want to From 82a887c8e8591f040582f3bb5f23c91891d1beab Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Thu, 3 Jan 2019 12:00:20 -0600 Subject: [PATCH 3/3] Fix checkstyle --- .../unsafe/src/main/java/org/apache/spark/unsafe/Platform.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java index dd80eae7f8e5..1adf7abfc8a6 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java @@ -218,8 +218,10 @@ public static ByteBuffer allocateDirectBuffer(int size) { try { return ByteBuffer.allocateDirect(size); } catch (OutOfMemoryError oome) { + // checkstyle.off: RegexpSinglelineJava throw new OutOfMemoryError("Failed to allocate direct buffer (" + oome.getMessage() + "); try increasing -XX:MaxDirectMemorySize=... to, for example, your heap size"); + // checkstyle.on: RegexpSinglelineJava } } // Otherwise, use internal JDK APIs to allocate a DirectByteBuffer while ignoring the JVM's