Skip to content

Commit e1f6845

Browse files
tedyuzsxwing
authored andcommitted
[SPARK-12181] Check Cached unaligned-access capability before using Unsafe
## What changes were proposed in this pull request? For MemoryMode.OFF_HEAP, Unsafe.getInt etc. are used with no restriction. However, the Oracle implementation uses these methods only if the class variable unaligned (commented as "Cached unaligned-access capability") is true, which seems to be calculated whether the architecture is i386, x86, amd64, or x86_64. I think we should perform similar check for the use of Unsafe. Reference: https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java#L112 ## How was this patch tested? Unit test suite Author: tedyu <[email protected]> Closes #11943 from tedyu/master.
1 parent 366cac6 commit e1f6845

File tree

2 files changed

+31
-0
lines changed

2 files changed

+31
-0
lines changed

common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.spark.unsafe;
1919

2020
import java.lang.reflect.Field;
21+
import java.lang.reflect.Method;
2122

2223
import sun.misc.Unsafe;
2324

@@ -37,6 +38,33 @@ public final class Platform {
3738

3839
public static final int DOUBLE_ARRAY_OFFSET;
3940

41+
private static final boolean unaligned;
42+
static {
43+
boolean _unaligned;
44+
// use reflection to access unaligned field
45+
try {
46+
Class<?> bitsClass =
47+
Class.forName("java.nio.Bits", false, ClassLoader.getSystemClassLoader());
48+
Method unalignedMethod = bitsClass.getDeclaredMethod("unaligned");
49+
unalignedMethod.setAccessible(true);
50+
_unaligned = Boolean.TRUE.equals(unalignedMethod.invoke(null));
51+
} catch (Throwable t) {
52+
// We at least know x86 and x64 support unaligned access.
53+
String arch = System.getProperty("os.arch", "");
54+
//noinspection DynamicRegexReplaceableByCompiledPattern
55+
_unaligned = arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64)$");
56+
}
57+
unaligned = _unaligned;
58+
}
59+
60+
/**
61+
* @return true when running JVM is having sun's Unsafe package available in it and underlying
62+
* system having unaligned-access capability.
63+
*/
64+
public static boolean unaligned() {
65+
return unaligned;
66+
}
67+
4068
public static int getInt(Object object, long offset) {
4169
return _UNSAFE.getInt(object, offset);
4270
}

core/src/main/scala/org/apache/spark/memory/MemoryManager.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import org.apache.spark.SparkConf
2323
import org.apache.spark.internal.Logging
2424
import org.apache.spark.storage.BlockId
2525
import org.apache.spark.storage.memory.MemoryStore
26+
import org.apache.spark.unsafe.Platform
2627
import org.apache.spark.unsafe.array.ByteArrayMethods
2728
import org.apache.spark.unsafe.memory.MemoryAllocator
2829

@@ -190,6 +191,8 @@ private[spark] abstract class MemoryManager(
190191
if (conf.getBoolean("spark.memory.offHeap.enabled", false)) {
191192
require(conf.getSizeAsBytes("spark.memory.offHeap.size", 0) > 0,
192193
"spark.memory.offHeap.size must be > 0 when spark.memory.offHeap.enabled == true")
194+
require(Platform.unaligned(),
195+
"No support for unaligned Unsafe. Set spark.memory.offHeap.enabled to false.")
193196
MemoryMode.OFF_HEAP
194197
} else {
195198
MemoryMode.ON_HEAP

0 commit comments

Comments
 (0)