From 271bf6f1fba5b559438ca021fb48b7f1fd4c4c84 Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Thu, 21 Feb 2019 18:39:11 -0600 Subject: [PATCH] Don't use inaccessible fields in SizeEstimator, which comes up in Java 9+ --- .../org/apache/spark/util/SizeEstimator.scala | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala index e12b6b71578c1..4837b01d817ab 100644 --- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala +++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala @@ -334,9 +334,21 @@ object SizeEstimator extends Logging { if (fieldClass.isPrimitive) { sizeCount(primitiveSize(fieldClass)) += 1 } else { - field.setAccessible(true) // Enable future get()'s on this field + // Note: in Java 9+ this would be better with trySetAccessible and canAccess + try { + field.setAccessible(true) // Enable future get()'s on this field + pointerFields = field :: pointerFields + } catch { + // If the field isn't accessible, we can still record the pointer size + // but can't know more about the field, so ignore it + case _: SecurityException => + // do nothing + // Java 9+ can throw InaccessibleObjectException but the class is Java 9+-only + case re: RuntimeException + if re.getClass.getSimpleName == "InaccessibleObjectException" => + // do nothing + } sizeCount(pointerSize) += 1 - pointerFields = field :: pointerFields } } }