From c9f4788f98e53b6a17da36a61db6308b523b9bf1 Mon Sep 17 00:00:00 2001 From: allfro Date: Sat, 12 Sep 2015 03:29:48 -0400 Subject: [PATCH] HADOOP-12406 Note: I am not an expert at JAVA, Class loaders, or Hadoop. I am just a hacker. My solution might be entirely wrong. AbstractMapWritable.readFields throws a ClassNotFoundException when reading custom writables. Debugging the job using remote debugging in IntelliJ revealed that the class loader being used in Class.forName() is different than that used by the Thread's current context (Thread.currentThread().getContextClassLoader()). The class path for the system class loader does not include the libraries of the job jar. However, the class path for the context class loader does. The proposed patch changes the class loading mechanism in readFields to use the Thread's context class loader instead of the system's default class loader. --- .../apache/hadoop/io/AbstractMapWritable.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java index cc1e517eaa586..9e0674b942514 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java @@ -195,23 +195,23 @@ public void write(DataOutput out) throws IOException { @Override public void readFields(DataInput in) throws IOException { - + // Get the number of "unknown" classes - newClasses = in.readByte(); - + + // Get the class loader of the current thread because + // Class.forName does not have the job jar in its path + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + // Then read in the class names and add them to our tables - for (int i = 0; i < newClasses; i++) { byte id = in.readByte(); String className = in.readUTF(); try { - addToMap(Class.forName(className), id); - + addToMap(classLoader.loadClass(className), id); } catch (ClassNotFoundException e) { - throw new IOException("can't find class: " + className + " because "+ - e.getMessage()); + throw new IOException(e); } } - } + } }