From 7cb0c446d9908ff53f3106d7a9446daa497bdc0f Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2018 10:14:46 +0800 Subject: [PATCH 1/4] child first classpath first has no regular parent --- .../org/apache/spark/sql/hive/HiveUtils.scala | 4 ++- .../spark/sql/hive/HiveUtilsSuite.scala | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index c489690af8cd1..c7717d70c996f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -47,7 +47,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf._ import org.apache.spark.sql.internal.StaticSQLConf.{CATALOG_IMPLEMENTATION, WAREHOUSE_PATH} import org.apache.spark.sql.types._ -import org.apache.spark.util.Utils +import org.apache.spark.util.{ChildFirstURLClassLoader, Utils} private[spark] object HiveUtils extends Logging { @@ -312,6 +312,8 @@ private[spark] object HiveUtils extends Logging { // starting from the given classLoader. def allJars(classLoader: ClassLoader): Array[URL] = classLoader match { case null => Array.empty[URL] + case childFirst: ChildFirstURLClassLoader => + childFirst.getURLs() ++ allJars(Utils.getSparkClassLoader) case urlClassLoader: URLClassLoader => urlClassLoader.getURLs ++ allJars(urlClassLoader.getParent) case other => allJars(other.getParent) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala index fdbfcf1a68440..cfa59667c21ee 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala @@ -17,11 +17,16 @@ package org.apache.spark.sql.hive +import java.net.URL + import org.apache.hadoop.hive.conf.HiveConf.ConfVars +import org.apache.spark.SparkConf +import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.sql.QueryTest import org.apache.spark.sql.hive.test.TestHiveSingleton import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader} class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton { @@ -42,4 +47,31 @@ class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton assert(hiveConf("foo") === "bar") } } + + test("ChildFirstURLClassLoader's parent is null") { + val conf = new SparkConf + val contextClassLoader = Thread.currentThread().getContextClassLoader + val loader = new FakeChildFirstURLClassLoader(Array(), contextClassLoader) + Thread.currentThread().setContextClassLoader(loader) + intercept[IllegalArgumentException]( + HiveUtils.newClientForMetadata(conf, SparkHadoopUtil.newConfiguration(conf))) + Thread.currentThread().setContextClassLoader(contextClassLoader) + } + + test("ChildFirstURLClassLoader's parent is null, get spark classloader instead") { + val conf = new SparkConf + val contextClassLoader = Thread.currentThread().getContextClassLoader + val loader = new ChildFirstURLClassLoader(Array(), contextClassLoader) + Thread.currentThread().setContextClassLoader(loader) + HiveUtils.newClientForMetadata(conf, SparkHadoopUtil.newConfiguration(conf)) + Thread.currentThread().setContextClassLoader(contextClassLoader) + } } + +/** + * A Fake [[ChildFirstURLClassLoader]] used for test + * @param urls + * @param parent + */ +class FakeChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader) + extends MutableURLClassLoader(urls, null) From bb27e1714a35db284d594aa71c745af7ce1a8364 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2018 10:45:52 +0800 Subject: [PATCH 2/4] doc --- .../test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala index cfa59667c21ee..f30a939b88fb4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala @@ -70,8 +70,6 @@ class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton /** * A Fake [[ChildFirstURLClassLoader]] used for test - * @param urls - * @param parent */ -class FakeChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader) +private[spark] class FakeChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader) extends MutableURLClassLoader(urls, null) From e1a9f2e12d17aeb3f7d20ed6695424c2c2765d16 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2018 14:23:09 +0800 Subject: [PATCH 3/4] surround with try finally --- .../spark/sql/hive/HiveUtilsSuite.scala | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala index f30a939b88fb4..6a864990b96de 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala @@ -52,19 +52,31 @@ class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton val conf = new SparkConf val contextClassLoader = Thread.currentThread().getContextClassLoader val loader = new FakeChildFirstURLClassLoader(Array(), contextClassLoader) - Thread.currentThread().setContextClassLoader(loader) - intercept[IllegalArgumentException]( - HiveUtils.newClientForMetadata(conf, SparkHadoopUtil.newConfiguration(conf))) - Thread.currentThread().setContextClassLoader(contextClassLoader) + try { + Thread.currentThread().setContextClassLoader(loader) + intercept[IllegalArgumentException]( + HiveUtils.newClientForMetadata( + conf, + SparkHadoopUtil.newConfiguration(conf), + HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true))) + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader) + } } test("ChildFirstURLClassLoader's parent is null, get spark classloader instead") { val conf = new SparkConf val contextClassLoader = Thread.currentThread().getContextClassLoader val loader = new ChildFirstURLClassLoader(Array(), contextClassLoader) - Thread.currentThread().setContextClassLoader(loader) - HiveUtils.newClientForMetadata(conf, SparkHadoopUtil.newConfiguration(conf)) - Thread.currentThread().setContextClassLoader(contextClassLoader) + try { + Thread.currentThread().setContextClassLoader(loader) + HiveUtils.newClientForMetadata( + conf, + SparkHadoopUtil.newConfiguration(conf), + HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true)) + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader) + } } } From fea2490027c611fb9be61ea33ad4de962ccf2ffa Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 4 Jan 2018 15:02:21 +0800 Subject: [PATCH 4/4] rm one unit test --- .../spark/sql/hive/HiveUtilsSuite.scala | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala index 6a864990b96de..8697d47e89e89 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala @@ -48,22 +48,6 @@ class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton } } - test("ChildFirstURLClassLoader's parent is null") { - val conf = new SparkConf - val contextClassLoader = Thread.currentThread().getContextClassLoader - val loader = new FakeChildFirstURLClassLoader(Array(), contextClassLoader) - try { - Thread.currentThread().setContextClassLoader(loader) - intercept[IllegalArgumentException]( - HiveUtils.newClientForMetadata( - conf, - SparkHadoopUtil.newConfiguration(conf), - HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true))) - } finally { - Thread.currentThread().setContextClassLoader(contextClassLoader) - } - } - test("ChildFirstURLClassLoader's parent is null, get spark classloader instead") { val conf = new SparkConf val contextClassLoader = Thread.currentThread().getContextClassLoader @@ -79,9 +63,3 @@ class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton } } } - -/** - * A Fake [[ChildFirstURLClassLoader]] used for test - */ -private[spark] class FakeChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader) - extends MutableURLClassLoader(urls, null)