From 7bcb8d41856f100d30cb2086dff0f9fd05796322 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 25 Sep 2019 16:24:28 -0700 Subject: [PATCH 1/3] [SPARK-29250][BUILD] Upgrade to Hadoop 3.2.1 --- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 34 +++++++++++++------------ pom.xml | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index aa1a8ccc42e3..405ffd2bf1b7 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -20,6 +20,8 @@ automaton-1.11-8.jar avro-1.8.2.jar avro-ipc-1.8.2.jar avro-mapred-1.8.2-hadoop2.jar +bcpkix-jdk15on-1.60.jar +bcprov-jdk15on-1.60.jar bonecp-0.8.0.RELEASE.jar breeze-macros_2.12-1.0.jar breeze_2.12-1.0.jar @@ -64,20 +66,20 @@ gson-2.2.4.jar guava-14.0.1.jar guice-4.0.jar guice-servlet-4.0.jar -hadoop-annotations-3.2.0.jar -hadoop-auth-3.2.0.jar -hadoop-client-3.2.0.jar -hadoop-common-3.2.0.jar -hadoop-hdfs-client-3.2.0.jar -hadoop-mapreduce-client-common-3.2.0.jar -hadoop-mapreduce-client-core-3.2.0.jar -hadoop-mapreduce-client-jobclient-3.2.0.jar -hadoop-yarn-api-3.2.0.jar -hadoop-yarn-client-3.2.0.jar -hadoop-yarn-common-3.2.0.jar -hadoop-yarn-registry-3.2.0.jar -hadoop-yarn-server-common-3.2.0.jar -hadoop-yarn-server-web-proxy-3.2.0.jar +hadoop-annotations-3.2.1.jar +hadoop-auth-3.2.1.jar +hadoop-client-3.2.1.jar +hadoop-common-3.2.1.jar +hadoop-hdfs-client-3.2.1.jar +hadoop-mapreduce-client-common-3.2.1.jar +hadoop-mapreduce-client-core-3.2.1.jar +hadoop-mapreduce-client-jobclient-3.2.1.jar +hadoop-yarn-api-3.2.1.jar +hadoop-yarn-client-3.2.1.jar +hadoop-yarn-common-3.2.1.jar +hadoop-yarn-registry-3.2.1.jar +hadoop-yarn-server-common-3.2.1.jar +hadoop-yarn-server-web-proxy-3.2.1.jar hive-beeline-2.3.6.jar hive-cli-2.3.6.jar hive-common-2.3.6.jar @@ -106,8 +108,8 @@ jackson-core-2.10.0.jar jackson-core-asl-1.9.13.jar jackson-databind-2.10.0.jar jackson-dataformat-yaml-2.10.0.jar -jackson-jaxrs-base-2.9.5.jar -jackson-jaxrs-json-provider-2.9.5.jar +jackson-jaxrs-base-2.9.8.jar +jackson-jaxrs-json-provider-2.9.8.jar jackson-mapper-asl-1.9.13.jar jackson-module-jaxb-annotations-2.10.0.jar jackson-module-paranamer-2.10.0.jar diff --git a/pom.xml b/pom.xml index e72fcd90a38d..eb6d9c2c6346 100644 --- a/pom.xml +++ b/pom.xml @@ -2938,7 +2938,7 @@ hadoop-3.2 - 3.2.0 + 3.2.1 2.13.0 From 7ac9e3eed6a29dcbab3a4c2de41a94a897739dc0 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 27 Dec 2019 12:30:29 +0900 Subject: [PATCH 2/3] Guava 0.27-jre --- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 8 +++++++- pom.xml | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 405ffd2bf1b7..d1075363dfa9 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -7,6 +7,7 @@ accessors-smart-1.2.jar activation-1.1.1.jar aircompressor-0.10.jar algebra_2.12-2.0.0-M2.jar +animal-sniffer-annotations-1.17.jar antlr-runtime-3.5.2.jar antlr4-runtime-4.7.1.jar aopalliance-1.0.jar @@ -26,6 +27,7 @@ bonecp-0.8.0.RELEASE.jar breeze-macros_2.12-1.0.jar breeze_2.12-1.0.jar cats-kernel_2.12-2.0.0-M4.jar +checker-qual-2.5.2.jar chill-java-0.9.3.jar chill_2.12-0.9.3.jar commons-beanutils-1.9.4.jar @@ -59,11 +61,13 @@ derby-10.12.1.1.jar dnsjava-2.1.7.jar dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar ehcache-3.3.1.jar +error_prone_annotations-2.2.0.jar +failureaccess-1.0.jar flatbuffers-java-1.9.0.jar generex-1.0.2.jar geronimo-jcache_1.0_spec-1.0-alpha-1.jar gson-2.2.4.jar -guava-14.0.1.jar +guava-27.0-jre.jar guice-4.0.jar guice-servlet-4.0.jar hadoop-annotations-3.2.1.jar @@ -103,6 +107,7 @@ httpclient-4.5.6.jar httpcore-4.4.12.jar istack-commons-runtime-3.0.8.jar ivy-2.4.0.jar +j2objc-annotations-1.1.jar jackson-annotations-2.10.0.jar jackson-core-2.10.0.jar jackson-core-asl-1.9.13.jar @@ -173,6 +178,7 @@ kubernetes-model-common-4.6.4.jar leveldbjni-all-1.8.jar libfb303-0.9.3.jar libthrift-0.12.0.jar +listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j-1.2.17.jar logging-interceptor-3.12.6.jar lz4-java-1.7.0.jar diff --git a/pom.xml b/pom.xml index eb6d9c2c6346..a44e2bbf378c 100644 --- a/pom.xml +++ b/pom.xml @@ -199,6 +199,7 @@ and ./python/setup.py too. --> 0.15.1 + 14.0.1 org.fusesource.leveldbjni @@ -411,7 +412,7 @@ com.google.guava guava - 14.0.1 + ${guava.version} provided @@ -2940,6 +2941,7 @@ 3.2.1 2.13.0 + 27.0-jre From 3224e9079a0c5e08af7d58ae40364e79328f5a36 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 27 Dec 2019 19:12:21 +0900 Subject: [PATCH 3/3] try --- .../org/apache/spark/sql/hive/client/IsolatedClientLoader.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index 5da7b70cfc7a..2ca49cc70839 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -192,7 +192,6 @@ private[hive] class IsolatedClientLoader( name.startsWith("org.apache.spark.") || (sharesHadoopClasses && isHadoopClass) || name.startsWith("scala.") || - (name.startsWith("com.google") && !name.startsWith("com.google.cloud")) || name.startsWith("java.") || name.startsWith("javax.sql.") || sharedPrefixes.exists(name.startsWith)