Upgrade guava and hadoop.

Author: viirya

dev/deps/spark-deps-hadoop-3.2-hive-2.3

@@ -7,6 +7,7 @@ accessors-smart/1.2//accessors-smart-1.2.jar
 activation/1.1.1//activation-1.1.1.jar
 aircompressor/0.10//aircompressor-0.10.jar
 algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar
+animal-sniffer-annotations/1.17//animal-sniffer-annotations-1.17.jar
 antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
 antlr4-runtime/4.7.1//antlr4-runtime-4.7.1.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
@@ -20,10 +21,13 @@ automaton/1.11-8//automaton-1.11-8.jar
 avro-ipc/1.8.2//avro-ipc-1.8.2.jar
 avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
 avro/1.8.2//avro-1.8.2.jar
+bcpkix-jdk15on/1.60//bcpkix-jdk15on-1.60.jar
+bcprov-jdk15on/1.60//bcprov-jdk15on-1.60.jar
 bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
 breeze_2.12/1.0//breeze_2.12-1.0.jar
 cats-kernel_2.12/2.0.0-M4//cats-kernel_2.12-2.0.0-M4.jar
+checker-qual/2.5.2//checker-qual-2.5.2.jar
 chill-java/0.9.5//chill-java-0.9.5.jar
 chill_2.12/0.9.5//chill_2.12-0.9.5.jar
 commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
@@ -57,27 +61,29 @@ derby/10.12.1.1//derby-10.12.1.1.jar
 dnsjava/2.1.7//dnsjava-2.1.7.jar
 dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
 ehcache/3.3.1//ehcache-3.3.1.jar
+error_prone_annotations/2.2.0//error_prone_annotations-2.2.0.jar
+failureaccess/1.0//failureaccess-1.0.jar
 flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
 generex/1.0.2//generex-1.0.2.jar
 geronimo-jcache_1.0_spec/1.0-alpha-1//geronimo-jcache_1.0_spec-1.0-alpha-1.jar
 gson/2.2.4//gson-2.2.4.jar
-guava/14.0.1//guava-14.0.1.jar
+guava/27.0-jre//guava-27.0-jre.jar
 guice-servlet/4.0//guice-servlet-4.0.jar
 guice/4.0//guice-4.0.jar
-hadoop-annotations/3.2.0//hadoop-annotations-3.2.0.jar
-hadoop-auth/3.2.0//hadoop-auth-3.2.0.jar
-hadoop-client/3.2.0//hadoop-client-3.2.0.jar
-hadoop-common/3.2.0//hadoop-common-3.2.0.jar
-hadoop-hdfs-client/3.2.0//hadoop-hdfs-client-3.2.0.jar
-hadoop-mapreduce-client-common/3.2.0//hadoop-mapreduce-client-common-3.2.0.jar
-hadoop-mapreduce-client-core/3.2.0//hadoop-mapreduce-client-core-3.2.0.jar
-hadoop-mapreduce-client-jobclient/3.2.0//hadoop-mapreduce-client-jobclient-3.2.0.jar
-hadoop-yarn-api/3.2.0//hadoop-yarn-api-3.2.0.jar
-hadoop-yarn-client/3.2.0//hadoop-yarn-client-3.2.0.jar
-hadoop-yarn-common/3.2.0//hadoop-yarn-common-3.2.0.jar
-hadoop-yarn-registry/3.2.0//hadoop-yarn-registry-3.2.0.jar
-hadoop-yarn-server-common/3.2.0//hadoop-yarn-server-common-3.2.0.jar
-hadoop-yarn-server-web-proxy/3.2.0//hadoop-yarn-server-web-proxy-3.2.0.jar
+hadoop-annotations/3.2.1//hadoop-annotations-3.2.1.jar
+hadoop-auth/3.2.1//hadoop-auth-3.2.1.jar
+hadoop-client/3.2.1//hadoop-client-3.2.1.jar
+hadoop-common/3.2.1//hadoop-common-3.2.1.jar
+hadoop-hdfs-client/3.2.1//hadoop-hdfs-client-3.2.1.jar
+hadoop-mapreduce-client-common/3.2.1//hadoop-mapreduce-client-common-3.2.1.jar
+hadoop-mapreduce-client-core/3.2.1//hadoop-mapreduce-client-core-3.2.1.jar
+hadoop-mapreduce-client-jobclient/3.2.1//hadoop-mapreduce-client-jobclient-3.2.1.jar
+hadoop-yarn-api/3.2.1//hadoop-yarn-api-3.2.1.jar
+hadoop-yarn-client/3.2.1//hadoop-yarn-client-3.2.1.jar
+hadoop-yarn-common/3.2.1//hadoop-yarn-common-3.2.1.jar
+hadoop-yarn-registry/3.2.1//hadoop-yarn-registry-3.2.1.jar
+hadoop-yarn-server-common/3.2.1//hadoop-yarn-server-common-3.2.1.jar
+hadoop-yarn-server-web-proxy/3.2.1//hadoop-yarn-server-web-proxy-3.2.1.jar
 hive-beeline/2.3.7//hive-beeline-2.3.7.jar
 hive-cli/2.3.7//hive-cli-2.3.7.jar
 hive-common/2.3.7//hive-common-2.3.7.jar
@@ -100,14 +106,15 @@ httpclient/4.5.6//httpclient-4.5.6.jar
 httpcore/4.4.12//httpcore-4.4.12.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
 ivy/2.4.0//ivy-2.4.0.jar
+j2objc-annotations/1.1//j2objc-annotations-1.1.jar
 jackson-annotations/2.10.0//jackson-annotations-2.10.0.jar
 jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.10.0//jackson-core-2.10.0.jar
 jackson-databind/2.10.0//jackson-databind-2.10.0.jar
 jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
 jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar
-jackson-jaxrs-base/2.9.5//jackson-jaxrs-base-2.9.5.jar
-jackson-jaxrs-json-provider/2.9.5//jackson-jaxrs-json-provider-2.9.5.jar
+jackson-jaxrs-base/2.9.8//jackson-jaxrs-base-2.9.8.jar
+jackson-jaxrs-json-provider/2.9.8//jackson-jaxrs-json-provider-2.9.8.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
 jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar
@@ -171,6 +178,7 @@ kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
+listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
 log4j/1.2.17//log4j-1.2.17.jar
 logging-interceptor/3.12.6//logging-interceptor-3.12.6.jar
 lz4-java/1.7.1//lz4-java-1.7.1.jar

pom.xml

@@ -120,7 +120,7 @@
     <sbt.project.name>spark</sbt.project.name>
     <slf4j.version>1.7.30</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
-    <hadoop.version>3.2.0</hadoop.version>
+    <hadoop.version>3.2.1</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>
     <yarn.version>${hadoop.version}</yarn.version>
     <zookeeper.version>3.4.14</zookeeper.version>
@@ -180,7 +180,7 @@
     <!-- org.apache.commons/commons-pool2/-->
     <commons-pool2.version>2.6.2</commons-pool2.version>
     <datanucleus-core.version>4.1.17</datanucleus-core.version>
-    <guava.version>14.0.1</guava.version>
+    <guava.version>27.0-jre</guava.version>
     <janino.version>3.1.2</janino.version>
     <jersey.version>2.30</jersey.version>
     <joda.version>2.10.5</joda.version>
@@ -3129,6 +3129,7 @@
         <hadoop.version>2.7.4</hadoop.version>
         <curator.version>2.7.1</curator.version>
         <commons-io.version>2.4</commons-io.version>
+        <guava.version>14.0.1</guava.version>
       </properties>
     </profile>
 

project/SparkBuild.scala

@@ -626,7 +626,7 @@ object KubernetesIntegrationTests {
  * Overrides to work around sbt's dependency resolution being different from Maven's.
  */
 object DependencyOverrides {
-  lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1")
+  lazy val guavaVersion = sys.props.get("guava.version").getOrElse("27.0-jre")
   lazy val settings = Seq(
     dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
     dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",