Hadoop 2.9.0-palantir.5 (apache-spark-on-k8s#303)

Author: sjrand

core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala

@@ -962,6 +962,8 @@ class SparkSubmitSuite
     if (supportMockHttpFs) {
       hadoopConf.set("fs.http.impl", classOf[TestFileSystem].getCanonicalName)
       hadoopConf.set("fs.http.impl.disable.cache", "true")
+    } else {
+      hadoopConf.set("fs.http.impl", "invalidClassName")
     }
 
     val tmpDir = Utils.createTempDir()

dev/deps/spark-deps-hadoop-palantir

@@ -1,3 +1,4 @@
+HikariCP-java7-2.4.12.jar
 JavaEWAH-0.3.2.jar
 RoaringBitmap-0.6.43.jar
 ST4-4.0.4.jar
@@ -21,9 +22,7 @@ automaton-1.11-8.jar
 avro-1.8.1.jar
 avro-ipc-1.8.1.jar
 avro-mapred-1.8.1-hadoop2.jar
-aws-java-sdk-core-1.11.45.jar
-aws-java-sdk-kms-1.11.45.jar
-aws-java-sdk-s3-1.11.45.jar
+aws-java-sdk-bundle-1.11.201.jar
 base64-2.3.8.jar
 bcpkix-jdk15on-1.58.jar
 bcprov-jdk15on-1.58.jar
@@ -72,40 +71,41 @@ dropwizard-metrics-0.9.2.jar
 dropwizard-metrics-influxdb-1.1.8.jar
 dropwizard-util-0.9.2.jar
 dropwizard-validation-0.9.2.jar
+ehcache-3.3.1.jar
 eigenbase-properties-1.1.5.jar
 fastutil-7.0.13.jar
 flatbuffers-1.2.0-3f79e055.jar
 generex-1.0.1.jar
+geronimo-jcache_1.0_spec-1.0-alpha-1.jar
 gson-2.2.4.jar
 guava-14.0.1.jar
 guice-3.0.jar
 guice-servlet-3.0.jar
-hadoop-annotations-2.8.2-palantir.2.jar
-hadoop-auth-2.8.2-palantir.2.jar
-hadoop-aws-2.8.2-palantir.2.jar
-hadoop-client-2.8.2-palantir.2.jar
-hadoop-common-2.8.2-palantir.2.jar
-hadoop-hdfs-client-2.8.2-palantir.2.jar
-hadoop-mapreduce-client-app-2.8.2-palantir.2.jar
-hadoop-mapreduce-client-common-2.8.2-palantir.2.jar
-hadoop-mapreduce-client-core-2.8.2-palantir.2.jar
-hadoop-mapreduce-client-jobclient-2.8.2-palantir.2.jar
-hadoop-mapreduce-client-shuffle-2.8.2-palantir.2.jar
-hadoop-openstack-2.8.2-palantir.2.jar
-hadoop-yarn-api-2.8.2-palantir.2.jar
-hadoop-yarn-client-2.8.2-palantir.2.jar
-hadoop-yarn-common-2.8.2-palantir.2.jar
-hadoop-yarn-server-common-2.8.2-palantir.2.jar
-hadoop-yarn-server-web-proxy-2.8.2-palantir.2.jar
+hadoop-annotations-2.9.0-palantir.5.jar
+hadoop-auth-2.9.0-palantir.5.jar
+hadoop-aws-2.9.0-palantir.5.jar
+hadoop-client-2.9.0-palantir.5.jar
+hadoop-common-2.9.0-palantir.5.jar
+hadoop-hdfs-client-2.9.0-palantir.5.jar
+hadoop-mapreduce-client-app-2.9.0-palantir.5.jar
+hadoop-mapreduce-client-common-2.9.0-palantir.5.jar
+hadoop-mapreduce-client-core-2.9.0-palantir.5.jar
+hadoop-mapreduce-client-jobclient-2.9.0-palantir.5.jar
+hadoop-mapreduce-client-shuffle-2.9.0-palantir.5.jar
+hadoop-openstack-2.9.0-palantir.5.jar
+hadoop-yarn-api-2.9.0-palantir.5.jar
+hadoop-yarn-client-2.9.0-palantir.5.jar
+hadoop-yarn-common-2.9.0-palantir.5.jar
+hadoop-yarn-server-common-2.9.0-palantir.5.jar
+hadoop-yarn-server-web-proxy-2.9.0-palantir.5.jar
 hibernate-validator-5.2.2.Final.jar
 hk2-api-2.5.0-b32.jar
 hk2-locator-2.5.0-b32.jar
 hk2-utils-2.5.0-b32.jar
 hppc-0.7.2.jar
-htrace-core4-4.0.1-incubating.jar
+htrace-core4-4.1.0-incubating.jar
 httpclient-4.5.4.jar
 httpcore-4.4.8.jar
-ion-java-1.0.1.jar
 ivy-2.4.0.jar
 jackson-annotations-2.6.7.jar
 jackson-core-2.6.7.jar
@@ -152,7 +152,6 @@ jets3t-0.9.4.jar
 jetty-6.1.26.jar
 jetty-sslengine-6.1.26.jar
 jetty-util-6.1.26.jar
-jmespath-java-1.0.jar
 joda-time-2.9.9.jar
 jodd-core-3.5.2.jar
 json-smart-1.1.1.jar
@@ -183,6 +182,7 @@ metrics-json-3.2.5.jar
 metrics-jvm-3.2.5.jar
 mimepull-1.9.6.jar
 minlog-1.3.0.jar
+mssql-jdbc-6.2.1.jre7.jar
 netty-3.10.6.Final.jar
 netty-all-4.1.17.Final.jar
 nimbus-jose-jwt-3.9.jar
@@ -222,10 +222,12 @@ spire-macros_2.11-0.13.0.jar
 spire_2.11-0.13.0.jar
 stax-api-1.0-2.jar
 stax-api-1.0.1.jar
+stax2-api-3.1.4.jar
 stream-2.7.0.jar
 stringtemplate-3.2.1.jar
 univocity-parsers-2.5.9.jar
 validation-api-1.1.0.Final.jar
+woodstox-core-5.0.3.jar
 xbean-asm5-shaded-4.4.jar
 xmlenc-0.52.jar
 xz-1.5.jar

dists/hadoop-palantir/pom.xml

@@ -34,7 +34,7 @@
     <sbt.project.name>spark-dist-hadoop-palantir</sbt.project.name>
     <build.testJarPhase>none</build.testJarPhase>
     <build.copyDependenciesPhase>none</build.copyDependenciesPhase>
-    <hadoop.version>2.8.2-palantir.2</hadoop.version>
+    <hadoop.version>2.9.0-palantir.5</hadoop.version>
     <curator.version>2.7.1</curator.version>
   </properties>
 

pom.xml

@@ -2967,7 +2967,7 @@
     <profile>
       <id>hadoop-palantir</id>
       <properties>
-        <hadoop.version>2.8.2-palantir.2</hadoop.version>
+        <hadoop.version>2.9.0-palantir.5</hadoop.version>
         <curator.version>2.7.1</curator.version>
       </properties>
       <dependencyManagement>