diff --git a/LICENSE-binary b/LICENSE-binary index 32c7fc315d1a3..dfbb58736c07d 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -316,10 +316,14 @@ com.fasterxml.jackson.module:jackson-module-scala_2.12 com.github.mifmif:generex com.google.code.findbugs:jsr305 com.google.code.gson:gson +com.google.errorprone:error_prone_annotations com.google.flatbuffers:flatbuffers-java +com.google.guava:failureaccess com.google.guava:guava +com.google.guava:listenablefuture com.google.inject:guice com.google.inject.extensions:guice-servlet +com.google.j2objc:j2objc-annotations com.twitter:parquet-hadoop-bundle commons-cli:commons-cli commons-dbcp:commons-dbcp @@ -471,6 +475,7 @@ MIT License ----------- com.microsoft.sqlserver:mssql-jdbc +org.checkerframework:checker-qual org.typelevel:spire_2.12 org.typelevel:spire-macros_2.12 org.typelevel:spire-platform_2.12 diff --git a/assembly/pom.xml b/assembly/pom.xml index 74c2f44121fca..e3119cf56f341 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -78,7 +78,11 @@ com.google.guava @@ -165,6 +169,13 @@ hive + + + org.apache.spark + spark-hive-shaded_${scala.binary.version} + ${project.version} + ${hive.deps.scope} + org.apache.spark spark-hive_${scala.binary.version} @@ -174,7 +185,14 @@ hive-thriftserver + + + org.apache.spark + spark-hive-shaded_${scala.binary.version} + ${project.version} + ${hive.deps.scope} + org.apache.spark spark-hive-thriftserver_${scala.binary.version} diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index d8f3e2b36403a..cdc394a893ff7 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -1,13 +1,10 @@ -HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar -ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar annotations/17.0.0//annotations-17.0.0.jar -antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar antlr4-runtime/4.8//antlr4-runtime-4.8.jar aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar aopalliance/1.0//aopalliance-1.0.jar @@ -27,10 +24,10 @@ avro-ipc/1.10.2//avro-ipc-1.10.2.jar avro-mapred/1.10.2//avro-mapred-1.10.2.jar avro/1.10.2//avro-1.10.2.jar blas/2.2.0//blas-2.2.0.jar -bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar breeze_2.12/1.2//breeze_2.12-1.2.jar cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar +checker-qual/3.8.0//checker-qual-3.8.0.jar chill-java/0.10.0//chill-java-0.10.0.jar chill_2.12/0.10.0//chill_2.12-0.10.0.jar commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar @@ -41,7 +38,6 @@ commons-compiler/3.0.16//commons-compiler-3.0.16.jar commons-compress/1.21//commons-compress-1.21.jar commons-configuration/1.6//commons-configuration-1.6.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar -commons-dbcp/1.4//commons-dbcp-1.4.jar commons-digester/1.8//commons-digester-1.8.jar commons-httpclient/3.1//commons-httpclient-3.1.jar commons-io/2.4//commons-io-2.4.jar @@ -50,7 +46,6 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar commons-logging/1.1.3//commons-logging-1.1.3.jar commons-math3/3.4.1//commons-math3-3.4.1.jar commons-net/3.1//commons-net-3.1.jar -commons-pool/1.5.4//commons-pool-1.5.4.jar commons-text/1.6//commons-text-1.6.jar compress-lzf/1.0.3//compress-lzf-1.0.3.jar core/1.1.2//core-1.1.2.jar @@ -61,11 +56,12 @@ datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar derby/10.14.2.0//derby-10.14.2.0.jar -dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar +error_prone_annotations/2.5.1//error_prone_annotations-2.5.1.jar +failureaccess/1.0.1//failureaccess-1.0.1.jar flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar generex/1.0.2//generex-1.0.2.jar gson/2.2.4//gson-2.2.4.jar -guava/14.0.1//guava-14.0.1.jar +guava/30.1.1-jre//guava-30.1.1-jre.jar guice-servlet/3.0//guice-servlet-3.0.jar guice/3.0//guice-3.0.jar hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar @@ -83,21 +79,7 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar -hive-beeline/2.3.9//hive-beeline-2.3.9.jar -hive-cli/2.3.9//hive-cli-2.3.9.jar -hive-common/2.3.9//hive-common-2.3.9.jar -hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar -hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar -hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar -hive-metastore/2.3.9//hive-metastore-2.3.9.jar -hive-serde/2.3.9//hive-serde-2.3.9.jar -hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar -hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar -hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar -hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar -hive-shims/2.3.9//hive-shims-2.3.9.jar hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar -hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar @@ -106,6 +88,7 @@ httpclient/4.5.13//httpclient-4.5.13.jar httpcore/4.4.14//httpcore-4.4.14.jar istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar ivy/2.5.0//ivy-2.5.0.jar +j2objc-annotations/1.3//j2objc-annotations-1.3.jar jackson-annotations/2.12.5//jackson-annotations-2.12.5.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar jackson-core/2.12.5//jackson-core-2.12.5.jar @@ -126,7 +109,6 @@ janino/3.0.16//janino-3.0.16.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.inject/1//javax.inject-1.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar -javolution/5.5.1//javolution-5.5.1.jar jaxb-api/2.2.11//jaxb-api-2.2.11.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar @@ -144,7 +126,6 @@ jline/2.14.6//jline-2.14.6.jar joda-time/2.10.10//joda-time-2.10.10.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar -json/1.8//json-1.8.jar json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar @@ -179,6 +160,7 @@ lapack/2.2.0//lapack-2.2.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.12.0//libthrift-0.12.0.jar +listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.8.0//lz4-java-1.8.0.jar @@ -229,12 +211,10 @@ spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar spire_2.12/0.17.0//spire_2.12-0.17.0.jar stax-api/1.0.1//stax-api-1.0.1.jar stream/2.9.6//stream-2.9.6.jar -super-csv/2.2.0//super-csv-2.2.0.jar threeten-extra/1.5.0//threeten-extra-1.5.0.jar tink/1.6.0//tink-1.6.0.jar transaction-api/1.1//transaction-api-1.1.jar univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar -velocity/1.5//velocity-1.5.jar xbean-asm9-shaded/4.20//xbean-asm9-shaded-4.20.jar xercesImpl/2.12.0//xercesImpl-2.12.0.jar xml-apis/1.4.01//xml-apis-1.4.01.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 41c8a24c8fff2..76b6ff52f42ce 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -1,13 +1,10 @@ -HikariCP/2.5.1//HikariCP-2.5.1.jar JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar -ST4/4.0.4//ST4-4.0.4.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.21//aircompressor-0.21.jar algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar annotations/17.0.0//annotations-17.0.0.jar -antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar antlr4-runtime/4.8//antlr4-runtime-4.8.jar aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar arpack/2.2.0//arpack-2.2.0.jar @@ -22,10 +19,10 @@ avro-ipc/1.10.2//avro-ipc-1.10.2.jar avro-mapred/1.10.2//avro-mapred-1.10.2.jar avro/1.10.2//avro-1.10.2.jar blas/2.2.0//blas-2.2.0.jar -bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar breeze_2.12/1.2//breeze_2.12-1.2.jar cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar +checker-qual/3.8.0//checker-qual-3.8.0.jar chill-java/0.10.0//chill-java-0.10.0.jar chill_2.12/0.10.0//chill_2.12-0.10.0.jar commons-cli/1.2//commons-cli-1.2.jar @@ -34,14 +31,12 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar commons-compiler/3.0.16//commons-compiler-3.0.16.jar commons-compress/1.21//commons-compress-1.21.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar -commons-dbcp/1.4//commons-dbcp-1.4.jar commons-io/2.11.0//commons-io-2.11.0.jar commons-lang/2.6//commons-lang-2.6.jar commons-lang3/3.12.0//commons-lang3-3.12.0.jar commons-logging/1.1.3//commons-logging-1.1.3.jar commons-math3/3.4.1//commons-math3-3.4.1.jar commons-net/3.1//commons-net-3.1.jar -commons-pool/1.5.4//commons-pool-1.5.4.jar commons-text/1.6//commons-text-1.6.jar compress-lzf/1.0.3//compress-lzf-1.0.3.jar core/1.1.2//core-1.1.2.jar @@ -52,30 +47,17 @@ datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar derby/10.14.2.0//derby-10.14.2.0.jar -dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar +error_prone_annotations/2.5.1//error_prone_annotations-2.5.1.jar +failureaccess/1.0.1//failureaccess-1.0.1.jar flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar generex/1.0.2//generex-1.0.2.jar -gson/2.2.4//gson-2.2.4.jar -guava/14.0.1//guava-14.0.1.jar +gson/2.8.6//gson-2.8.6.jar +guava/30.1.1-jre//guava-30.1.1-jre.jar hadoop-client-api/3.3.1//hadoop-client-api-3.3.1.jar hadoop-client-runtime/3.3.1//hadoop-client-runtime-3.3.1.jar hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar hadoop-yarn-server-web-proxy/3.3.1//hadoop-yarn-server-web-proxy-3.3.1.jar -hive-beeline/2.3.9//hive-beeline-2.3.9.jar -hive-cli/2.3.9//hive-cli-2.3.9.jar -hive-common/2.3.9//hive-common-2.3.9.jar -hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar -hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar -hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar -hive-metastore/2.3.9//hive-metastore-2.3.9.jar -hive-serde/2.3.9//hive-serde-2.3.9.jar -hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar -hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar -hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar -hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar -hive-shims/2.3.9//hive-shims-2.3.9.jar hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar -hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar @@ -84,6 +66,7 @@ httpclient/4.5.13//httpclient-4.5.13.jar httpcore/4.4.14//httpcore-4.4.14.jar istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar ivy/2.5.0//ivy-2.5.0.jar +j2objc-annotations/1.3//j2objc-annotations-1.3.jar jackson-annotations/2.12.5//jackson-annotations-2.12.5.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar jackson-core/2.12.5//jackson-core-2.12.5.jar @@ -101,7 +84,6 @@ jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar janino/3.0.16//janino-3.0.16.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar -javolution/5.5.1//javolution-5.5.1.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar jdo-api/3.0.1//jdo-api-3.0.1.jar @@ -115,7 +97,6 @@ jline/2.14.6//jline-2.14.6.jar joda-time/2.10.10//joda-time-2.10.10.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar -json/1.8//json-1.8.jar json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar @@ -149,6 +130,7 @@ lapack/2.2.0//lapack-2.2.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.12.0//libthrift-0.12.0.jar +listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.8.0//lz4-java-1.8.0.jar @@ -199,12 +181,10 @@ spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar spire_2.12/0.17.0//spire_2.12-0.17.0.jar stax-api/1.0.1//stax-api-1.0.1.jar stream/2.9.6//stream-2.9.6.jar -super-csv/2.2.0//super-csv-2.2.0.jar threeten-extra/1.5.0//threeten-extra-1.5.0.jar tink/1.6.0//tink-1.6.0.jar transaction-api/1.1//transaction-api-1.1.jar univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar -velocity/1.5//velocity-1.5.jar xbean-asm9-shaded/4.20//xbean-asm9-shaded-4.20.jar xz/1.8//xz-1.8.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar diff --git a/pom.xml b/pom.xml index 2cece056b1c8c..b835ab9295d31 100644 --- a/pom.xml +++ b/pom.xml @@ -96,6 +96,7 @@ streaming sql/catalyst sql/core + sql/hive-shaded sql/hive assembly examples @@ -181,7 +182,14 @@ 2.11.1 4.1.17 - 14.0.1 + 4.2.4 + 4.1.19 + 3.2.0-m3 + 3.0.1 + 1.0.1 + + 30.1.1-jre + 14.0.1 3.0.16 2.34 2.10.10 @@ -197,6 +205,7 @@ 1.2 1.60 1.6.0 + 2.14.6 + + + org.apache.commons + commons-lang3 + + + joda-time + joda-time + + + io.dropwizard.metrics + metrics-core + + + io.dropwizard.metrics + metrics-jvm + + + io.dropwizard.metrics + metrics-json + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + jline + jline + @@ -1932,6 +1990,27 @@ org.pentaho pentaho-aggdesigner-algorithm + + + org.datanucleus + datanucleus-core + + + commons-io + commons-io + + + oro + oro + + + org.apache.ivy + ivy + + + stax + stax-api + @@ -2000,6 +2079,15 @@ org.codehaus.groovy groovy-all + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + @@ -2052,6 +2140,35 @@ * + + + org.apache.derby + derby + + + commons-pool + commons-pool + + + org.datanucleus + datanucleus-core + + + org.datanucleus + datanucleus-api-jdo + + + org.datanucleus + datanucleus-rdbms + + + org.datanucleus + javax.jdo + + + javax.jdo + jdo-api + @@ -2235,6 +2352,10 @@ org.slf4j slf4j-api + + org.apache.commons + commons-lang3 + @@ -2242,7 +2363,6 @@ org.apache.hive hive-llap-client ${hive23.version} - test org.apache.hive @@ -2268,6 +2388,10 @@ org.apache.zookeeper zookeeper + + org.apache.commons + commons-lang3 + org.slf4j slf4j-api @@ -2376,6 +2500,31 @@ datanucleus-core ${datanucleus-core.version} + + org.datanucleus + datanucleus-api-jdo + ${datanucleus-api-jdo.version} + + + org.datanucleus + datanucleus-rdbms + ${datanucleus-rdbms.version} + + + org.datanucleus + javax.jdo + ${datanucleus-javax.jdo-version} + + + javax.jdo + jdo-api + ${javax.jdo.version} + + + stax + stax-api + ${stax-api.version} + org.apache.thrift libthrift diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index b1531a601ca08..528a61ff9f270 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -43,8 +43,8 @@ object BuildCommons { private val buildLocation = file(".").getAbsoluteFile.getParentFile - val sqlProjects@Seq(catalyst, sql, hive, hiveThriftServer, tokenProviderKafka010, sqlKafka010, avro) = Seq( - "catalyst", "sql", "hive", "hive-thriftserver", "token-provider-kafka-0-10", "sql-kafka-0-10", "avro" + val sqlProjects@Seq(catalyst, sql, hiveShaded, hive, hiveThriftServer, tokenProviderKafka010, sqlKafka010, avro) = Seq( + "catalyst", "sql", "hive-shaded", "hive", "hive-thriftserver", "token-provider-kafka-0-10", "sql-kafka-0-10", "avro" ).map(ProjectRef(buildLocation, _)) val streamingProjects@Seq(streaming, streamingKafka010) = @@ -366,8 +366,8 @@ object SparkBuild extends PomBuild { val mimaProjects = allProjects.filterNot { x => Seq( - spark, hive, hiveThriftServer, catalyst, repl, networkCommon, networkShuffle, networkYarn, - unsafe, tags, tokenProviderKafka010, sqlKafka010, kvstore, avro + spark, hiveShaded, hive, hiveThriftServer, catalyst, repl, networkCommon, networkShuffle, + networkYarn, unsafe, tags, tokenProviderKafka010, sqlKafka010, kvstore, avro ).contains(x) } @@ -667,7 +667,7 @@ object KubernetesIntegrationTests { * Overrides to work around sbt's dependency resolution being different from Maven's. */ object DependencyOverrides { - lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1") + lazy val guavaVersion = sys.props.get("guava.version").getOrElse("30.1.1-jre") lazy val settings = Seq( dependencyOverrides += "com.google.guava" % "guava" % guavaVersion, dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0", @@ -990,10 +990,10 @@ object Unidoc { (ScalaUnidoc / unidoc / unidocProjectFilter) := inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes, - yarn, tags, streamingKafka010, sqlKafka010), + yarn, tags, streamingKafka010, sqlKafka010, hiveShaded), (JavaUnidoc / unidoc / unidocProjectFilter) := inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes, - yarn, tags, streamingKafka010, sqlKafka010), + yarn, tags, streamingKafka010, sqlKafka010, hiveShaded), (ScalaUnidoc / unidoc / unidocAllClasspaths) := { ignoreClasspaths((ScalaUnidoc / unidoc / unidocAllClasspaths).value) diff --git a/sql/hive-shaded/pom.xml b/sql/hive-shaded/pom.xml new file mode 100644 index 0000000000000..10be1ea95d113 --- /dev/null +++ b/sql/hive-shaded/pom.xml @@ -0,0 +1,238 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.12 + 3.3.0-SNAPSHOT + ../../pom.xml + + + spark-hive-shaded_2.12 + jar + Spark Project Hive Shaded + http://spark.apache.org/ + + hive-shaded + + + + + ${hive.group} + hive-common + ${hive.common.scope} + + + ${hive.group} + hive-exec + ${hive.classifier} + + + ${hive.group} + hive-metastore + + + ${hive.group} + hive-serde + ${hive.serde.scope} + + + ${hive.group} + hive-shims + ${hive.shims.scope} + + + org.apache.hive + hive-llap-common + ${hive.llap.scope} + + + org.apache.hive + hive-llap-client + ${hive.llap.scope} + + + + com.google.guava + guava + ${hive.guava.version} + compile + + + + + + hive-thriftserver + + + ${hive.group} + hive-cli + + + ${hive.group} + hive-jdbc + + + ${hive.group} + hive-beeline + + + ${hive.group} + hive-service-rpc + + + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + true + false + + + *:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + parquet-logging.properties + hive-log4j2.properties + hive-exec-log4j2.properties + beeline-log4j2.properties + tez-container-log4j2.properties + bonecp-default-config.xml + + + + + + com.google.common + ${spark.shade.packageName}.hive.guava + + + com.google.protobuf + ${spark.shade.packageName}.hive.com.google.protobuf + + + com.google.gson + ${spark.shade.packageName}.hive.com.google.gson + + + com.github.joshelser + ${spark.shade.packageName}.hive.com.github.joshelser + + + com.jolbox.bonecp + ${spark.shade.packageName}.hive.com.jolbox.bonecp + + + com.zaxxer.hikari + ${spark.shade.packageName}.hive.com.zaxxer.hikari + + + au.com.bytecode + ${spark.shade.packageName}.hive.au.com.bytecode + + + org.apache.commons + ${spark.shade.packageName}.hive.org.apache.commons + + + org.antlr.runtime + ${spark.shade.packageName}.hive.org.antlr.runtime + + + org.apache.velocity + ${spark.shade.packageName}.hive.org.apache.velocity + + + org.stringtemplate.v4 + ${spark.shade.packageName}.hive.org.stringtemplate.v4 + + + javolution + ${spark.shade.packageName}.hive.javolution + + + org.supercsv + ${spark.shade.packageName}.hive.org.supercsv + + + + + + reference.conf + + + log4j.properties + + + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + compile + package + + attach-artifact + + + + + ${basedir}/target/${project.artifactId}-${project.version}.jar + jar + optional + + + + + + + + + diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 764712fcbb482..f90e2f7f57f84 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -40,6 +40,11 @@ spark-hive_${scala.binary.version} ${project.version} + + org.apache.spark + spark-hive-shaded_${scala.binary.version} + ${project.version} + org.apache.spark spark-core_${scala.binary.version} @@ -71,22 +76,6 @@ com.google.guava guava - - ${hive.group} - hive-cli - - - ${hive.group} - hive-jdbc - - - ${hive.group} - hive-beeline - - - ${hive.group} - hive-service-rpc - org.eclipse.jetty jetty-server @@ -97,6 +86,10 @@ jetty-servlet provided + + org.apache.httpcomponents + httpcore + org.seleniumhq.selenium diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 86cd1eddde330..c72135d56ee96 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -40,6 +40,11 @@ spark-core_${scala.binary.version} ${project.version} + + org.apache.spark + spark-hive-shaded_${scala.binary.version} + ${project.version} + org.apache.spark spark-core_${scala.binary.version} @@ -85,51 +90,6 @@ scala-parallel-collections_${scala.binary.version} --> - - - ${hive.group} - hive-common - ${hive.common.scope} - - - ${hive.group} - hive-exec - ${hive.classifier} - - - ${hive.group} - hive-metastore - - - ${hive.group} - hive-serde - ${hive.serde.scope} - - - ${hive.group} - hive-shims - ${hive.shims.scope} - - - org.apache.hive - hive-llap-common - ${hive.llap.scope} - - - org.apache.hive - hive-llap-client - ${hive.llap.scope} - org.apache.avro @@ -168,6 +128,26 @@ org.datanucleus datanucleus-core + + org.datanucleus + datanucleus-api-jdo + + + org.datanucleus + datanucleus-rdbms + + + org.datanucleus + javax.jdo + + + javax.jdo + jdo-api + + + stax + stax-api + org.apache.hadoop ${hadoop-client-runtime.artifact} @@ -185,6 +165,10 @@ org.apache.derby derby + + jline + jline + org.scala-lang scala-compiler @@ -237,27 +221,6 @@ -da -Xmx4g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true - - org.apache.maven.plugins - maven-enforcer-plugin - - - enforce-versions - - enforce - - - - - - *:hive-cli - - - - - - - diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index 9aa6a09fd57af..c64af2dd29eb7 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -202,7 +202,6 @@ private[hive] class IsolatedClientLoader( name.startsWith("org.apache.spark.") || isHadoopClass || name.startsWith("scala.") || - (name.startsWith("com.google") && !name.startsWith("com.google.cloud")) || name.startsWith("java.") || name.startsWith("javax.sql.") || sharedPrefixes.exists(name.startsWith)