diff --git a/LICENSE-binary b/LICENSE-binary
index 32c7fc315d1a3..dfbb58736c07d 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -316,10 +316,14 @@ com.fasterxml.jackson.module:jackson-module-scala_2.12
com.github.mifmif:generex
com.google.code.findbugs:jsr305
com.google.code.gson:gson
+com.google.errorprone:error_prone_annotations
com.google.flatbuffers:flatbuffers-java
+com.google.guava:failureaccess
com.google.guava:guava
+com.google.guava:listenablefuture
com.google.inject:guice
com.google.inject.extensions:guice-servlet
+com.google.j2objc:j2objc-annotations
com.twitter:parquet-hadoop-bundle
commons-cli:commons-cli
commons-dbcp:commons-dbcp
@@ -471,6 +475,7 @@ MIT License
-----------
com.microsoft.sqlserver:mssql-jdbc
+org.checkerframework:checker-qual
org.typelevel:spire_2.12
org.typelevel:spire-macros_2.12
org.typelevel:spire-platform_2.12
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 74c2f44121fca..e3119cf56f341 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -78,7 +78,11 @@
com.google.guava
@@ -165,6 +169,13 @@
hive
+
+
+ org.apache.spark
+ spark-hive-shaded_${scala.binary.version}
+ ${project.version}
+ ${hive.deps.scope}
+
org.apache.spark
spark-hive_${scala.binary.version}
@@ -174,7 +185,14 @@
hive-thriftserver
+
+
+ org.apache.spark
+ spark-hive-shaded_${scala.binary.version}
+ ${project.version}
+ ${hive.deps.scope}
+
org.apache.spark
spark-hive-thriftserver_${scala.binary.version}
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index d8f3e2b36403a..cdc394a893ff7 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -1,13 +1,10 @@
-HikariCP/2.5.1//HikariCP-2.5.1.jar
JLargeArrays/1.5//JLargeArrays-1.5.jar
JTransforms/3.1//JTransforms-3.1.jar
RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar
-ST4/4.0.4//ST4-4.0.4.jar
activation/1.1.1//activation-1.1.1.jar
aircompressor/0.21//aircompressor-0.21.jar
algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar
annotations/17.0.0//annotations-17.0.0.jar
-antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
antlr4-runtime/4.8//antlr4-runtime-4.8.jar
aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
aopalliance/1.0//aopalliance-1.0.jar
@@ -27,10 +24,10 @@ avro-ipc/1.10.2//avro-ipc-1.10.2.jar
avro-mapred/1.10.2//avro-mapred-1.10.2.jar
avro/1.10.2//avro-1.10.2.jar
blas/2.2.0//blas-2.2.0.jar
-bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar
breeze_2.12/1.2//breeze_2.12-1.2.jar
cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar
+checker-qual/3.8.0//checker-qual-3.8.0.jar
chill-java/0.10.0//chill-java-0.10.0.jar
chill_2.12/0.10.0//chill_2.12-0.10.0.jar
commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
@@ -41,7 +38,6 @@ commons-compiler/3.0.16//commons-compiler-3.0.16.jar
commons-compress/1.21//commons-compress-1.21.jar
commons-configuration/1.6//commons-configuration-1.6.jar
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
-commons-dbcp/1.4//commons-dbcp-1.4.jar
commons-digester/1.8//commons-digester-1.8.jar
commons-httpclient/3.1//commons-httpclient-3.1.jar
commons-io/2.4//commons-io-2.4.jar
@@ -50,7 +46,6 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
commons-math3/3.4.1//commons-math3-3.4.1.jar
commons-net/3.1//commons-net-3.1.jar
-commons-pool/1.5.4//commons-pool-1.5.4.jar
commons-text/1.6//commons-text-1.6.jar
compress-lzf/1.0.3//compress-lzf-1.0.3.jar
core/1.1.2//core-1.1.2.jar
@@ -61,11 +56,12 @@ datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar
datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar
datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
derby/10.14.2.0//derby-10.14.2.0.jar
-dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
+error_prone_annotations/2.5.1//error_prone_annotations-2.5.1.jar
+failureaccess/1.0.1//failureaccess-1.0.1.jar
flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
generex/1.0.2//generex-1.0.2.jar
gson/2.2.4//gson-2.2.4.jar
-guava/14.0.1//guava-14.0.1.jar
+guava/30.1.1-jre//guava-30.1.1-jre.jar
guice-servlet/3.0//guice-servlet-3.0.jar
guice/3.0//guice-3.0.jar
hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar
@@ -83,21 +79,7 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
-hive-beeline/2.3.9//hive-beeline-2.3.9.jar
-hive-cli/2.3.9//hive-cli-2.3.9.jar
-hive-common/2.3.9//hive-common-2.3.9.jar
-hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar
-hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar
-hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar
-hive-metastore/2.3.9//hive-metastore-2.3.9.jar
-hive-serde/2.3.9//hive-serde-2.3.9.jar
-hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
-hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar
-hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar
-hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar
-hive-shims/2.3.9//hive-shims-2.3.9.jar
hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
-hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar
hk2-api/2.6.1//hk2-api-2.6.1.jar
hk2-locator/2.6.1//hk2-locator-2.6.1.jar
hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -106,6 +88,7 @@ httpclient/4.5.13//httpclient-4.5.13.jar
httpcore/4.4.14//httpcore-4.4.14.jar
istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
ivy/2.5.0//ivy-2.5.0.jar
+j2objc-annotations/1.3//j2objc-annotations-1.3.jar
jackson-annotations/2.12.5//jackson-annotations-2.12.5.jar
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/2.12.5//jackson-core-2.12.5.jar
@@ -126,7 +109,6 @@ janino/3.0.16//janino-3.0.16.jar
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
javax.inject/1//javax.inject-1.jar
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
-javolution/5.5.1//javolution-5.5.1.jar
jaxb-api/2.2.11//jaxb-api-2.2.11.jar
jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
@@ -144,7 +126,6 @@ jline/2.14.6//jline-2.14.6.jar
joda-time/2.10.10//joda-time-2.10.10.jar
jodd-core/3.5.2//jodd-core-3.5.2.jar
jpam/1.1//jpam-1.1.jar
-json/1.8//json-1.8.jar
json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar
json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar
json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar
@@ -179,6 +160,7 @@ lapack/2.2.0//lapack-2.2.0.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
libthrift/0.12.0//libthrift-0.12.0.jar
+listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
log4j/1.2.17//log4j-1.2.17.jar
logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
lz4-java/1.8.0//lz4-java-1.8.0.jar
@@ -229,12 +211,10 @@ spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar
spire_2.12/0.17.0//spire_2.12-0.17.0.jar
stax-api/1.0.1//stax-api-1.0.1.jar
stream/2.9.6//stream-2.9.6.jar
-super-csv/2.2.0//super-csv-2.2.0.jar
threeten-extra/1.5.0//threeten-extra-1.5.0.jar
tink/1.6.0//tink-1.6.0.jar
transaction-api/1.1//transaction-api-1.1.jar
univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
-velocity/1.5//velocity-1.5.jar
xbean-asm9-shaded/4.20//xbean-asm9-shaded-4.20.jar
xercesImpl/2.12.0//xercesImpl-2.12.0.jar
xml-apis/1.4.01//xml-apis-1.4.01.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 41c8a24c8fff2..76b6ff52f42ce 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -1,13 +1,10 @@
-HikariCP/2.5.1//HikariCP-2.5.1.jar
JLargeArrays/1.5//JLargeArrays-1.5.jar
JTransforms/3.1//JTransforms-3.1.jar
RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar
-ST4/4.0.4//ST4-4.0.4.jar
activation/1.1.1//activation-1.1.1.jar
aircompressor/0.21//aircompressor-0.21.jar
algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar
annotations/17.0.0//annotations-17.0.0.jar
-antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
antlr4-runtime/4.8//antlr4-runtime-4.8.jar
aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
arpack/2.2.0//arpack-2.2.0.jar
@@ -22,10 +19,10 @@ avro-ipc/1.10.2//avro-ipc-1.10.2.jar
avro-mapred/1.10.2//avro-mapred-1.10.2.jar
avro/1.10.2//avro-1.10.2.jar
blas/2.2.0//blas-2.2.0.jar
-bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar
breeze_2.12/1.2//breeze_2.12-1.2.jar
cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar
+checker-qual/3.8.0//checker-qual-3.8.0.jar
chill-java/0.10.0//chill-java-0.10.0.jar
chill_2.12/0.10.0//chill_2.12-0.10.0.jar
commons-cli/1.2//commons-cli-1.2.jar
@@ -34,14 +31,12 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-compiler/3.0.16//commons-compiler-3.0.16.jar
commons-compress/1.21//commons-compress-1.21.jar
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
-commons-dbcp/1.4//commons-dbcp-1.4.jar
commons-io/2.11.0//commons-io-2.11.0.jar
commons-lang/2.6//commons-lang-2.6.jar
commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
commons-math3/3.4.1//commons-math3-3.4.1.jar
commons-net/3.1//commons-net-3.1.jar
-commons-pool/1.5.4//commons-pool-1.5.4.jar
commons-text/1.6//commons-text-1.6.jar
compress-lzf/1.0.3//compress-lzf-1.0.3.jar
core/1.1.2//core-1.1.2.jar
@@ -52,30 +47,17 @@ datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar
datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar
datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
derby/10.14.2.0//derby-10.14.2.0.jar
-dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
+error_prone_annotations/2.5.1//error_prone_annotations-2.5.1.jar
+failureaccess/1.0.1//failureaccess-1.0.1.jar
flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
generex/1.0.2//generex-1.0.2.jar
-gson/2.2.4//gson-2.2.4.jar
-guava/14.0.1//guava-14.0.1.jar
+gson/2.8.6//gson-2.8.6.jar
+guava/30.1.1-jre//guava-30.1.1-jre.jar
hadoop-client-api/3.3.1//hadoop-client-api-3.3.1.jar
hadoop-client-runtime/3.3.1//hadoop-client-runtime-3.3.1.jar
hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar
hadoop-yarn-server-web-proxy/3.3.1//hadoop-yarn-server-web-proxy-3.3.1.jar
-hive-beeline/2.3.9//hive-beeline-2.3.9.jar
-hive-cli/2.3.9//hive-cli-2.3.9.jar
-hive-common/2.3.9//hive-common-2.3.9.jar
-hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar
-hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar
-hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar
-hive-metastore/2.3.9//hive-metastore-2.3.9.jar
-hive-serde/2.3.9//hive-serde-2.3.9.jar
-hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar
-hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar
-hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar
-hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar
-hive-shims/2.3.9//hive-shims-2.3.9.jar
hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar
-hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar
hk2-api/2.6.1//hk2-api-2.6.1.jar
hk2-locator/2.6.1//hk2-locator-2.6.1.jar
hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -84,6 +66,7 @@ httpclient/4.5.13//httpclient-4.5.13.jar
httpcore/4.4.14//httpcore-4.4.14.jar
istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
ivy/2.5.0//ivy-2.5.0.jar
+j2objc-annotations/1.3//j2objc-annotations-1.3.jar
jackson-annotations/2.12.5//jackson-annotations-2.12.5.jar
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/2.12.5//jackson-core-2.12.5.jar
@@ -101,7 +84,6 @@ jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
janino/3.0.16//janino-3.0.16.jar
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
-javolution/5.5.1//javolution-5.5.1.jar
jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
jdo-api/3.0.1//jdo-api-3.0.1.jar
@@ -115,7 +97,6 @@ jline/2.14.6//jline-2.14.6.jar
joda-time/2.10.10//joda-time-2.10.10.jar
jodd-core/3.5.2//jodd-core-3.5.2.jar
jpam/1.1//jpam-1.1.jar
-json/1.8//json-1.8.jar
json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar
json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar
json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar
@@ -149,6 +130,7 @@ lapack/2.2.0//lapack-2.2.0.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
libthrift/0.12.0//libthrift-0.12.0.jar
+listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
log4j/1.2.17//log4j-1.2.17.jar
logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
lz4-java/1.8.0//lz4-java-1.8.0.jar
@@ -199,12 +181,10 @@ spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar
spire_2.12/0.17.0//spire_2.12-0.17.0.jar
stax-api/1.0.1//stax-api-1.0.1.jar
stream/2.9.6//stream-2.9.6.jar
-super-csv/2.2.0//super-csv-2.2.0.jar
threeten-extra/1.5.0//threeten-extra-1.5.0.jar
tink/1.6.0//tink-1.6.0.jar
transaction-api/1.1//transaction-api-1.1.jar
univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
-velocity/1.5//velocity-1.5.jar
xbean-asm9-shaded/4.20//xbean-asm9-shaded-4.20.jar
xz/1.8//xz-1.8.jar
zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
diff --git a/pom.xml b/pom.xml
index 2cece056b1c8c..b835ab9295d31 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,6 +96,7 @@
streaming
sql/catalyst
sql/core
+ sql/hive-shaded
sql/hive
assembly
examples
@@ -181,7 +182,14 @@
2.11.1
4.1.17
- 14.0.1
+ 4.2.4
+ 4.1.19
+ 3.2.0-m3
+ 3.0.1
+ 1.0.1
+
+ 30.1.1-jre
+ 14.0.1
3.0.16
2.34
2.10.10
@@ -197,6 +205,7 @@
1.2
1.60
1.6.0
+ 2.14.6
+
+
+ org.apache.commons
+ commons-lang3
+
+
+ joda-time
+ joda-time
+
+
+ io.dropwizard.metrics
+ metrics-core
+
+
+ io.dropwizard.metrics
+ metrics-jvm
+
+
+ io.dropwizard.metrics
+ metrics-json
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
+ jline
+ jline
+
@@ -1932,6 +1990,27 @@
org.pentaho
pentaho-aggdesigner-algorithm
+
+
+ org.datanucleus
+ datanucleus-core
+
+
+ commons-io
+ commons-io
+
+
+ oro
+ oro
+
+
+ org.apache.ivy
+ ivy
+
+
+ stax
+ stax-api
+
@@ -2000,6 +2079,15 @@
org.codehaus.groovy
groovy-all
+
+
+ org.apache.httpcomponents
+ httpclient
+
+
+ org.apache.httpcomponents
+ httpcore
+
@@ -2052,6 +2140,35 @@
*
+
+
+ org.apache.derby
+ derby
+
+
+ commons-pool
+ commons-pool
+
+
+ org.datanucleus
+ datanucleus-core
+
+
+ org.datanucleus
+ datanucleus-api-jdo
+
+
+ org.datanucleus
+ datanucleus-rdbms
+
+
+ org.datanucleus
+ javax.jdo
+
+
+ javax.jdo
+ jdo-api
+
@@ -2235,6 +2352,10 @@
org.slf4j
slf4j-api
+
+ org.apache.commons
+ commons-lang3
+
@@ -2242,7 +2363,6 @@
org.apache.hive
hive-llap-client
${hive23.version}
- test
org.apache.hive
@@ -2268,6 +2388,10 @@
org.apache.zookeeper
zookeeper
+
+ org.apache.commons
+ commons-lang3
+
org.slf4j
slf4j-api
@@ -2376,6 +2500,31 @@
datanucleus-core
${datanucleus-core.version}
+
+ org.datanucleus
+ datanucleus-api-jdo
+ ${datanucleus-api-jdo.version}
+
+
+ org.datanucleus
+ datanucleus-rdbms
+ ${datanucleus-rdbms.version}
+
+
+ org.datanucleus
+ javax.jdo
+ ${datanucleus-javax.jdo-version}
+
+
+ javax.jdo
+ jdo-api
+ ${javax.jdo.version}
+
+
+ stax
+ stax-api
+ ${stax-api.version}
+
org.apache.thrift
libthrift
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index b1531a601ca08..528a61ff9f270 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -43,8 +43,8 @@ object BuildCommons {
private val buildLocation = file(".").getAbsoluteFile.getParentFile
- val sqlProjects@Seq(catalyst, sql, hive, hiveThriftServer, tokenProviderKafka010, sqlKafka010, avro) = Seq(
- "catalyst", "sql", "hive", "hive-thriftserver", "token-provider-kafka-0-10", "sql-kafka-0-10", "avro"
+ val sqlProjects@Seq(catalyst, sql, hiveShaded, hive, hiveThriftServer, tokenProviderKafka010, sqlKafka010, avro) = Seq(
+ "catalyst", "sql", "hive-shaded", "hive", "hive-thriftserver", "token-provider-kafka-0-10", "sql-kafka-0-10", "avro"
).map(ProjectRef(buildLocation, _))
val streamingProjects@Seq(streaming, streamingKafka010) =
@@ -366,8 +366,8 @@ object SparkBuild extends PomBuild {
val mimaProjects = allProjects.filterNot { x =>
Seq(
- spark, hive, hiveThriftServer, catalyst, repl, networkCommon, networkShuffle, networkYarn,
- unsafe, tags, tokenProviderKafka010, sqlKafka010, kvstore, avro
+ spark, hiveShaded, hive, hiveThriftServer, catalyst, repl, networkCommon, networkShuffle,
+ networkYarn, unsafe, tags, tokenProviderKafka010, sqlKafka010, kvstore, avro
).contains(x)
}
@@ -667,7 +667,7 @@ object KubernetesIntegrationTests {
* Overrides to work around sbt's dependency resolution being different from Maven's.
*/
object DependencyOverrides {
- lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1")
+ lazy val guavaVersion = sys.props.get("guava.version").getOrElse("30.1.1-jre")
lazy val settings = Seq(
dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
@@ -990,10 +990,10 @@ object Unidoc {
(ScalaUnidoc / unidoc / unidocProjectFilter) :=
inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
- yarn, tags, streamingKafka010, sqlKafka010),
+ yarn, tags, streamingKafka010, sqlKafka010, hiveShaded),
(JavaUnidoc / unidoc / unidocProjectFilter) :=
inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
- yarn, tags, streamingKafka010, sqlKafka010),
+ yarn, tags, streamingKafka010, sqlKafka010, hiveShaded),
(ScalaUnidoc / unidoc / unidocAllClasspaths) := {
ignoreClasspaths((ScalaUnidoc / unidoc / unidocAllClasspaths).value)
diff --git a/sql/hive-shaded/pom.xml b/sql/hive-shaded/pom.xml
new file mode 100644
index 0000000000000..10be1ea95d113
--- /dev/null
+++ b/sql/hive-shaded/pom.xml
@@ -0,0 +1,238 @@
+
+
+
+
+ 4.0.0
+
+ org.apache.spark
+ spark-parent_2.12
+ 3.3.0-SNAPSHOT
+ ../../pom.xml
+
+
+ spark-hive-shaded_2.12
+ jar
+ Spark Project Hive Shaded
+ http://spark.apache.org/
+
+ hive-shaded
+
+
+
+
+ ${hive.group}
+ hive-common
+ ${hive.common.scope}
+
+
+ ${hive.group}
+ hive-exec
+ ${hive.classifier}
+
+
+ ${hive.group}
+ hive-metastore
+
+
+ ${hive.group}
+ hive-serde
+ ${hive.serde.scope}
+
+
+ ${hive.group}
+ hive-shims
+ ${hive.shims.scope}
+
+
+ org.apache.hive
+ hive-llap-common
+ ${hive.llap.scope}
+
+
+ org.apache.hive
+ hive-llap-client
+ ${hive.llap.scope}
+
+
+
+ com.google.guava
+ guava
+ ${hive.guava.version}
+ compile
+
+
+
+
+
+ hive-thriftserver
+
+
+ ${hive.group}
+ hive-cli
+
+
+ ${hive.group}
+ hive-jdbc
+
+
+ ${hive.group}
+ hive-beeline
+
+
+ ${hive.group}
+ hive-service-rpc
+
+
+
+
+
+
+ target/scala-${scala.binary.version}/classes
+ target/scala-${scala.binary.version}/test-classes
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+ package
+
+ shade
+
+
+ true
+ true
+ false
+
+
+ *:*
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+ parquet-logging.properties
+ hive-log4j2.properties
+ hive-exec-log4j2.properties
+ beeline-log4j2.properties
+ tez-container-log4j2.properties
+ bonecp-default-config.xml
+
+
+
+
+
+ com.google.common
+ ${spark.shade.packageName}.hive.guava
+
+
+ com.google.protobuf
+ ${spark.shade.packageName}.hive.com.google.protobuf
+
+
+ com.google.gson
+ ${spark.shade.packageName}.hive.com.google.gson
+
+
+ com.github.joshelser
+ ${spark.shade.packageName}.hive.com.github.joshelser
+
+
+ com.jolbox.bonecp
+ ${spark.shade.packageName}.hive.com.jolbox.bonecp
+
+
+ com.zaxxer.hikari
+ ${spark.shade.packageName}.hive.com.zaxxer.hikari
+
+
+ au.com.bytecode
+ ${spark.shade.packageName}.hive.au.com.bytecode
+
+
+ org.apache.commons
+ ${spark.shade.packageName}.hive.org.apache.commons
+
+
+ org.antlr.runtime
+ ${spark.shade.packageName}.hive.org.antlr.runtime
+
+
+ org.apache.velocity
+ ${spark.shade.packageName}.hive.org.apache.velocity
+
+
+ org.stringtemplate.v4
+ ${spark.shade.packageName}.hive.org.stringtemplate.v4
+
+
+ javolution
+ ${spark.shade.packageName}.hive.javolution
+
+
+ org.supercsv
+ ${spark.shade.packageName}.hive.org.supercsv
+
+
+
+
+
+ reference.conf
+
+
+ log4j.properties
+
+
+
+
+
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ compile
+ package
+
+ attach-artifact
+
+
+
+
+ ${basedir}/target/${project.artifactId}-${project.version}.jar
+ jar
+ optional
+
+
+
+
+
+
+
+
+
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 764712fcbb482..f90e2f7f57f84 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -40,6 +40,11 @@
spark-hive_${scala.binary.version}
${project.version}
+
+ org.apache.spark
+ spark-hive-shaded_${scala.binary.version}
+ ${project.version}
+
org.apache.spark
spark-core_${scala.binary.version}
@@ -71,22 +76,6 @@
com.google.guava
guava
-
- ${hive.group}
- hive-cli
-
-
- ${hive.group}
- hive-jdbc
-
-
- ${hive.group}
- hive-beeline
-
-
- ${hive.group}
- hive-service-rpc
-
org.eclipse.jetty
jetty-server
@@ -97,6 +86,10 @@
jetty-servlet
provided
+
+ org.apache.httpcomponents
+ httpcore
+
org.seleniumhq.selenium
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 86cd1eddde330..c72135d56ee96 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -40,6 +40,11 @@
spark-core_${scala.binary.version}
${project.version}
+
+ org.apache.spark
+ spark-hive-shaded_${scala.binary.version}
+ ${project.version}
+
org.apache.spark
spark-core_${scala.binary.version}
@@ -85,51 +90,6 @@
scala-parallel-collections_${scala.binary.version}
-->
-
-
- ${hive.group}
- hive-common
- ${hive.common.scope}
-
-
- ${hive.group}
- hive-exec
- ${hive.classifier}
-
-
- ${hive.group}
- hive-metastore
-
-
- ${hive.group}
- hive-serde
- ${hive.serde.scope}
-
-
- ${hive.group}
- hive-shims
- ${hive.shims.scope}
-
-
- org.apache.hive
- hive-llap-common
- ${hive.llap.scope}
-
-
- org.apache.hive
- hive-llap-client
- ${hive.llap.scope}
-
org.apache.avro
@@ -168,6 +128,26 @@
org.datanucleus
datanucleus-core
+
+ org.datanucleus
+ datanucleus-api-jdo
+
+
+ org.datanucleus
+ datanucleus-rdbms
+
+
+ org.datanucleus
+ javax.jdo
+
+
+ javax.jdo
+ jdo-api
+
+
+ stax
+ stax-api
+
org.apache.hadoop
${hadoop-client-runtime.artifact}
@@ -185,6 +165,10 @@
org.apache.derby
derby
+
+ jline
+ jline
+
org.scala-lang
scala-compiler
@@ -237,27 +221,6 @@
-da -Xmx4g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true
-
- org.apache.maven.plugins
- maven-enforcer-plugin
-
-
- enforce-versions
-
- enforce
-
-
-
-
-
- *:hive-cli
-
-
-
-
-
-
-
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index 9aa6a09fd57af..c64af2dd29eb7 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -202,7 +202,6 @@ private[hive] class IsolatedClientLoader(
name.startsWith("org.apache.spark.") ||
isHadoopClass ||
name.startsWith("scala.") ||
- (name.startsWith("com.google") && !name.startsWith("com.google.cloud")) ||
name.startsWith("java.") ||
name.startsWith("javax.sql.") ||
sharedPrefixes.exists(name.startsWith)