Skip to content

Commit a3f7cff

Browse files
wangyumwangyum
committed
Exclude org.eclipse.jetty.orbit:javax.servlet, org.apache.hbase:hbase-client and co.cask.tephra:*
1 parent d22e7e0 commit a3f7cff

File tree

3 files changed

+76
-14
lines changed

3 files changed

+76
-14
lines changed

‎pom.xml‎

Lines changed: 70 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@
128128
<hive.classifier></hive.classifier>
129129
<!-- Version used in Maven Hive dependency -->
130130
<hive.version>1.2.1.spark2</hive.version>
131+
<hive23.version>2.3.4</hive23.version>
131132
<!-- Version used for internal directory structure -->
132133
<hive.version.short>1.2.1</hive.version.short>
133134
<!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
@@ -1414,7 +1415,7 @@
14141415
<groupId>commons-logging</groupId>
14151416
<artifactId>commons-logging</artifactId>
14161417
</exclusion>
1417-
<!-- Begin of Hive 2.3.4 exclusion -->
1418+
<!-- Begin of Hive 2.3 exclusion -->
14181419
<!-- jetty-all conflict with jetty 9.4.12.v20180830 -->
14191420
<exclusion>
14201421
<groupId>org.eclipse.jetty.aggregate</groupId>
@@ -1425,7 +1426,17 @@
14251426
<groupId>org.apache.logging.log4j</groupId>
14261427
<artifactId>*</artifactId>
14271428
</exclusion>
1428-
<!-- End of Hive 2.3.4 exclusion -->
1429+
<!-- Hive includes javax.servlet to fix the Hive on Spark test failure; see HIVE-12783 -->
1430+
<exclusion>
1431+
<groupId>org.eclipse.jetty.orbit</groupId>
1432+
<artifactId>javax.servlet</artifactId>
1433+
</exclusion>
1434+
<!-- hive-storage-api is needed and must be explicitly included later -->
1435+
<exclusion>
1436+
<groupId>org.apache.hive</groupId>
1437+
<artifactId>hive-storage-api</artifactId>
1438+
</exclusion>
1439+
<!-- End of Hive 2.3 exclusion -->
14291440
</exclusions>
14301441
</dependency>
14311442

@@ -1544,7 +1555,7 @@
15441555
<groupId>org.json</groupId>
15451556
<artifactId>json</artifactId>
15461557
</exclusion>
1547-
<!-- Begin of Hive 2.3.4 exclusion -->
1558+
<!-- Begin of Hive 2.3 exclusion -->
15481559
<!-- Do not need Tez -->
15491560
<exclusion>
15501561
<groupId>${hive.group}</groupId>
@@ -1564,7 +1575,7 @@
15641575
<groupId>org.apache.logging.log4j</groupId>
15651576
<artifactId>*</artifactId>
15661577
</exclusion>
1567-
<!-- End of Hive 2.3.4 exclusion -->
1578+
<!-- End of Hive 2.3 exclusion -->
15681579
</exclusions>
15691580
</dependency>
15701581
<dependency>
@@ -1673,6 +1684,17 @@
16731684
<groupId>org.slf4j</groupId>
16741685
<artifactId>slf4j-log4j12</artifactId>
16751686
</exclusion>
1687+
<!-- Begin of Hive 2.3 exclusion -->
1688+
<!-- Hive removes the HBase Metastore; see HIVE-17234 -->
1689+
<exclusion>
1690+
<groupId>org.apache.hbase</groupId>
1691+
<artifactId>hbase-client</artifactId>
1692+
</exclusion>
1693+
<exclusion>
1694+
<groupId>co.cask.tephra</groupId>
1695+
<artifactId>*</artifactId>
1696+
</exclusion>
1697+
<!-- End of Hive 2.3 exclusion -->
16761698
</exclusions>
16771699
</dependency>
16781700

@@ -1730,7 +1752,7 @@
17301752
<groupId>org.codehaus.groovy</groupId>
17311753
<artifactId>groovy-all</artifactId>
17321754
</exclusion>
1733-
<!-- Begin of Hive 2.3.4 exclusion -->
1755+
<!-- Begin of Hive 2.3 exclusion -->
17341756
<!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
17351757
<exclusion>
17361758
<groupId>org.apache.parquet</groupId>
@@ -1745,7 +1767,7 @@
17451767
<groupId>tomcat</groupId>
17461768
<artifactId>jasper-runtime</artifactId>
17471769
</exclusion>
1748-
<!-- End of Hive 2.3.4 exclusion -->
1770+
<!-- End of Hive 2.3 exclusion -->
17491771
</exclusions>
17501772
</dependency>
17511773

@@ -1811,21 +1833,22 @@
18111833
<groupId>org.codehaus.groovy</groupId>
18121834
<artifactId>groovy-all</artifactId>
18131835
</exclusion>
1814-
<!-- Begin of Hive 2.3.4 exclusion -->
1836+
<!-- Begin of Hive 2.3 exclusion -->
18151837
<!-- Exclude log4j-slf4j-impl, otherwise throw NCDFE when starting spark-shell -->
18161838
<exclusion>
18171839
<groupId>org.apache.logging.log4j</groupId>
18181840
<artifactId>log4j-slf4j-impl</artifactId>
18191841
</exclusion>
1820-
<!-- End of Hive 2.3.4 exclusion -->
1842+
<!-- End of Hive 2.3 exclusion -->
18211843
</exclusions>
18221844
</dependency>
18231845

1824-
<!-- Hive 2.3 need hive-llap-client, We add it here, otherwise the scope won't work -->
1846+
<!-- hive-llap-common is needed when registry UDFs in Hive 2.3.
1847+
We add it here, otherwise -Phive-provided won't work. -->
18251848
<dependency>
18261849
<groupId>org.apache.hive</groupId>
1827-
<artifactId>hive-llap-client</artifactId>
1828-
<version>2.3.4</version>
1850+
<artifactId>hive-llap-common</artifactId>
1851+
<version>${hive23.version}</version>
18291852
<scope>${hive.deps.scope}</scope>
18301853
<exclusions>
18311854
<exclusion>
@@ -1836,6 +1859,31 @@
18361859
<groupId>org.apache.hive</groupId>
18371860
<artifactId>hive-serde</artifactId>
18381861
</exclusion>
1862+
<exclusion>
1863+
<groupId>org.slf4j</groupId>
1864+
<artifactId>slf4j-api</artifactId>
1865+
</exclusion>
1866+
</exclusions>
1867+
</dependency>
1868+
<!-- hive-llap-client is needed when run MapReduce test in Hive 2.3. -->
1869+
<dependency>
1870+
<groupId>org.apache.hive</groupId>
1871+
<artifactId>hive-llap-client</artifactId>
1872+
<version>${hive23.version}</version>
1873+
<scope>test</scope>
1874+
<exclusions>
1875+
<exclusion>
1876+
<groupId>org.apache.hive</groupId>
1877+
<artifactId>hive-common</artifactId>
1878+
</exclusion>
1879+
<exclusion>
1880+
<groupId>org.apache.hive</groupId>
1881+
<artifactId>hive-serde</artifactId>
1882+
</exclusion>
1883+
<exclusion>
1884+
<groupId>org.apache.hive</groupId>
1885+
<artifactId>hive-llap-common</artifactId>
1886+
</exclusion>
18391887
<exclusion>
18401888
<groupId>org.apache.curator</groupId>
18411889
<artifactId>curator-framework</artifactId>
@@ -1844,6 +1892,14 @@
18441892
<groupId>org.apache.curator</groupId>
18451893
<artifactId>apache-curator</artifactId>
18461894
</exclusion>
1895+
<exclusion>
1896+
<groupId>org.apache.zookeeper</groupId>
1897+
<artifactId>zookeeper</artifactId>
1898+
</exclusion>
1899+
<exclusion>
1900+
<groupId>org.slf4j</groupId>
1901+
<artifactId>slf4j-api</artifactId>
1902+
</exclusion>
18471903
</exclusions>
18481904
</dependency>
18491905

@@ -2741,11 +2797,11 @@
27412797
<zookeeper.version>3.4.13</zookeeper.version>
27422798
<hive.group>org.apache.hive</hive.group>
27432799
<hive.classifier>core</hive.classifier>
2744-
<hive.version>2.3.4</hive.version>
2745-
<hive.version.short>${hive.version}</hive.version.short>
2800+
<hive.version>${hive23.version}</hive.version>
2801+
<hive.version.short>2.3.4</hive.version.short>
27462802
<hive.parquet.version>${parquet.version}</hive.parquet.version>
2747-
<orc.classifier></orc.classifier>
27482803
<hive.parquet.group>org.apache.parquet</hive.parquet.group>
2804+
<orc.classifier></orc.classifier>
27492805
<datanucleus-core.version>4.1.17</datanucleus-core.version>
27502806
</properties>
27512807
<dependencies>

‎sql/hive/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,10 @@
223223
<groupId>${hive.group}</groupId>
224224
<artifactId>hive-shims</artifactId>
225225
</dependency>
226+
<dependency>
227+
<groupId>org.apache.hive</groupId>
228+
<artifactId>hive-llap-common</artifactId>
229+
</dependency>
226230
<dependency>
227231
<groupId>org.apache.hive</groupId>
228232
<artifactId>hive-llap-client</artifactId>

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -710,6 +710,8 @@ private[hive] class HiveClientImpl(
710710
/**
711711
* Execute the command using Hive and return the results as a sequence. Each element
712712
* in the sequence is one row.
713+
* Since upgrading the built-in Hive to 2.3, hive-llap-client is needed when
714+
* running MapReduce jobs with `runHive`.
713715
*/
714716
protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = withHiveState {
715717
logDebug(s"Running hiveql '$cmd'")

0 commit comments

Comments
 (0)