From 3620b243219e4747e31207c7d1cf4bc9a5822fd3 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Mon, 2 Sep 2019 18:08:55 +0800 Subject: [PATCH 01/11] use Spark's HiveClientImpls's addJar to replace ThriftserverShimUtils's method --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 8 +++++--- .../sql/hive/thriftserver/ThriftserverShimUtils.scala | 6 ------ .../sql/hive/thriftserver/ThriftserverShimUtils.scala | 7 ------- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index b9614d49eadbd..db5bb3c8511e3 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -42,7 +42,7 @@ import org.apache.spark.SparkConf import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.hive.HiveUtils +import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider import org.apache.spark.util.ShutdownHookManager @@ -139,10 +139,12 @@ private[hive] object SparkSQLCLIDriver extends Logging { // Hadoop-20 and above - we need to augment classpath using hiveconf // components. // See also: code in ExecDriver.java - var loader = conf.getClassLoader + val loader = conf.getClassLoader val auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS) if (StringUtils.isNotBlank(auxJars)) { - loader = ThriftserverShimUtils.addToClassPath(loader, StringUtils.split(auxJars, ",")) + val hiveClient = SparkSQLEnv.sqlContext.sharedState.externalCatalog.unwrapped + .asInstanceOf[HiveExternalCatalog].client + StringUtils.split(auxJars, ",").foreach(hiveClient.addJar(_)) } conf.setClassLoader(loader) Thread.currentThread().setContextClassLoader(loader) diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 87c0f8f6a571a..0ff76c73c1d3e 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -51,12 +51,6 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType - private[thriftserver] def addToClassPath( - loader: ClassLoader, - auxJars: Array[String]): ClassLoader = { - Utilities.addToClassPath(loader, auxJars) - } - private[thriftserver] val testedProtocolVersions = Seq( HIVE_CLI_SERVICE_PROTOCOL_V1, HIVE_CLI_SERVICE_PROTOCOL_V2, diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 124c9937c0fca..ee6a79509846b 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -56,13 +56,6 @@ private[thriftserver] object ThriftserverShimUtils { private[thriftserver] def toJavaSQLType(s: String): Int = Type.getType(s).toJavaSQLType - private[thriftserver] def addToClassPath( - loader: ClassLoader, - auxJars: Array[String]): ClassLoader = { - val addAction = new AddToClassPathAction(loader, auxJars.toList.asJava) - AccessController.doPrivileged(addAction) - } - private[thriftserver] val testedProtocolVersions = Seq( HIVE_CLI_SERVICE_PROTOCOL_V1, HIVE_CLI_SERVICE_PROTOCOL_V2, From 826370a846a2334165a4f77e32b57f1f8bbe0371 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Mon, 2 Sep 2019 18:19:57 +0800 Subject: [PATCH 02/11] remove unused import --- .../spark/sql/hive/thriftserver/ThriftserverShimUtils.scala | 1 - .../spark/sql/hive/thriftserver/ThriftserverShimUtils.scala | 4 ---- 2 files changed, 5 deletions(-) diff --git a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index 0ff76c73c1d3e..4eb5f5da8fdcc 100644 --- a/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v1.2.1/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.hive.thriftserver import org.apache.commons.logging.LogFactory -import org.apache.hadoop.hive.ql.exec.Utilities import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hive.service.cli.{RowSet, RowSetFactory, TableSchema, Type} import org.apache.hive.service.cli.thrift.TProtocolVersion._ diff --git a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala index ee6a79509846b..2c1a0bd685e3f 100644 --- a/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala +++ b/sql/hive-thriftserver/v2.3.5/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala @@ -17,10 +17,6 @@ package org.apache.spark.sql.hive.thriftserver -import java.security.AccessController - -import scala.collection.JavaConverters._ - import org.apache.hadoop.hive.ql.exec.AddToClassPathAction import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hadoop.hive.serde2.thrift.Type From 6208d42834f917caa8a9ceba2ce7316c695148ac Mon Sep 17 00:00:00 2001 From: angerszhu Date: Mon, 2 Sep 2019 19:12:14 +0800 Subject: [PATCH 03/11] add UT --- .../sql/hive/thriftserver/CliSuite.scala | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 6e042ac41d9da..dd644f9d25d3f 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -305,4 +305,30 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { "SELECT example_max(1);" -> "1" ) } + + test("SPARK-28840 test --jars command") { + val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath + runCliWithin( + 1.minute, + Seq(s"--jars", s"$jarFile"))( + s"CREATE TEMPORARY FUNCTION testjar AS" + + s" 'org.apache.spark.sql.hive.execution.UDTFStack';" -> "", + "SELECT testjar(1,'A', 10);" -> "A\t10" + ) + } + + test("SPARK-28840 test --jars and hive.aux.jars.path command") { + val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath + val hiveContribJar = HiveTestUtils.getHiveContribJar.getCanonicalPath + runCliWithin( + 1.minute, + Seq(s"--jars", s"$jarFile", "--conf", + s"spark.hadoop.${ConfVars.HIVEAUXJARS}=$hiveContribJar"))( + s"CREATE TEMPORARY FUNCTION testjar AS" + + s" 'org.apache.spark.sql.hive.execution.UDTFStack';" -> "", + "SELECT testjar(1,'A', 10);" -> "A\t10", + s"CREATE TEMPORARY FUNCTION example_max AS '${classOf[UDAFExampleMax].getName}';" -> "", + "SELECT example_max(1);" -> "1" + ) + } } From cdb503e6ffb078a90edfbe4815ae34d110194526 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Mon, 2 Sep 2019 21:53:28 +0800 Subject: [PATCH 04/11] fix NPE problem --- .../hive/thriftserver/SparkSQLCLIDriver.scala | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index db5bb3c8511e3..04cd9f35d633f 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -135,23 +135,6 @@ private[hive] object SparkSQLCLIDriver extends Logging { val remoteMode = isRemoteMode(sessionState) // "-h" option has been passed, so connect to Hive thrift server. - if (!remoteMode) { - // Hadoop-20 and above - we need to augment classpath using hiveconf - // components. - // See also: code in ExecDriver.java - val loader = conf.getClassLoader - val auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS) - if (StringUtils.isNotBlank(auxJars)) { - val hiveClient = SparkSQLEnv.sqlContext.sharedState.externalCatalog.unwrapped - .asInstanceOf[HiveExternalCatalog].client - StringUtils.split(auxJars, ",").foreach(hiveClient.addJar(_)) - } - conf.setClassLoader(loader) - Thread.currentThread().setContextClassLoader(loader) - } else { - // Hive 1.2 + not supported in CLI - throw new RuntimeException("Remote operations not supported") - } // Respect the configurations set by --hiveconf from the command line // (based on Hive's CliDriver). val hiveConfFromCmd = sessionState.getOverriddenConfigurations.entrySet().asScala @@ -166,6 +149,16 @@ private[hive] object SparkSQLCLIDriver extends Logging { val cli = new SparkSQLCLIDriver cli.setHiveVariables(oproc.getHiveVariables) + // Hadoop-20 and above - we need to augment classpath using hiveconf + // components. + // See also: code in ExecDriver.java + val auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS) + if (StringUtils.isNotBlank(auxJars)) { + val hiveClient = SparkSQLEnv.sqlContext.sharedState.externalCatalog.unwrapped + .asInstanceOf[HiveExternalCatalog].client + StringUtils.split(auxJars, ",").foreach(hiveClient.addJar(_)) + } + // TODO work around for set the log output to console, because the HiveContext // will set the output into an invalid buffer. sessionState.in = System.in From b9866022cb55b021e838007055a1e1f0bf17b230 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Mon, 2 Sep 2019 23:19:43 +0800 Subject: [PATCH 05/11] use resource loader --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 04cd9f35d633f..d0aa0e1c20dc5 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -133,7 +133,6 @@ private[hive] object SparkSQLCLIDriver extends Logging { // Clean up after we exit ShutdownHookManager.addShutdownHook { () => SparkSQLEnv.stop() } - val remoteMode = isRemoteMode(sessionState) // "-h" option has been passed, so connect to Hive thrift server. // Respect the configurations set by --hiveconf from the command line // (based on Hive's CliDriver). @@ -154,9 +153,8 @@ private[hive] object SparkSQLCLIDriver extends Logging { // See also: code in ExecDriver.java val auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS) if (StringUtils.isNotBlank(auxJars)) { - val hiveClient = SparkSQLEnv.sqlContext.sharedState.externalCatalog.unwrapped - .asInstanceOf[HiveExternalCatalog].client - StringUtils.split(auxJars, ",").foreach(hiveClient.addJar(_)) + val resourceLoader = SparkSQLEnv.sqlContext.sessionState.resourceLoader + StringUtils.split(auxJars, ",").foreach(resourceLoader.addJar(_)) } // TODO work around for set the log output to console, because the HiveContext From ded1aaad5622d33f121f0316eb31009e79a548b3 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Tue, 3 Sep 2019 00:02:46 +0800 Subject: [PATCH 06/11] only fix add jar through conf --- .../hive/thriftserver/SparkSQLCLIDriver.scala | 4 ++- .../sql/hive/thriftserver/CliSuite.scala | 26 ------------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index d0aa0e1c20dc5..01c6b4812aaac 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -133,7 +133,8 @@ private[hive] object SparkSQLCLIDriver extends Logging { // Clean up after we exit ShutdownHookManager.addShutdownHook { () => SparkSQLEnv.stop() } - // "-h" option has been passed, so connect to Hive thrift server. + Thread.currentThread().setContextClassLoader(conf.getClassLoader) + // Respect the configurations set by --hiveconf from the command line // (based on Hive's CliDriver). val hiveConfFromCmd = sessionState.getOverriddenConfigurations.entrySet().asScala @@ -148,6 +149,7 @@ private[hive] object SparkSQLCLIDriver extends Logging { val cli = new SparkSQLCLIDriver cli.setHiveVariables(oproc.getHiveVariables) + // "-h" option has been passed, so connect to Hive thrift server. // Hadoop-20 and above - we need to augment classpath using hiveconf // components. // See also: code in ExecDriver.java diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index dd644f9d25d3f..6e042ac41d9da 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -305,30 +305,4 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { "SELECT example_max(1);" -> "1" ) } - - test("SPARK-28840 test --jars command") { - val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath - runCliWithin( - 1.minute, - Seq(s"--jars", s"$jarFile"))( - s"CREATE TEMPORARY FUNCTION testjar AS" + - s" 'org.apache.spark.sql.hive.execution.UDTFStack';" -> "", - "SELECT testjar(1,'A', 10);" -> "A\t10" - ) - } - - test("SPARK-28840 test --jars and hive.aux.jars.path command") { - val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath - val hiveContribJar = HiveTestUtils.getHiveContribJar.getCanonicalPath - runCliWithin( - 1.minute, - Seq(s"--jars", s"$jarFile", "--conf", - s"spark.hadoop.${ConfVars.HIVEAUXJARS}=$hiveContribJar"))( - s"CREATE TEMPORARY FUNCTION testjar AS" + - s" 'org.apache.spark.sql.hive.execution.UDTFStack';" -> "", - "SELECT testjar(1,'A', 10);" -> "A\t10", - s"CREATE TEMPORARY FUNCTION example_max AS '${classOf[UDAFExampleMax].getName}';" -> "", - "SELECT example_max(1);" -> "1" - ) - } } From c47204b567e6716381773cd960fc3a689c3955d2 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Tue, 3 Sep 2019 12:57:49 +0800 Subject: [PATCH 07/11] fix hive 2.3 --jar problem --- .../sql/hive/thriftserver/SparkSQLCLIDriver.scala | 4 ++-- .../apache/spark/sql/hive/thriftserver/CliSuite.scala | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 01c6b4812aaac..ca83f01bf719a 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -111,6 +111,8 @@ private[hive] object SparkSQLCLIDriver extends Logging { // Set all properties specified via command line. val conf: HiveConf = sessionState.getConf + // Use startup ClassLoader replace hiveConf's UDFClassLoader + conf.setClassLoader(Thread.currentThread().getContextClassLoader) sessionState.cmdProperties.entrySet().asScala.foreach { item => val key = item.getKey.toString val value = item.getValue.toString @@ -133,8 +135,6 @@ private[hive] object SparkSQLCLIDriver extends Logging { // Clean up after we exit ShutdownHookManager.addShutdownHook { () => SparkSQLEnv.stop() } - Thread.currentThread().setContextClassLoader(conf.getClassLoader) - // Respect the configurations set by --hiveconf from the command line // (based on Hive's CliDriver). val hiveConfFromCmd = sessionState.getOverriddenConfigurations.entrySet().asScala diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 6e042ac41d9da..2e07b3cae60f9 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -305,4 +305,15 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { "SELECT example_max(1);" -> "1" ) } + + test("SPARK-28840 test --jars command") { + val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath + runCliWithin( + 1.minute, + Seq(s"--jars", s"$jarFile"))( + s"CREATE TEMPORARY FUNCTION testjar AS" + + s" 'org.apache.spark.sql.hive.execution.UDTFStack';" -> "", + "SELECT testjar(1,'A', 10);" -> "A\t10" + ) + } } From 750e38332bd97c3de00847f5d235326d17a49221 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Tue, 3 Sep 2019 13:06:05 +0800 Subject: [PATCH 08/11] change comment --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index ca83f01bf719a..ad6cb78047059 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -149,10 +149,9 @@ private[hive] object SparkSQLCLIDriver extends Logging { val cli = new SparkSQLCLIDriver cli.setHiveVariables(oproc.getHiveVariables) - // "-h" option has been passed, so connect to Hive thrift server. - // Hadoop-20 and above - we need to augment classpath using hiveconf - // components. - // See also: code in ExecDriver.java + // In SparkSQL CLI, we may want to use jars augmented by hiveconf + // hive.aux.jars.path, here we add jars augmented by hiveconf to + // Spark's SessionResourceLoader to obtain these jars. val auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS) if (StringUtils.isNotBlank(auxJars)) { val resourceLoader = SparkSQLEnv.sqlContext.sessionState.resourceLoader From b106a2dcd3e553a50f661241563f89da28d73f24 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Tue, 3 Sep 2019 13:06:43 +0800 Subject: [PATCH 09/11] . --- .../scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 2e07b3cae60f9..a14d271677988 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -306,7 +306,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { ) } - test("SPARK-28840 test --jars command") { + test("SPARK-28954 test --jars command") { val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath runCliWithin( 1.minute, From 7fc001159f6e647ce4f5e705fcd643c2fa4107ab Mon Sep 17 00:00:00 2001 From: angerszhu Date: Tue, 3 Sep 2019 13:36:13 +0800 Subject: [PATCH 10/11] fix code style and throw exception when start with wrong hive version --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 6 +++++- .../org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index ad6cb78047059..fe5639bdee90f 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -42,7 +42,7 @@ import org.apache.spark.SparkConf import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} +import org.apache.spark.sql.hive.HiveUtils import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider import org.apache.spark.util.ShutdownHookManager @@ -135,6 +135,10 @@ private[hive] object SparkSQLCLIDriver extends Logging { // Clean up after we exit ShutdownHookManager.addShutdownHook { () => SparkSQLEnv.stop() } + isRemoteMode(sessionState) { + // Hive 1.2 + not supported in CLI + throw new RuntimeException("Remote operations not supported") + } // Respect the configurations set by --hiveconf from the command line // (based on Hive's CliDriver). val hiveConfFromCmd = sessionState.getOverriddenConfigurations.entrySet().asScala diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index a14d271677988..815ad33982515 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -310,7 +310,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath runCliWithin( 1.minute, - Seq(s"--jars", s"$jarFile"))( + Seq("--jars", s"$jarFile"))( s"CREATE TEMPORARY FUNCTION testjar AS" + s" 'org.apache.spark.sql.hive.execution.UDTFStack';" -> "", "SELECT testjar(1,'A', 10);" -> "A\t10" From 2d32ea36f3cd05016854fbf421881559fc405a26 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Tue, 3 Sep 2019 13:57:49 +0800 Subject: [PATCH 11/11] add if --- .../apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index fe5639bdee90f..71e751fd2882e 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -135,7 +135,7 @@ private[hive] object SparkSQLCLIDriver extends Logging { // Clean up after we exit ShutdownHookManager.addShutdownHook { () => SparkSQLEnv.stop() } - isRemoteMode(sessionState) { + if (isRemoteMode(sessionState)) { // Hive 1.2 + not supported in CLI throw new RuntimeException("Remote operations not supported") }