From 80f9b3fdf00ec6dc2dbbe2210c124667d6b7331a Mon Sep 17 00:00:00 2001 From: tedyu Date: Wed, 3 Feb 2016 13:45:06 -0800 Subject: [PATCH 1/8] Wrap HiveClientImpl#conf with withHiveState --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index cf1ff55c96fc9..51d4fea526971 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -134,7 +134,9 @@ private[hive] class HiveClientImpl( } /** Returns the configuration for the current session. */ - def conf: HiveConf = SessionState.get().getConf + def conf: HiveConf = withHiveState { + SessionState.get().getConf + } override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue) From 2767d5692e773f5f503b66d9a79e1048512deed1 Mon Sep 17 00:00:00 2001 From: tedyu Date: Wed, 3 Feb 2016 15:24:04 -0800 Subject: [PATCH 2/8] Revert --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 51d4fea526971..cf1ff55c96fc9 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -134,9 +134,7 @@ private[hive] class HiveClientImpl( } /** Returns the configuration for the current session. */ - def conf: HiveConf = withHiveState { - SessionState.get().getConf - } + def conf: HiveConf = SessionState.get().getConf override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue) From 6ee7d9824a2e4cac043a0000c14b745b6c3c5de2 Mon Sep 17 00:00:00 2001 From: tedyu Date: Wed, 3 Feb 2016 15:26:28 -0800 Subject: [PATCH 3/8] Protect against SessionState being null when accessing HiveClientImpl#conf --- .../spark/sql/hive/client/HiveClientImpl.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index cf1ff55c96fc9..4e51f29c1cf66 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -134,7 +134,18 @@ private[hive] class HiveClientImpl( } /** Returns the configuration for the current session. */ - def conf: HiveConf = SessionState.get().getConf + def conf: HiveConf = { + var ss = SessionState.get() + // SessionState is lazy initialization, it can be null here + if (ss == null) { + val original = Thread.currentThread().getContextClassLoader + val conf = new HiveConf(classOf[SessionState]) + conf.setClassLoader(original) + ss = new SessionState(conf) + SessionState.start(ss) + } + ss.getConf + } override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue) From aff16d6be038d59dae114f72b5c915a29ecde919 Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 4 Feb 2016 07:51:16 -0800 Subject: [PATCH 4/8] Address Herman's review comment --- .../org/apache/spark/sql/hive/HiveQl.scala | 14 +----- .../sql/hive/client/HiveClientImpl.scala | 13 +----- .../spark/sql/hive/client/HiveConfUtil.scala | 43 +++++++++++++++++++ 3 files changed, 46 insertions(+), 24 deletions(-) create mode 100644 sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 752c037a842a8..1f5d910021996 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -22,6 +22,7 @@ import java.util.Locale import scala.collection.JavaConverters._ import org.apache.hadoop.hive.common.`type`.HiveDecimal +import org.apache.hadoop.hive.client.HiveConfUtil import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.apache.hadoop.hive.ql.exec.{FunctionInfo, FunctionRegistry} @@ -168,18 +169,7 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging /** * Returns the HiveConf */ - private[this] def hiveConf: HiveConf = { - var ss = SessionState.get() - // SessionState is lazy initialization, it can be null here - if (ss == null) { - val original = Thread.currentThread().getContextClassLoader - val conf = new HiveConf(classOf[SessionState]) - conf.setClassLoader(original) - ss = new SessionState(conf) - SessionState.start(ss) - } - ss.getConf - } + private[this] def hiveConf: HiveConf = HiveConfUtil.conf() protected def getProperties(node: ASTNode): Seq[(String, String)] = node match { case Token("TOK_TABLEPROPLIST", list) => diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 4e51f29c1cf66..f8dfb9b9b26aa 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -134,18 +134,7 @@ private[hive] class HiveClientImpl( } /** Returns the configuration for the current session. */ - def conf: HiveConf = { - var ss = SessionState.get() - // SessionState is lazy initialization, it can be null here - if (ss == null) { - val original = Thread.currentThread().getContextClassLoader - val conf = new HiveConf(classOf[SessionState]) - conf.setClassLoader(original) - ss = new SessionState(conf) - SessionState.start(ss) - } - ss.getConf - } + def conf: HiveConf = HiveConfUtil.conf() override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala new file mode 100644 index 0000000000000..b36307b459a6a --- /dev/null +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.client + +import scala.collection.JavaConverters._ +import scala.language.reflectiveCalls + +import org.apache.hadoop.hive.conf.HiveConf +import org.apache.hadoop.hive.ql.session.SessionState + +/** + * Configuration utility for Hive + */ +object HiveConfUtil { + /** Returns the configuration for the current session. */ + def conf: HiveConf = { + var ss = SessionState.get() + // SessionState is lazy initialization, it can be null here + if (ss == null) { + val original = Thread.currentThread().getContextClassLoader + val conf = new HiveConf(classOf[SessionState]) + conf.setClassLoader(original) + ss = new SessionState(conf) + SessionState.start(ss) + } + ss.getConf + } +} From 438e2a3dfbd1f1e5121aee8fdac3eb9c60d77f17 Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 4 Feb 2016 08:40:54 -0800 Subject: [PATCH 5/8] Fix order of import --- sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 1f5d910021996..fc749129dbcd4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -21,8 +21,8 @@ import java.util.Locale import scala.collection.JavaConverters._ -import org.apache.hadoop.hive.common.`type`.HiveDecimal import org.apache.hadoop.hive.client.HiveConfUtil +import org.apache.hadoop.hive.common.`type`.HiveDecimal import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.apache.hadoop.hive.ql.exec.{FunctionInfo, FunctionRegistry} From 0a43c19e0c89304bac31ca18c1127305a74bd38b Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 4 Feb 2016 09:04:43 -0800 Subject: [PATCH 6/8] Drop incorrect import --- sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index fc749129dbcd4..1336ff5e9b948 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -21,7 +21,6 @@ import java.util.Locale import scala.collection.JavaConverters._ -import org.apache.hadoop.hive.client.HiveConfUtil import org.apache.hadoop.hive.common.`type`.HiveDecimal import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.conf.HiveConf.ConfVars From f20a61f0c327b74130543e2034774035951c298e Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 4 Feb 2016 09:42:34 -0800 Subject: [PATCH 7/8] Remove parenthesis --- sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala | 2 +- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 1336ff5e9b948..0f62c424fa1bb 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -168,7 +168,7 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging /** * Returns the HiveConf */ - private[this] def hiveConf: HiveConf = HiveConfUtil.conf() + private[this] def hiveConf: HiveConf = HiveConfUtil.conf protected def getProperties(node: ASTNode): Seq[(String, String)] = node match { case Token("TOK_TABLEPROPLIST", list) => diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index f8dfb9b9b26aa..759bc15af170d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -134,7 +134,7 @@ private[hive] class HiveClientImpl( } /** Returns the configuration for the current session. */ - def conf: HiveConf = HiveConfUtil.conf() + def conf: HiveConf = HiveConfUtil.conf override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue) From 8b9e24c8ead6ef070e37656e65e5dcfeac79b11e Mon Sep 17 00:00:00 2001 From: tedyu Date: Fri, 5 Feb 2016 10:44:52 -0800 Subject: [PATCH 8/8] Revert to v1 --- .../org/apache/spark/sql/hive/HiveQl.scala | 13 +++++- .../sql/hive/client/HiveClientImpl.scala | 13 +++++- .../spark/sql/hive/client/HiveConfUtil.scala | 43 ------------------- 3 files changed, 24 insertions(+), 45 deletions(-) delete mode 100644 sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 0f62c424fa1bb..752c037a842a8 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -168,7 +168,18 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging /** * Returns the HiveConf */ - private[this] def hiveConf: HiveConf = HiveConfUtil.conf + private[this] def hiveConf: HiveConf = { + var ss = SessionState.get() + // SessionState is lazy initialization, it can be null here + if (ss == null) { + val original = Thread.currentThread().getContextClassLoader + val conf = new HiveConf(classOf[SessionState]) + conf.setClassLoader(original) + ss = new SessionState(conf) + SessionState.start(ss) + } + ss.getConf + } protected def getProperties(node: ASTNode): Seq[(String, String)] = node match { case Token("TOK_TABLEPROPLIST", list) => diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 759bc15af170d..4e51f29c1cf66 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -134,7 +134,18 @@ private[hive] class HiveClientImpl( } /** Returns the configuration for the current session. */ - def conf: HiveConf = HiveConfUtil.conf + def conf: HiveConf = { + var ss = SessionState.get() + // SessionState is lazy initialization, it can be null here + if (ss == null) { + val original = Thread.currentThread().getContextClassLoader + val conf = new HiveConf(classOf[SessionState]) + conf.setClassLoader(original) + ss = new SessionState(conf) + SessionState.start(ss) + } + ss.getConf + } override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala deleted file mode 100644 index b36307b459a6a..0000000000000 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveConfUtil.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hive.client - -import scala.collection.JavaConverters._ -import scala.language.reflectiveCalls - -import org.apache.hadoop.hive.conf.HiveConf -import org.apache.hadoop.hive.ql.session.SessionState - -/** - * Configuration utility for Hive - */ -object HiveConfUtil { - /** Returns the configuration for the current session. */ - def conf: HiveConf = { - var ss = SessionState.get() - // SessionState is lazy initialization, it can be null here - if (ss == null) { - val original = Thread.currentThread().getContextClassLoader - val conf = new HiveConf(classOf[SessionState]) - conf.setClassLoader(original) - ss = new SessionState(conf) - SessionState.start(ss) - } - ss.getConf - } -}