From 4e951314b473f8ac98335c05a15dca5907e8d769 Mon Sep 17 00:00:00 2001 From: Burak Yavuz Date: Fri, 19 Aug 2016 08:50:35 -0700 Subject: [PATCH 1/4] get patch --- .../test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala index 162c0b56c6e11..5f662ebf83e14 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala @@ -70,7 +70,7 @@ class MultiSQLContextsSuite extends SparkFunSuite with BeforeAndAfterAll { val message = intercept[SparkException] { new SQLContext(sparkContext) }.getMessage - assert(message.contains("Only one SQLContext/HiveContext may be running")) + assert(message.contains("the shared context can be accessed as sqlContext")) } } finally { sparkContext.stop() From 5b54c8a415f285210d458afa915ce9b18a3184c0 Mon Sep 17 00:00:00 2001 From: Burak Yavuz Date: Mon, 22 Aug 2016 11:12:44 -0700 Subject: [PATCH 2/4] make requestTotalExecutors public API --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 8f5689cae84d1..9a6840d1328f8 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1413,7 +1413,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli * This includes running, pending, and completed tasks. * @return whether the request is acknowledged by the cluster manager. */ - private[spark] override def requestTotalExecutors( + override def requestTotalExecutors( numExecutors: Int, localityAwareTasks: Int, hostToLocalTaskCount: scala.collection.immutable.Map[String, Int] From 432c4b3e597eed06e97c7d6d88218fdd7c67ca6f Mon Sep 17 00:00:00 2001 From: Burak Yavuz Date: Tue, 13 Sep 2016 14:29:46 -0700 Subject: [PATCH 3/4] save --- .../apache/spark/sql/hive/HiveContext.scala | 7 ++++ .../spark/sql/hive/HiveContextSuite.scala | 32 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index e83941c2ecf66..fa644405de732 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -753,6 +753,13 @@ private[hive] object HiveContext { // hive.metastore.uris is not set. propMap.put(ConfVars.METASTOREURIS.varname, "") + // The execution client will generate garbage events, therefore the listeners that are generated + // for the execution clients are useless. In order to not output garbage, we don't generate + // these listeners. + propMap.put(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, "") + propMap.put(ConfVars.METASTORE_EVENT_LISTENERS.varname, "") + propMap.put(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname, "") + propMap.toMap } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala new file mode 100644 index 0000000000000..048a0c019b16f --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars + +import org.apache.spark.sql.hive.test.TestHiveSingleton +import org.apache.spark.sql.QueryTest + +class HiveContextSuite extends QueryTest with TestHiveSingleton { + test("newTemporaryConfiguration overwrites listener configurations") { + val conf = HiveContext.newTemporaryConfiguration() + assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "") + assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "") + assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "") + } +} From 0b3c5bca00a2adcadbe1a2d8da04de270a5b2163 Mon Sep 17 00:00:00 2001 From: Burak Yavuz Date: Tue, 13 Sep 2016 14:32:05 -0700 Subject: [PATCH 4/4] revert other changes --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- .../test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 9a6840d1328f8..8f5689cae84d1 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1413,7 +1413,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli * This includes running, pending, and completed tasks. * @return whether the request is acknowledged by the cluster manager. */ - override def requestTotalExecutors( + private[spark] override def requestTotalExecutors( numExecutors: Int, localityAwareTasks: Int, hostToLocalTaskCount: scala.collection.immutable.Map[String, Int] diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala index 5f662ebf83e14..162c0b56c6e11 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MultiSQLContextsSuite.scala @@ -70,7 +70,7 @@ class MultiSQLContextsSuite extends SparkFunSuite with BeforeAndAfterAll { val message = intercept[SparkException] { new SQLContext(sparkContext) }.getMessage - assert(message.contains("the shared context can be accessed as sqlContext")) + assert(message.contains("Only one SQLContext/HiveContext may be running")) } } finally { sparkContext.stop()