From 0740f28b04a43ac739d6f45a7ffc6fa23fe7b96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Tue, 1 Jul 2014 11:29:12 +0800 Subject: [PATCH 1/9] Update SQLConf.scala use concurrent.ConcurrentHashMap instead of util.Collections.synchronizedMap --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index 2fe7f9466399..2d037cbcfdc1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -50,8 +50,7 @@ trait SQLConf { /** ********************** SQLConf functionality methods ************ */ @transient - private val settings = java.util.Collections.synchronizedMap( - new java.util.HashMap[String, String]()) + private val settings = new java.util.concurrent.ConcurrentHashMap[String, String]() def set(props: Properties): Unit = { props.asScala.foreach { case (k, v) => this.settings.put(k, v) } From 1d3e4a29940c1e9c8fef2d4d683d519a45bea51d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Tue, 1 Jul 2014 14:36:57 +0800 Subject: [PATCH 2/9] Update SQLConf.scala add some synchronized --- .../main/scala/org/apache/spark/sql/SQLConf.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index 2d037cbcfdc1..add50a6bced3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -53,7 +53,9 @@ trait SQLConf { private val settings = new java.util.concurrent.ConcurrentHashMap[String, String]() def set(props: Properties): Unit = { - props.asScala.foreach { case (k, v) => this.settings.put(k, v) } + settings.synchronized { + props.asScala.foreach { case (k, v) => this.settings.put(k, v) } + } } def set(key: String, value: String): Unit = { @@ -63,20 +65,21 @@ trait SQLConf { } def get(key: String): String = { - if (!settings.containsKey(key)) { - throw new NoSuchElementException(key) - } settings.get(key) } def get(key: String, defaultValue: String): String = { - if (!settings.containsKey(key)) defaultValue else settings.get(key) + settings.synchronized { + if (!settings.containsKey(key)) defaultValue else settings.get(key) + } } def getAll: Array[(String, String)] = settings.asScala.toArray def getOption(key: String): Option[String] = { - if (!settings.containsKey(key)) None else Some(settings.get(key)) + settings.synchronized { + if (!settings.containsKey(key)) None else Some(settings.get(key)) + } } def contains(key: String): Boolean = settings.containsKey(key) From 843581c57d987e1001254886e681a60594cef590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Tue, 1 Jul 2014 15:27:56 +0800 Subject: [PATCH 3/9] Update SQLConf.scala --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index add50a6bced3..b7a9f6f3dc63 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -53,9 +53,7 @@ trait SQLConf { private val settings = new java.util.concurrent.ConcurrentHashMap[String, String]() def set(props: Properties): Unit = { - settings.synchronized { props.asScala.foreach { case (k, v) => this.settings.put(k, v) } - } } def set(key: String, value: String): Unit = { @@ -70,7 +68,7 @@ trait SQLConf { def get(key: String, defaultValue: String): String = { settings.synchronized { - if (!settings.containsKey(key)) defaultValue else settings.get(key) + Option(settings.get(key)).getOrElse(defaultValue) } } @@ -78,7 +76,7 @@ trait SQLConf { def getOption(key: String): Option[String] = { settings.synchronized { - if (!settings.containsKey(key)) None else Some(settings.get(key)) + Option(settings.get(key)) } } From 7304d9bbd3ce2319c1ceaf9ec31637eeb6fa5ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Tue, 1 Jul 2014 15:34:28 +0800 Subject: [PATCH 4/9] Update SQLConf.scala --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index b7a9f6f3dc63..9da5f65d61d6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -67,17 +67,13 @@ trait SQLConf { } def get(key: String, defaultValue: String): String = { - settings.synchronized { Option(settings.get(key)).getOrElse(defaultValue) - } } def getAll: Array[(String, String)] = settings.asScala.toArray def getOption(key: String): Option[String] = { - settings.synchronized { Option(settings.get(key)) - } } def contains(key: String): Boolean = settings.containsKey(key) From 593096bfd92dbdc36ed76fa999331b7e491a2418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Tue, 1 Jul 2014 15:50:17 +0800 Subject: [PATCH 5/9] Update SQLConf.scala --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index 9da5f65d61d6..ee7928228ab4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -53,7 +53,7 @@ trait SQLConf { private val settings = new java.util.concurrent.ConcurrentHashMap[String, String]() def set(props: Properties): Unit = { - props.asScala.foreach { case (k, v) => this.settings.put(k, v) } + props.asScala.foreach { case (k, v) => this.settings.put(k, v) } } def set(key: String, value: String): Unit = { @@ -67,13 +67,13 @@ trait SQLConf { } def get(key: String, defaultValue: String): String = { - Option(settings.get(key)).getOrElse(defaultValue) + Option(settings.get(key)).getOrElse(defaultValue) } def getAll: Array[(String, String)] = settings.asScala.toArray def getOption(key: String): Option[String] = { - Option(settings.get(key)) + Option(settings.get(key)) } def contains(key: String): Boolean = settings.containsKey(key) From 864faa0d14f8975a96b02eb1d3593216cea1978d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Tue, 1 Jul 2014 16:05:37 +0800 Subject: [PATCH 6/9] Update SQLConf.scala --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index ee7928228ab4..708b9437e281 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -50,8 +50,8 @@ trait SQLConf { /** ********************** SQLConf functionality methods ************ */ @transient - private val settings = new java.util.concurrent.ConcurrentHashMap[String, String]() - + private val settings = java.util.Collections.synchronizedMap( + new java.util.HashMap[String, String]()) def set(props: Properties): Unit = { props.asScala.foreach { case (k, v) => this.settings.put(k, v) } } From 36b6dbda8638b2cb69333aa05165a4d60644183e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Tue, 1 Jul 2014 16:06:56 +0800 Subject: [PATCH 7/9] Update SQLConf.scala --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index 708b9437e281..25e42719815c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -52,6 +52,7 @@ trait SQLConf { @transient private val settings = java.util.Collections.synchronizedMap( new java.util.HashMap[String, String]()) + def set(props: Properties): Unit = { props.asScala.foreach { case (k, v) => this.settings.put(k, v) } } From 63da04353d906b2398ad9d4e9608410a9105334e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Thu, 3 Jul 2014 11:58:23 +0800 Subject: [PATCH 8/9] Update SQLConf.scala --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index 25e42719815c..b0ac3eb8b12f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -64,7 +64,7 @@ trait SQLConf { } def get(key: String): String = { - settings.get(key) + Option(settings.get(key)).orElse(throw new NoSuchElementException(key)) } def get(key: String, defaultValue: String): String = { From 51ec55d62e39487a6a3f0dadddac0b8d63fec63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?baishuo=28=E7=99=BD=E7=A1=95=29?= Date: Fri, 4 Jul 2014 11:11:39 +0800 Subject: [PATCH 9/9] Update SQLConf.scala --- sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index b0ac3eb8b12f..66b31a9084de 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -64,7 +64,7 @@ trait SQLConf { } def get(key: String): String = { - Option(settings.get(key)).orElse(throw new NoSuchElementException(key)) + Option(settings.get(key)).getOrElse(throw new NoSuchElementException(key)) } def get(key: String, defaultValue: String): String = {