From 4f8368ea727e5689e96794884b8d1baf3eccb5d5 Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sat, 25 Feb 2017 14:36:31 -0800 Subject: [PATCH 01/13] allow setting docker user when running spark on mesos with docker containerizer --- .../mesos/MesosSchedulerBackendUtil.scala | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala index a2adb228dc29..050a10f16c99 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala @@ -120,8 +120,12 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { .map(parsePortMappingsSpec) .getOrElse(List.empty) + val executorUser = conf + .getOption("spark.mesos.executor.docker.user") + if (containerType == ContainerInfo.Type.DOCKER) { - containerInfo.setDocker(dockerInfo(image, forcePullImage, portMaps)) + containerInfo + .setDocker(dockerInfo(image, forcePullImage, portMaps, executorUser)) } else { containerInfo.setMesos(mesosInfo(image, forcePullImage)) } @@ -144,12 +148,21 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { private def dockerInfo( image: String, forcePullImage: Boolean, - portMaps: List[ContainerInfo.DockerInfo.PortMapping]): DockerInfo = { + portMaps: List[ContainerInfo.DockerInfo.PortMapping], + executorUser: Option[String] = None): DockerInfo = { val dockerBuilder = ContainerInfo.DockerInfo.newBuilder() .setImage(image) .setForcePullImage(forcePullImage) portMaps.foreach(dockerBuilder.addPortMappings(_)) + if (!executorUser.isEmpty) { + val parameter: DockerInfo.Parameter.Builder = DockerInfo.Parameter + .newBuilder() + .setKey("user") + .setValue(executorUser.get()) + dockerBuilder.addParameter(parameter) + } + dockerBuilder.build } From bba57f9491703b4b06e82144a57660cbafa193ee Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sat, 25 Feb 2017 17:34:49 -0800 Subject: [PATCH 02/13] allow arbitrary parameters to pass to docker through spark conf --- .../mesos/MesosSchedulerBackendUtil.scala | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala index 050a10f16c99..3f952067eec8 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala @@ -99,6 +99,27 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { .toList } + /** + * Parse a comma-delimited list of arbitrary parameters, each of which + * takes the form key=value + */ + def parseParamsSpec(params: String): List[DockerInfo.Parameter] = { + params.split(",").map(_.split("=")).flatMap { kv: Array[String] => + val param: DockerInfo.Parameter.Builder = DockerInfo.Parameter + .newBuilder() + kv match { + case Array(key, value) => + Some(param.setKey(key).setValue(value)) + case kv => + logWarning(s"Unable to parse arbitary parameters: $params. " + + "Expected form: \"key=value(, ...)\"") + None + } + } + .map { _.build() } + .toList + } + def containerInfo(conf: SparkConf): ContainerInfo = { val containerType = if (conf.contains("spark.mesos.executor.docker.image") && conf.get("spark.mesos.containerizer", "docker") == "docker") { @@ -120,12 +141,14 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { .map(parsePortMappingsSpec) .getOrElse(List.empty) - val executorUser = conf - .getOption("spark.mesos.executor.docker.user") + val params = conf + .getOption("spark.mesos.executor.docker.params") + .map(parseParamsSpec) + .getOrElse(List.empty) if (containerType == ContainerInfo.Type.DOCKER) { containerInfo - .setDocker(dockerInfo(image, forcePullImage, portMaps, executorUser)) + .setDocker(dockerInfo(image, forcePullImage, portMaps, params)) } else { containerInfo.setMesos(mesosInfo(image, forcePullImage)) } @@ -149,19 +172,12 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { image: String, forcePullImage: Boolean, portMaps: List[ContainerInfo.DockerInfo.PortMapping], - executorUser: Option[String] = None): DockerInfo = { + params: List[ContainerInfo.DockerInfo.Parameter]): DockerInfo = { val dockerBuilder = ContainerInfo.DockerInfo.newBuilder() .setImage(image) .setForcePullImage(forcePullImage) portMaps.foreach(dockerBuilder.addPortMappings(_)) - - if (!executorUser.isEmpty) { - val parameter: DockerInfo.Parameter.Builder = DockerInfo.Parameter - .newBuilder() - .setKey("user") - .setValue(executorUser.get()) - dockerBuilder.addParameter(parameter) - } + params.foreach(dockerBuilder.addParameter(_)) dockerBuilder.build } From ae30e239e574cebc9774087e038aa0853d9939fc Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sun, 26 Feb 2017 13:32:45 -0800 Subject: [PATCH 03/13] add some debug prints --- .../mesos/MesosSchedulerBackendUtil.scala | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala index 3f952067eec8..4fde78382f01 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala @@ -17,7 +17,7 @@ package org.apache.spark.scheduler.cluster.mesos -import org.apache.mesos.Protos.{ContainerInfo, Image, NetworkInfo, Volume} +import org.apache.mesos.Protos.{ContainerInfo, Image, NetworkInfo, Parameter, Volume} import org.apache.mesos.Protos.ContainerInfo.{DockerInfo, MesosInfo} import org.apache.spark.{SparkConf, SparkException} @@ -103,10 +103,13 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { * Parse a comma-delimited list of arbitrary parameters, each of which * takes the form key=value */ - def parseParamsSpec(params: String): List[DockerInfo.Parameter] = { + def parseParamsSpec(params: String): List[Parameter] = { + logWarning(s"DEBUG: parseParamsSpec() params=$params") params.split(",").map(_.split("=")).flatMap { kv: Array[String] => - val param: DockerInfo.Parameter.Builder = DockerInfo.Parameter - .newBuilder() + val param: Parameter.Builder = Parameter.newBuilder() + val k = kv(0) + val v = kv(1) + logWarning(s"DEBUG: k=$k, v=$v") kv match { case Array(key, value) => Some(param.setKey(key).setValue(value)) @@ -146,6 +149,9 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { .map(parseParamsSpec) .getOrElse(List.empty) + val size = params.length + logWarning(s"DEBUG: param size is $size") + if (containerType == ContainerInfo.Type.DOCKER) { containerInfo .setDocker(dockerInfo(image, forcePullImage, portMaps, params)) @@ -172,12 +178,12 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { image: String, forcePullImage: Boolean, portMaps: List[ContainerInfo.DockerInfo.PortMapping], - params: List[ContainerInfo.DockerInfo.Parameter]): DockerInfo = { + params: List[Parameter]): DockerInfo = { val dockerBuilder = ContainerInfo.DockerInfo.newBuilder() .setImage(image) .setForcePullImage(forcePullImage) portMaps.foreach(dockerBuilder.addPortMappings(_)) - params.foreach(dockerBuilder.addParameter(_)) + params.foreach(dockerBuilder.addParameters(_)) dockerBuilder.build } From ecb7a8e87589d4b72fe836f91cdf4d8a7e5a53bc Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Tue, 28 Feb 2017 17:54:34 -0800 Subject: [PATCH 04/13] remove debug print --- .../cluster/mesos/MesosSchedulerBackendUtil.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala index 4fde78382f01..43ae1e98dc9a 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala @@ -104,12 +104,8 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { * takes the form key=value */ def parseParamsSpec(params: String): List[Parameter] = { - logWarning(s"DEBUG: parseParamsSpec() params=$params") params.split(",").map(_.split("=")).flatMap { kv: Array[String] => val param: Parameter.Builder = Parameter.newBuilder() - val k = kv(0) - val v = kv(1) - logWarning(s"DEBUG: k=$k, v=$v") kv match { case Array(key, value) => Some(param.setKey(key).setValue(value)) @@ -149,9 +145,6 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { .map(parseParamsSpec) .getOrElse(List.empty) - val size = params.length - logWarning(s"DEBUG: param size is $size") - if (containerType == ContainerInfo.Type.DOCKER) { containerInfo .setDocker(dockerInfo(image, forcePullImage, portMaps, params)) From 408793696d354170c7944945caabda586ff22a7e Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sun, 5 Mar 2017 10:14:32 -0800 Subject: [PATCH 05/13] add test --- .../mesos/MesosSchedulerUtilsSuite.scala | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala index ec47ab153177..e2399e9a652d 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala @@ -73,6 +73,7 @@ class MesosSchedulerUtilsSuite extends SparkFunSuite with Matchers with MockitoS } val utils = new MesosSchedulerUtils { } + // scalastyle:on structural.type test("use at-least minimum overhead") { @@ -253,4 +254,33 @@ class MesosSchedulerUtilsSuite extends SparkFunSuite with Matchers with MockitoS val portsToUse = getRangesFromResources(resourcesToBeUsed).map{r => r._1} portsToUse.isEmpty shouldBe true } + + test("Parse arbitrary parameter to pass into docker containerizer") { + val parsed = MesosSchedulerBackendUtil.parseParamsSpec("a=1,b=2,c=3") + parsed(0).getKey shouldBe "a" + parsed(0).getValue shouldBe "1" + parsed(1).getKey shouldBe "b" + parsed(1).getValue shouldBe "2" + parsed(2).getKey shouldBe "c" + parsed(2).getValue shouldBe "3" + + val invalid = MesosSchedulerBackendUtil.parseParamsSpec("a,b") + invalid.length shouldBe 0 + } + + test("ContainerInfo contains parsed arbitrary parameters") { + val conf = new SparkConf() + conf.set("spark.mesos.executor.docker.params", "a=1,b=2,c=3") + conf.set("spark.mesos.executor.docker.image", "test") + + val containerInfo = MesosSchedulerBackendUtil.containerInfo(conf) + val params = containerInfo.getDocker.getParametersList + params.size() shouldBe 3 + params.get(0).getKey shouldBe "a" + params.get(0).getValue shouldBe "1" + params.get(1).getKey shouldBe "b" + params.get(1).getValue shouldBe "2" + params.get(2).getKey shouldBe "c" + params.get(2).getValue shouldBe "3" + } } From 54f9ec802002e702f1c3954d97be47bcdb5f7786 Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sun, 5 Mar 2017 10:38:10 -0800 Subject: [PATCH 06/13] add MesosSchedulerBackendUtilSuite.scala --- .../MesosSchedulerBackendUtilSuite.scala | 54 +++++++++++++++++++ .../mesos/MesosSchedulerUtilsSuite.scala | 30 ----------- 2 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala new file mode 100644 index 000000000000..e9d5e5264c9e --- /dev/null +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.scheduler.cluster.mesos + +import org.scalatest._ +import org.scalatest.mock.MockitoSugar +import org.apache.spark.{SparkConf, SparkFunSuite} + +class MesosSchedulerBackendUtilSuite extends SparkFunSuite with Matchers with MockitoSugar { + + test("Parse arbitrary parameter to pass into docker containerizer") { + val parsed = MesosSchedulerBackendUtil.parseParamsSpec("a=1,b=2,c=3") + parsed(0).getKey shouldBe "a" + parsed(0).getValue shouldBe "1" + parsed(1).getKey shouldBe "b" + parsed(1).getValue shouldBe "2" + parsed(2).getKey shouldBe "c" + parsed(2).getValue shouldBe "3" + + val invalid = MesosSchedulerBackendUtil.parseParamsSpec("a,b") + invalid.length shouldBe 0 + } + + test("ContainerInfo contains parsed arbitrary parameters") { + val conf = new SparkConf() + conf.set("spark.mesos.executor.docker.params", "a=1,b=2,c=3") + conf.set("spark.mesos.executor.docker.image", "test") + + val containerInfo = MesosSchedulerBackendUtil.containerInfo(conf) + val params = containerInfo.getDocker.getParametersList + params.size() shouldBe 3 + params.get(0).getKey shouldBe "a" + params.get(0).getValue shouldBe "1" + params.get(1).getKey shouldBe "b" + params.get(1).getValue shouldBe "2" + params.get(2).getKey shouldBe "c" + params.get(2).getValue shouldBe "3" + } +} \ No newline at end of file diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala index e2399e9a652d..ec47ab153177 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtilsSuite.scala @@ -73,7 +73,6 @@ class MesosSchedulerUtilsSuite extends SparkFunSuite with Matchers with MockitoS } val utils = new MesosSchedulerUtils { } - // scalastyle:on structural.type test("use at-least minimum overhead") { @@ -254,33 +253,4 @@ class MesosSchedulerUtilsSuite extends SparkFunSuite with Matchers with MockitoS val portsToUse = getRangesFromResources(resourcesToBeUsed).map{r => r._1} portsToUse.isEmpty shouldBe true } - - test("Parse arbitrary parameter to pass into docker containerizer") { - val parsed = MesosSchedulerBackendUtil.parseParamsSpec("a=1,b=2,c=3") - parsed(0).getKey shouldBe "a" - parsed(0).getValue shouldBe "1" - parsed(1).getKey shouldBe "b" - parsed(1).getValue shouldBe "2" - parsed(2).getKey shouldBe "c" - parsed(2).getValue shouldBe "3" - - val invalid = MesosSchedulerBackendUtil.parseParamsSpec("a,b") - invalid.length shouldBe 0 - } - - test("ContainerInfo contains parsed arbitrary parameters") { - val conf = new SparkConf() - conf.set("spark.mesos.executor.docker.params", "a=1,b=2,c=3") - conf.set("spark.mesos.executor.docker.image", "test") - - val containerInfo = MesosSchedulerBackendUtil.containerInfo(conf) - val params = containerInfo.getDocker.getParametersList - params.size() shouldBe 3 - params.get(0).getKey shouldBe "a" - params.get(0).getValue shouldBe "1" - params.get(1).getKey shouldBe "b" - params.get(1).getValue shouldBe "2" - params.get(2).getKey shouldBe "c" - params.get(2).getValue shouldBe "3" - } } From 423dfa8dd612df922bc977da6cde04bb9473fe65 Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sun, 5 Mar 2017 14:33:09 -0800 Subject: [PATCH 07/13] add newline --- .../cluster/mesos/MesosSchedulerBackendUtilSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala index e9d5e5264c9e..3c4044be5ff4 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala @@ -51,4 +51,4 @@ class MesosSchedulerBackendUtilSuite extends SparkFunSuite with Matchers with Mo params.get(2).getKey shouldBe "c" params.get(2).getValue shouldBe "3" } -} \ No newline at end of file +} From 0696d4f5bb43b5a6aec37700eb34dae375f611e9 Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sun, 5 Mar 2017 14:47:58 -0800 Subject: [PATCH 08/13] add empty line --- .../scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala index 3c4044be5ff4..209bef186fcb 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.scheduler.cluster.mesos import org.scalatest._ import org.scalatest.mock.MockitoSugar + import org.apache.spark.{SparkConf, SparkFunSuite} class MesosSchedulerBackendUtilSuite extends SparkFunSuite with Matchers with MockitoSugar { From 03e89eb2bae3b08207429a6b772d7dcae45b554c Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Thu, 9 Mar 2017 21:56:22 -0800 Subject: [PATCH 09/13] add documentation --- docs/running-on-mesos.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 8d5ad12cb85b..91a371956b94 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -356,6 +356,16 @@ See the [configuration page](configuration.html) for information on Spark config By default Mesos agents will not pull images they already have cached. + + spark.mesos.executor.docker.params + (none) + + Set the list of custom parameters which will be passed into the docker run command when launching the Spark executor on mesos using docker containerizer. The format of this property is a comma-separated list of + key/value pairs. That is they take the form: + + 
key1=val1,key2=val2,key3=val3
+ + spark.mesos.executor.docker.volumes (none) From 3417721ed237a039f5c3fd9aa5d473aa5b4b551f Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sat, 11 Mar 2017 18:15:24 -0800 Subject: [PATCH 10/13] address comments --- docs/running-on-mesos.md | 6 +-- .../mesos/MesosSchedulerBackendUtil.scala | 12 +++--- .../MesosSchedulerBackendUtilSuite.scala | 38 +++++++++---------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 91a371956b94..bd4963a08bfd 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -357,11 +357,11 @@ See the [configuration page](configuration.html) for information on Spark config - spark.mesos.executor.docker.params + spark.mesos.executor.docker.parameters (none) - Set the list of custom parameters which will be passed into the docker run command when launching the Spark executor on mesos using docker containerizer. The format of this property is a comma-separated list of - key/value pairs. That is they take the form: + Set the list of custom parameters which will be passed into the docker run command when launching the Spark executor on Mesos using the docker containerizer. The format of this property is a comma-separated list of + key/value pairs. Example: 
key1=val1,key2=val2,key3=val3
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala index 43ae1e98dc9a..be772111c205 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala @@ -100,16 +100,16 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { } /** - * Parse a comma-delimited list of arbitrary parameters, each of which + * Parse a list of docker parameters, each of which * takes the form key=value */ - def parseParamsSpec(params: String): List[Parameter] = { - params.split(",").map(_.split("=")).flatMap { kv: Array[String] => + private def parseParamsSpec(params: String): List[Parameter] = { + params.split(",").map(_.split("=")).flatMap { spec: Array[String] => val param: Parameter.Builder = Parameter.newBuilder() - kv match { + spec match { case Array(key, value) => Some(param.setKey(key).setValue(value)) - case kv => + case spec => logWarning(s"Unable to parse arbitary parameters: $params. " + "Expected form: \"key=value(, ...)\"") None @@ -141,7 +141,7 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { .getOrElse(List.empty) val params = conf - .getOption("spark.mesos.executor.docker.params") + .getOption("spark.mesos.executor.docker.parameters") .map(parseParamsSpec) .getOrElse(List.empty) diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala index 209bef186fcb..8f1cb82ccb3e 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala @@ -24,32 +24,30 @@ import org.apache.spark.{SparkConf, SparkFunSuite} class MesosSchedulerBackendUtilSuite extends SparkFunSuite with Matchers with MockitoSugar { - test("Parse arbitrary parameter to pass into docker containerizer") { - val parsed = MesosSchedulerBackendUtil.parseParamsSpec("a=1,b=2,c=3") - parsed(0).getKey shouldBe "a" - parsed(0).getValue shouldBe "1" - parsed(1).getKey shouldBe "b" - parsed(1).getValue shouldBe "2" - parsed(2).getKey shouldBe "c" - parsed(2).getValue shouldBe "3" - - val invalid = MesosSchedulerBackendUtil.parseParamsSpec("a,b") - invalid.length shouldBe 0 + test("ContainerInfo fails to parse invalid docker parameters") { + val conf = new SparkConf() + conf.set("spark.mesos.executor.docker.parameters", "a,b") + conf.set("spark.mesos.executor.docker.image", "test") + + val containerInfo = MesosSchedulerBackendUtil.containerInfo(conf) + val params = containerInfo.getDocker.getParametersList + + assert(params.length == 0) } - test("ContainerInfo contains parsed arbitrary parameters") { + test("ContainerInfo parses docker parameters") { val conf = new SparkConf() - conf.set("spark.mesos.executor.docker.params", "a=1,b=2,c=3") + conf.set("spark.mesos.executor.docker.parameters", "a=1,b=2,c=3") conf.set("spark.mesos.executor.docker.image", "test") val containerInfo = MesosSchedulerBackendUtil.containerInfo(conf) val params = containerInfo.getDocker.getParametersList - params.size() shouldBe 3 - params.get(0).getKey shouldBe "a" - params.get(0).getValue shouldBe "1" - params.get(1).getKey shouldBe "b" - params.get(1).getValue shouldBe "2" - params.get(2).getKey shouldBe "c" - params.get(2).getValue shouldBe "3" + assert(params.size() == 3) + assert(params.get(0).getKey == "a") + assert(params.get(0).getValue == "1") + assert(params.get(1).getKey == "b") + assert(params.get(1).getValue == "2") + assert(params.get(2).getKey == "c") + assert(params.get(2).getValue == "3") } } From 737acf07ceea8f4bc92b9eaa8c572af19b2e0b88 Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sat, 11 Mar 2017 18:19:35 -0800 Subject: [PATCH 11/13] address comment/syntax fix --- .../cluster/mesos/MesosSchedulerBackendUtilSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala index 8f1cb82ccb3e..491d33e9fc1f 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala @@ -32,7 +32,7 @@ class MesosSchedulerBackendUtilSuite extends SparkFunSuite with Matchers with Mo val containerInfo = MesosSchedulerBackendUtil.containerInfo(conf) val params = containerInfo.getDocker.getParametersList - assert(params.length == 0) + assert(params.size() == 0) } test("ContainerInfo parses docker parameters") { From cbb784a1a278f2d0db5c5122d52c30dfd26fc3db Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Mon, 13 Mar 2017 22:27:47 -0700 Subject: [PATCH 12/13] remove unnecessary interface from test --- .../cluster/mesos/MesosSchedulerBackendUtilSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala index 491d33e9fc1f..caf9d89fdd20 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala @@ -22,7 +22,7 @@ import org.scalatest.mock.MockitoSugar import org.apache.spark.{SparkConf, SparkFunSuite} -class MesosSchedulerBackendUtilSuite extends SparkFunSuite with Matchers with MockitoSugar { +class MesosSchedulerBackendUtilSuite extends SparkFunSuite { test("ContainerInfo fails to parse invalid docker parameters") { val conf = new SparkConf() From 2f3f8b20942e8d24d582ad69a6ece1377b33ec65 Mon Sep 17 00:00:00 2001 From: Ji Yan Date: Sat, 15 Apr 2017 12:22:31 -0700 Subject: [PATCH 13/13] guard against having equality sign inside the parameter value --- .../scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala index be772111c205..fbcbc55099ec 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala @@ -104,7 +104,9 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging { * takes the form key=value */ private def parseParamsSpec(params: String): List[Parameter] = { - params.split(",").map(_.split("=")).flatMap { spec: Array[String] => + // split with limit of 2 to avoid parsing error when '=' + // exists in the parameter value + params.split(",").map(_.split("=", 2)).flatMap { spec: Array[String] => val param: Parameter.Builder = Parameter.newBuilder() spec match { case Array(key, value) =>