From 0556792ceb5ae75074526b9f2f2cdb4887724bec Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Tue, 17 Mar 2015 15:21:34 +0900 Subject: [PATCH 01/11] [SPARK-6350][Mesos] Make mesosExecutorCores configurable in mesos "fine-grained" mode - Defined executorCores from "spark.mesos.executor.cores" - Changed the amount of mesosExecutor's cores to executorCores. - Added new configuration option on running-on-mesos.md --- .../scheduler/cluster/mesos/MesosSchedulerBackend.scala | 9 +++++---- .../cluster/mesos/MesosSchedulerBackendSuite.scala | 4 ++-- docs/running-on-mesos.md | 8 ++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index b381436839227..f16263a7d0588 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -67,6 +67,8 @@ private[spark] class MesosSchedulerBackend( // The listener bus to publish executor added/removed events. val listenerBus = sc.listenerBus + + val executorCores = sc.conf.getInt("spark.mesos.executor.cores", 1) @volatile var appId: String = _ @@ -139,7 +141,7 @@ private[spark] class MesosSchedulerBackend( .setName("cpus") .setType(Value.Type.SCALAR) .setScalar(Value.Scalar.newBuilder() - .setValue(scheduler.CPUS_PER_TASK).build()) + .setValue(executorCores).build()) .build() val memory = Resource.newBuilder() .setName("mem") @@ -223,7 +225,7 @@ private[spark] class MesosSchedulerBackend( // TODO(pwendell): Should below be 1 + scheduler.CPUS_PER_TASK? (mem >= MemoryUtils.calculateTotalMemory(sc) && // need at least 1 for executor, 1 for task - cpus >= 2 * scheduler.CPUS_PER_TASK) || + cpus >= (executorCores + scheduler.CPUS_PER_TASK)) || (slaveIdsWithExecutors.contains(slaveId) && cpus >= scheduler.CPUS_PER_TASK) } @@ -234,8 +236,7 @@ private[spark] class MesosSchedulerBackend( } else { // If the executor doesn't exist yet, subtract CPU for executor // TODO(pwendell): Should below just subtract "1"? - getResource(o.getResourcesList, "cpus").toInt - - scheduler.CPUS_PER_TASK + getResource(o.getResourcesList, "cpus").toInt - executorCores } new WorkerOffer( o.getSlaveId.getValue, diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala index a311512e82c5e..334a09f9b07d3 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala @@ -118,12 +118,12 @@ class MesosSchedulerBackendSuite extends FunSuite with LocalSparkContext with Mo expectedWorkerOffers.append(new WorkerOffer( mesosOffers.get(0).getSlaveId.getValue, mesosOffers.get(0).getHostname, - 2 + minCpu - backend.executorCores )) expectedWorkerOffers.append(new WorkerOffer( mesosOffers.get(2).getSlaveId.getValue, mesosOffers.get(2).getHostname, - 2 + minCpu - backend.executorCores )) val taskDesc = new TaskDescription(1L, 0, "s1", "n1", 0, ByteBuffer.wrap(new Array[Byte](0))) when(taskScheduler.resourceOffers(expectedWorkerOffers)).thenReturn(Seq(Seq(taskDesc))) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index c984639bd34cf..353ebc3f02570 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -210,6 +210,14 @@ See the [configuration page](configuration.html) for information on Spark config Note that total amount of cores the executor will request in total will not exceed the spark.cores.max setting. + + spark.mesos.executor.cores + 1 + + Set the amount of cores to request for running a mesos executor. This setting is only used for Mesos fine-grained mode. + By default, executor will use the amount of cores even though no task is running on an executor. + + spark.mesos.executor.home driver side SPARK_HOME From 4b7c69e2a3eec71636629ca2df7a4b112e67d632 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 18 Mar 2015 17:23:04 +0900 Subject: [PATCH 02/11] [SPARK-6350][Mesos] Make mesosExecutorCores configurable in mesos "fine-grained" mode - Changed configruation name and description from "spark.mesos.executor.cores" to "spark.executor.frameworkCores" --- .../scheduler/cluster/mesos/MesosSchedulerBackend.scala | 2 +- docs/configuration.md | 8 ++++++++ docs/running-on-mesos.md | 8 -------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index f16263a7d0588..e1c2f293b6976 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -68,7 +68,7 @@ private[spark] class MesosSchedulerBackend( // The listener bus to publish executor added/removed events. val listenerBus = sc.listenerBus - val executorCores = sc.conf.getInt("spark.mesos.executor.cores", 1) + val executorCores = sc.conf.getInt("spark.executor.frameworkCores", 1) @volatile var appId: String = _ diff --git a/docs/configuration.md b/docs/configuration.md index d9e9e67026cbb..dc9c3bef75c03 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,6 +137,14 @@ of the most common options to set are: or in your default properties file. + spark.executor.frameworkCores + 1 + + Set the amount of cores allocated to the executor itself. This setting is currently used for Mesos fine-grained mode. + By default, executor will use the amount of cores even though no task is running on an executor. + + + spark.executor.memory 512m diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 353ebc3f02570..c984639bd34cf 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -210,14 +210,6 @@ See the [configuration page](configuration.html) for information on Spark config Note that total amount of cores the executor will request in total will not exceed the spark.cores.max setting. - - spark.mesos.executor.cores - 1 - - Set the amount of cores to request for running a mesos executor. This setting is only used for Mesos fine-grained mode. - By default, executor will use the amount of cores even though no task is running on an executor. - - spark.mesos.executor.home driver side SPARK_HOME From 5f3767eaf574c82a3b17e669fbbb5d9d54f3079c Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 22 Mar 2015 02:34:16 +0900 Subject: [PATCH 03/11] Revert "[SPARK-6350][Mesos] Make mesosExecutorCores configurable in mesos "fine-grained" mode" This reverts commit da3caa6a062d7dd59875b3f8b83febe0c95f5c1e. --- .../scheduler/cluster/mesos/MesosSchedulerBackend.scala | 2 +- docs/configuration.md | 8 -------- docs/running-on-mesos.md | 8 ++++++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index e1c2f293b6976..f16263a7d0588 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -68,7 +68,7 @@ private[spark] class MesosSchedulerBackend( // The listener bus to publish executor added/removed events. val listenerBus = sc.listenerBus - val executorCores = sc.conf.getInt("spark.executor.frameworkCores", 1) + val executorCores = sc.conf.getInt("spark.mesos.executor.cores", 1) @volatile var appId: String = _ diff --git a/docs/configuration.md b/docs/configuration.md index dc9c3bef75c03..d9e9e67026cbb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,14 +137,6 @@ of the most common options to set are: or in your default properties file. - spark.executor.frameworkCores - 1 - - Set the amount of cores allocated to the executor itself. This setting is currently used for Mesos fine-grained mode. - By default, executor will use the amount of cores even though no task is running on an executor. - - - spark.executor.memory 512m diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index c984639bd34cf..353ebc3f02570 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -210,6 +210,14 @@ See the [configuration page](configuration.html) for information on Spark config Note that total amount of cores the executor will request in total will not exceed the spark.cores.max setting. + + spark.mesos.executor.cores + 1 + + Set the amount of cores to request for running a mesos executor. This setting is only used for Mesos fine-grained mode. + By default, executor will use the amount of cores even though no task is running on an executor. + + spark.mesos.executor.home driver side SPARK_HOME From 1fe4c03d5be0ca3433cffae0c3394a1aea1c9e8f Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 25 Mar 2015 10:16:31 +0900 Subject: [PATCH 04/11] [SPARK-6453][Mesos] Some Mesos*Suite have a different package with their classes - Change available resources of cpus to integer value beacuse WorkerOffer support the amount cpus as integer value --- .../spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index f16263a7d0588..06d74e272177b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -68,7 +68,7 @@ private[spark] class MesosSchedulerBackend( // The listener bus to publish executor added/removed events. val listenerBus = sc.listenerBus - val executorCores = sc.conf.getInt("spark.mesos.executor.cores", 1) + val executorCores = sc.conf.getDouble("spark.mesos.executor.cores", 1) @volatile var appId: String = _ @@ -236,7 +236,7 @@ private[spark] class MesosSchedulerBackend( } else { // If the executor doesn't exist yet, subtract CPU for executor // TODO(pwendell): Should below just subtract "1"? - getResource(o.getResourcesList, "cpus").toInt - executorCores + (getResource(o.getResourcesList, "cpus") - executorCores).toInt } new WorkerOffer( o.getSlaveId.getValue, From c27efce5b6694c82e6fc626c18135d7f4dfddc10 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 25 Mar 2015 17:27:13 +0900 Subject: [PATCH 05/11] [SPARK-6453][Mesos] Some Mesos*Suite have a different package with their classes - Fixed Mesos*Suite for supporting integer WorkerOffers - Fixed Documentation --- .../scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala | 4 ++-- docs/running-on-mesos.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala index 334a09f9b07d3..f9223cea927b6 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala @@ -118,12 +118,12 @@ class MesosSchedulerBackendSuite extends FunSuite with LocalSparkContext with Mo expectedWorkerOffers.append(new WorkerOffer( mesosOffers.get(0).getSlaveId.getValue, mesosOffers.get(0).getHostname, - minCpu - backend.executorCores + (minCpu - backend.executorCores).toInt )) expectedWorkerOffers.append(new WorkerOffer( mesosOffers.get(2).getSlaveId.getValue, mesosOffers.get(2).getHostname, - minCpu - backend.executorCores + (minCpu - backend.executorCores).toInt )) val taskDesc = new TaskDescription(1L, 0, "s1", "n1", 0, ByteBuffer.wrap(new Array[Byte](0))) when(taskScheduler.resourceOffers(expectedWorkerOffers)).thenReturn(Seq(Seq(taskDesc))) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 353ebc3f02570..65d2beff7949e 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -212,7 +212,7 @@ See the [configuration page](configuration.html) for information on Spark config spark.mesos.executor.cores - 1 + 1.0 Set the amount of cores to request for running a mesos executor. This setting is only used for Mesos fine-grained mode. By default, executor will use the amount of cores even though no task is running on an executor. From 4ae7b0cf861ed8fe8c3d4f0a39f91478ad6bc83d Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 25 Mar 2015 17:27:54 +0900 Subject: [PATCH 06/11] [SPARK-6453][Mesos] Some Mesos*Suite have a different package with their classes - Removed TODO --- .../spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index 06d74e272177b..8f9f6ffeaf711 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -222,7 +222,6 @@ private[spark] class MesosSchedulerBackend( val mem = getResource(o.getResourcesList, "mem") val cpus = getResource(o.getResourcesList, "cpus") val slaveId = o.getSlaveId.getValue - // TODO(pwendell): Should below be 1 + scheduler.CPUS_PER_TASK? (mem >= MemoryUtils.calculateTotalMemory(sc) && // need at least 1 for executor, 1 for task cpus >= (executorCores + scheduler.CPUS_PER_TASK)) || @@ -235,7 +234,6 @@ private[spark] class MesosSchedulerBackend( getResource(o.getResourcesList, "cpus").toInt } else { // If the executor doesn't exist yet, subtract CPU for executor - // TODO(pwendell): Should below just subtract "1"? (getResource(o.getResourcesList, "cpus") - executorCores).toInt } new WorkerOffer( From 75493142b2ee1b3a0f6e558b0e0c1014cc6a9e2c Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 26 Mar 2015 11:46:39 +0900 Subject: [PATCH 07/11] [SPARK-6453][Mesos] Some Mesos*Suite have a different package with their classes - Fixed docs --- docs/running-on-mesos.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 65d2beff7949e..b0ea45dc6258a 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -214,8 +214,8 @@ See the [configuration page](configuration.html) for information on Spark config spark.mesos.executor.cores 1.0 - Set the amount of cores to request for running a mesos executor. This setting is only used for Mesos fine-grained mode. - By default, executor will use the amount of cores even though no task is running on an executor. + The setting, which can be a floating point number, controls the number of cores allocated + to the executor not for use by tasks. This setting is only used for Mesos fine-grained mode. From 8ba7694c4f2f2eeef55163eb380ea903a3f72275 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 27 Mar 2015 13:57:46 +0900 Subject: [PATCH 08/11] [SPARK-6350][Mesos] Make mesosExecutorCores configurable in mesos "fine-grained" mode - Fixed docs --- docs/running-on-mesos.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index b0ea45dc6258a..ce03ea11081d7 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -215,7 +215,8 @@ See the [configuration page](configuration.html) for information on Spark config 1.0 The setting, which can be a floating point number, controls the number of cores allocated - to the executor not for use by tasks. This setting is only used for Mesos fine-grained mode. + to the executor not for use by tasks. By default, executor will use the amount of cores + even though no task is running on an executor. This setting is only used for Mesos fine-grained mode. From 89edb4ff9ab5cef8c7c5b5776e77e7caf51cca67 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 27 Mar 2015 14:00:32 +0900 Subject: [PATCH 09/11] [SPARK-6350][Mesos] Make mesosExecutorCores configurable in mesos "fine-grained" mode - Fixed docs --- docs/running-on-mesos.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index ce03ea11081d7..94d7a85da2a05 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -214,9 +214,10 @@ See the [configuration page](configuration.html) for information on Spark config spark.mesos.executor.cores 1.0 - The setting, which can be a floating point number, controls the number of cores allocated - to the executor not for use by tasks. By default, executor will use the amount of cores - even though no task is running on an executor. This setting is only used for Mesos fine-grained mode. + Set the amount of cores, which can be a floating point number, to request for running a mesos executor. + The setting controls the number of cores allocated to the executor not for use by tasks. By default, + executor will use the amount of cores even though no task is running on an executor. This setting is + only used for Mesos fine-grained mode. From 2d4124110d36e04cb004688f1a7832ee0be5481b Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 30 Mar 2015 11:08:18 +0900 Subject: [PATCH 10/11] [SPARK-6350][Mesos] Make mesosExecutorCores configurable in mesos "fine-grained" mode - Fixed docs --- docs/running-on-mesos.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 94d7a85da2a05..0ed4391f3ee04 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -214,7 +214,7 @@ See the [configuration page](configuration.html) for information on Spark config spark.mesos.executor.cores 1.0 - Set the amount of cores, which can be a floating point number, to request for running a mesos executor. + Set the amount of cores, which can be a floating point number, to request for running a Mesos executor. The setting controls the number of cores allocated to the executor not for use by tasks. By default, executor will use the amount of cores even though no task is running on an executor. This setting is only used for Mesos fine-grained mode. From 9238d6e581e676e92e6cad95bc1e6fda1c4c031c Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 16 Apr 2015 22:19:30 +0900 Subject: [PATCH 11/11] [SPARK-6350][Mesos] Make mesosExecutorCores configurable in mesos "fine-grained" mode - Fixed docs - Changed configuration name - Made mesosExecutorCores private --- .../cluster/mesos/MesosSchedulerBackend.scala | 11 ++++++----- .../cluster/mesos/MesosSchedulerBackendSuite.scala | 4 ++-- docs/running-on-mesos.md | 10 +++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index 8f9f6ffeaf711..d9d62b0e287ed 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -68,7 +68,7 @@ private[spark] class MesosSchedulerBackend( // The listener bus to publish executor added/removed events. val listenerBus = sc.listenerBus - val executorCores = sc.conf.getDouble("spark.mesos.executor.cores", 1) + private[mesos] val mesosExecutorCores = sc.conf.getDouble("spark.mesos.mesosExecutor.cores", 1) @volatile var appId: String = _ @@ -141,7 +141,7 @@ private[spark] class MesosSchedulerBackend( .setName("cpus") .setType(Value.Type.SCALAR) .setScalar(Value.Scalar.newBuilder() - .setValue(executorCores).build()) + .setValue(mesosExecutorCores).build()) .build() val memory = Resource.newBuilder() .setName("mem") @@ -224,7 +224,7 @@ private[spark] class MesosSchedulerBackend( val slaveId = o.getSlaveId.getValue (mem >= MemoryUtils.calculateTotalMemory(sc) && // need at least 1 for executor, 1 for task - cpus >= (executorCores + scheduler.CPUS_PER_TASK)) || + cpus >= (mesosExecutorCores + scheduler.CPUS_PER_TASK)) || (slaveIdsWithExecutors.contains(slaveId) && cpus >= scheduler.CPUS_PER_TASK) } @@ -233,8 +233,9 @@ private[spark] class MesosSchedulerBackend( val cpus = if (slaveIdsWithExecutors.contains(o.getSlaveId.getValue)) { getResource(o.getResourcesList, "cpus").toInt } else { - // If the executor doesn't exist yet, subtract CPU for executor - (getResource(o.getResourcesList, "cpus") - executorCores).toInt + // If the Mesos executor has not been started on this slave yet, set aside a few + // cores for the Mesos executor by offering fewer cores to the Spark executor + (getResource(o.getResourcesList, "cpus") - mesosExecutorCores).toInt } new WorkerOffer( o.getSlaveId.getValue, diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala index f9223cea927b6..cdd7be0fbe5dd 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala @@ -118,12 +118,12 @@ class MesosSchedulerBackendSuite extends FunSuite with LocalSparkContext with Mo expectedWorkerOffers.append(new WorkerOffer( mesosOffers.get(0).getSlaveId.getValue, mesosOffers.get(0).getHostname, - (minCpu - backend.executorCores).toInt + (minCpu - backend.mesosExecutorCores).toInt )) expectedWorkerOffers.append(new WorkerOffer( mesosOffers.get(2).getSlaveId.getValue, mesosOffers.get(2).getHostname, - (minCpu - backend.executorCores).toInt + (minCpu - backend.mesosExecutorCores).toInt )) val taskDesc = new TaskDescription(1L, 0, "s1", "n1", 0, ByteBuffer.wrap(new Array[Byte](0))) when(taskScheduler.resourceOffers(expectedWorkerOffers)).thenReturn(Seq(Seq(taskDesc))) diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 0ed4391f3ee04..594bf78b67713 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -211,13 +211,13 @@ See the [configuration page](configuration.html) for information on Spark config - spark.mesos.executor.cores + spark.mesos.mesosExecutor.cores 1.0 - Set the amount of cores, which can be a floating point number, to request for running a Mesos executor. - The setting controls the number of cores allocated to the executor not for use by tasks. By default, - executor will use the amount of cores even though no task is running on an executor. This setting is - only used for Mesos fine-grained mode. + (Fine-grained mode only) Number of cores to give each Mesos executor. This does not + include the cores used to run the Spark tasks. In other words, even if no Spark task + is being run, each Mesos executor will occupy the number of cores configured here. + The value can be a floating point number.