diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala index 90dfe14352a8e..5442aa04c2188 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala @@ -63,6 +63,8 @@ private[spark] class CoarseMesosSchedulerBackend( // Maximum number of cores to acquire (TODO: we'll need more flexible controls here) val maxCores = conf.get("spark.cores.max", Int.MaxValue.toString).toInt + val totalExpectedCores = conf.getInt("spark.cores.max", 0) + // Cores we have acquired with each Mesos task ID val coresByTaskId = new HashMap[Int, Int] var totalCoresAcquired = 0 @@ -333,4 +335,7 @@ private[spark] class CoarseMesosSchedulerBackend( super.applicationId } + override def sufficientResourcesRegistered(): Boolean = { + totalCoreCount.get() >= totalExpectedCores * minRegisteredRatio + } } diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index cfb6592e14aa8..6c5c8601fa9fa 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -69,6 +69,11 @@ private[spark] class MesosSchedulerBackend( val listenerBus = sc.listenerBus @volatile var appId: String = _ + + if (sc.conf.contains("spark.scheduler.minRegisteredResourcesRatio")) { + logWarning("spark.scheduler.minRegisteredResourcesRatio is set, " + + "but it will be ignored in mesos fine-grained mode.") + } override def start() { synchronized { diff --git a/docs/configuration.md b/docs/configuration.md index 8dd2bad61344f..ac32744511790 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1018,10 +1018,10 @@ Apart from these, the following properties are also available, and may be useful
spark.scheduler.minRegisteredResourcesRatio