Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ private[spark] class CoarseMesosSchedulerBackend(
// Maximum number of cores to acquire (TODO: we'll need more flexible controls here)
val maxCores = conf.get("spark.cores.max", Int.MaxValue.toString).toInt

val totalExpectedCores = conf.getInt("spark.cores.max", 0)

// Cores we have acquired with each Mesos task ID
val coresByTaskId = new HashMap[Int, Int]
var totalCoresAcquired = 0
Expand Down Expand Up @@ -333,4 +335,7 @@ private[spark] class CoarseMesosSchedulerBackend(
super.applicationId
}

override def sufficientResourcesRegistered(): Boolean = {
totalCoreCount.get() >= totalExpectedCores * minRegisteredRatio
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ private[spark] class MesosSchedulerBackend(
val listenerBus = sc.listenerBus

@volatile var appId: String = _

if (sc.conf.contains("spark.scheduler.minRegisteredResourcesRatio")) {
logWarning("spark.scheduler.minRegisteredResourcesRatio is set, "
+ "but it will be ignored in mesos fine-grained mode.")
}

override def start() {
synchronized {
Expand Down
4 changes: 2 additions & 2 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1018,10 +1018,10 @@ Apart from these, the following properties are also available, and may be useful
</tr>
<tr>
<td><code>spark.scheduler.minRegisteredResourcesRatio</code></td>
<td>0.0 for Mesos and Standalone mode, 0.8 for YARN</td>
<td>0.0 for coarse-grained Mesos and Standalone mode, 0.8 for YARN</td>
<td>
The minimum ratio of registered resources (registered resources / total expected resources)
(resources are executors in yarn mode, CPU cores in standalone mode)
(resources are executors in yarn mode, CPU cores in standalone mode and coarse-grained mesos mode)
to wait for before scheduling begins. Specified as a double between 0.0 and 1.0.
Regardless of whether the minimum ratio of resources has been reached,
the maximum amount of time it will wait before scheduling begins is controlled by config
Expand Down