Skip to content

Commit 55dc94f

Browse files
committed
address review comments
1 parent 7521adf commit 55dc94f

File tree

2 files changed

+8
-7
lines changed

2 files changed

+8
-7
lines changed

core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,8 +1115,8 @@ private[spark] class TaskSetManager(
11151115

11161116
def executorDecommission(execId: String): Unit = {
11171117
recomputeLocality()
1118-
if (executorDecommissionKillInterval.nonEmpty) {
1119-
val executorKillTime = clock.getTimeMillis() + executorDecommissionKillInterval.get
1118+
executorDecommissionKillInterval.foreach { interval =>
1119+
val executorKillTime = clock.getTimeMillis() + interval
11201120
runningTasksSet.filter(taskInfos(_).executorId == execId).foreach { tid =>
11211121
tidToExecutorKillTimeMapping(tid) = executorKillTime
11221122
}

core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1942,6 +1942,9 @@ class TaskSetManagerSuite
19421942
// executorDecommissionSpeculationTriggerTimeoutOpt
19431943
// (TASK2 -> 15, TASK3 -> 15)
19441944
manager.executorDecommission("exec2")
1945+
assert(manager.tidToExecutorKillTimeMapping.keySet === Set(2, 3))
1946+
assert(manager.tidToExecutorKillTimeMapping(2) === 15)
1947+
assert(manager.tidToExecutorKillTimeMapping(3) === 15)
19451948

19461949
assert(manager.checkSpeculatableTasks(0))
19471950
// Task2 started at t=0, so it can still finish before t=15 (Median task runtime = 10)
@@ -1952,7 +1955,7 @@ class TaskSetManagerSuite
19521955

19531956
// Offer resource to start the speculative attempt for the running task
19541957
val taskOption3New = manager.resourceOffer("exec3", "host3", NO_PREF)._1
1955-
// Offer more resources. Noting should get scheduled now.
1958+
// Offer more resources. Nothing should get scheduled now.
19561959
assert(manager.resourceOffer("exec3", "host3", NO_PREF)._1.isEmpty)
19571960
assert(taskOption3New.isDefined)
19581961

@@ -1970,8 +1973,8 @@ class TaskSetManagerSuite
19701973
assert(manager.copiesRunning(3) === 2)
19711974

19721975
clock.advance(5) // time = 16 ms
1973-
// At t=16 ms, Task 4 has completed 16 ms. It is more than the
1974-
// SPECULATION_MULTIPLIER * medianRuntime = 1.5 * 10 = 15 ms. So now Task 4 will
1976+
// At t=16 ms, Task 2 has been running for 16 ms. It is more than the
1977+
// SPECULATION_MULTIPLIER * medianRuntime = 1.5 * 10 = 15 ms. So now Task 2 will
19751978
// be selected for speculation. Here we are verifying that regular speculation configs
19761979
// should still take effect even when a EXECUTOR_DECOMMISSION_KILL_INTERVAL is provided and
19771980
// corresponding executor is decommissioned
@@ -1993,8 +1996,6 @@ class TaskSetManagerSuite
19931996

19941997
// Offering additional resources should not lead to any speculative tasks being respawned
19951998
assert(manager.resourceOffer("exec1", "host1", ANY)._1.isEmpty)
1996-
assert(manager.resourceOffer("exec2", "host2", ANY)._1.isEmpty)
1997-
assert(manager.resourceOffer("exec3", "host3", ANY)._1.isEmpty)
19981999
}
19992000

20002001
test("SPARK-21040: Check speculative tasks are not launched when an executor" +

0 commit comments

Comments
 (0)