Skip to content

Commit 6d9500c

Browse files
haibchenhaibchen
committed
YARN-6705 Add separate NM preemption thresholds for cpu and memory (Haibo Chen)
1 parent 6ca0bc8 commit 6d9500c

File tree

3 files changed

+85
-22
lines changed

3 files changed

+85
-22
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1741,10 +1741,33 @@ public static boolean isAclEnabled(Configuration conf) {
17411741
public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
17421742
NM_PREFIX + "overallocation.memory-utilization-threshold";
17431743

1744-
public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
1745-
NM_PREFIX + "overallocation.preemption-threshold";
1746-
public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
1747-
= 0.96f;
1744+
/**
1745+
* The CPU utilization threshold, if went beyond for a few times in a row,
1746+
* OPPORTUNISTIC containers started due to overallocation should start
1747+
* getting preempted.
1748+
*/
1749+
public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD =
1750+
NM_PREFIX + "overallocation.preemption-threshold.cpu";
1751+
public static final float
1752+
DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = 0.99f;
1753+
1754+
/**
1755+
* The number of times that CPU utilization must go over the CPU preemption
1756+
* threshold consecutively before preemption starts to kick in.
1757+
*/
1758+
public static final String NM_OVERALLOCATION_PREEMPTION_CPU_COUNT =
1759+
NM_PREFIX + "overallocation.preemption-threshold-count.cpu";
1760+
public static final int DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = 4;
1761+
1762+
1763+
/**
1764+
* The memory utilization threshold beyond which OPPORTUNISTIC containers
1765+
* started due to overallocation should start getting preempted.
1766+
*/
1767+
public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD =
1768+
NM_PREFIX + "overallocation.preemption-threshold.memory";
1769+
public static final float
1770+
DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = 0.95f;
17481771

17491772
/**
17501773
* Interval of time the linux container executor should try cleaning up

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1639,11 +1639,37 @@
16391639

16401640
<property>
16411641
<description>When a node is over-allocated to improve utilization by
1642-
running OPPORTUNISTIC containers, this config captures the utilization
1643-
beyond which OPPORTUNISTIC containers should start getting preempted.
1642+
running OPPORTUNISTIC containers, this config captures the CPU
1643+
utilization beyond which OPPORTUNISTIC containers should start getting
1644+
preempted. This is used in combination with
1645+
yarn.nodemanager.overallocation.preemption-threshold-count.cpu, that is,
1646+
only when the CPU utilization goes over this threshold consecutively for
1647+
a few times will preemption kicks in.
16441648
</description>
1645-
<name>yarn.nodemanager.overallocation.preemption-threshold</name>
1646-
<value>0.96</value>
1649+
<name>yarn.nodemanager.overallocation.preemption-threshold.cpu</name>
1650+
<value>0.99</value>
1651+
</property>
1652+
1653+
<property>
1654+
<description>When a node is over-allocated to improve utilization by
1655+
running OPPORTUNISTIC containers, this config captures the number of
1656+
times that CPU utilization has to go above
1657+
${yarn.nodemanager.overallocation.preemption-threshold.cpu}
1658+
consecutively for NM to start preempting OPPORTUNISTIC containers
1659+
started due to overallocation.
1660+
</description>
1661+
<name>yarn.nodemanager.overallocation.preemption-threshold-count.cpu</name>
1662+
<value>4</value>
1663+
</property>
1664+
1665+
<property>
1666+
<description>When a node is over-allocated to improve utilization by
1667+
running OPPORTUNISTIC containers, this config captures the CPU
1668+
utilization beyond which OPPORTUNISTIC containers should start getting
1669+
preempted.
1670+
</description>
1671+
<name>yarn.nodemanager.overallocation.preemption-threshold.memory</name>
1672+
<value>0.95</value>
16471673
</property>
16481674

16491675
<property>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ public enum ContainerMetric {
103103
private ResourceUtilization containersUtilization;
104104

105105
private ResourceThresholds overAllocationPreemptionThresholds;
106+
private int overAlloctionPreemptionCpuCount = -1;
106107

107108
private volatile boolean stopped = false;
108109

@@ -237,7 +238,7 @@ private void initializeOverAllocation(Configuration conf) {
237238
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
238239
if (overAllocationMemoryUtilizationThreshold <= 0) {
239240
LOG.info("NodeManager oversubscription is disabled because the memory " +
240-
"utilization threshold is no larger than zero.");
241+
"overallocation threshold is no larger than zero.");
241242
return;
242243
}
243244

@@ -249,36 +250,49 @@ private void initializeOverAllocation(Configuration conf) {
249250
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
250251
if (overAllocationCpuUtilizationThreshold <= 0) {
251252
LOG.info("NodeManager oversubscription is disabled because the CPU " +
252-
"utilization threshold is no larger than zero.");
253+
"overallocation threshold is no larger than zero.");
253254
return;
254255
}
255256

256-
float preemptionThreshold = conf.getFloat(
257-
YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
258-
YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
259-
if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
260-
LOG.info("NodeManager oversubscription is disabled because preemption" +
261-
"threshold is no larger than the cpu utilization threshold.");
257+
float cpuPreemptionThreshold = conf.getFloat(
258+
YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
259+
YarnConfiguration.
260+
DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD);
261+
if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) {
262+
LOG.info("NodeManager oversubscription is disabled because the cpu " +
263+
" preemption threshold is no larger than the cpu overallocation" +
264+
" threshold.");
262265
return;
263266
}
264-
if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
265-
LOG.info("NodeManager oversubscription is disabled because preemption" +
266-
"threshold is no larger than the memory utilization threshold.");
267+
268+
float memoryPreemptionThreshold = conf.getFloat(
269+
YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD,
270+
YarnConfiguration.
271+
DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD);
272+
if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
273+
LOG.info("NodeManager oversubscription is disabled because the memory" +
274+
" preemption threshold is no larger than the memory overallocation" +
275+
" threshold.");
267276
return;
268277
}
269278

279+
this.overAlloctionPreemptionCpuCount = conf.getInt(
280+
YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT,
281+
YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT);
282+
270283
ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
271284
overAllocationCpuUtilizationThreshold,
272285
overAllocationMemoryUtilizationThreshold);
273286
((NodeManager.NMContext) context).setOverAllocationInfo(
274287
OverAllocationInfo.newInstance(resourceThresholds));
275-
this.overAllocationPreemptionThresholds =
276-
ResourceThresholds.newInstance(preemptionThreshold);
288+
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
289+
cpuPreemptionThreshold, memoryPreemptionThreshold);
277290

278291
LOG.info("NodeManager oversubscription enabled with overallocation " +
279292
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
280293
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
281-
" threshold: " + preemptionThreshold);
294+
" threshold (memory:" + memoryPreemptionThreshold + ", CPU:" +
295+
cpuPreemptionThreshold + ")");
282296
}
283297

284298
private boolean isResourceCalculatorAvailable() {

0 commit comments

Comments
 (0)