From 147c2d492aa61854011dff48ef8017bd5c017cce Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Thu, 10 Mar 2022 16:29:32 +0800 Subject: [PATCH] YARN-11084. Introduce new config to specify AM default node-label when not specified --- .../hadoop/yarn/conf/YarnConfiguration.java | 3 ++ .../src/main/resources/yarn-default.xml | 8 +++++ .../server/resourcemanager/RMAppManager.java | 15 +++++++-- .../resourcemanager/TestAppManager.java | 31 +++++++++++++++++++ .../src/site/markdown/NodeLabel.md | 8 +++++ 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e61b5416e22a4..df01a12f0a395 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4409,6 +4409,9 @@ public static boolean areNodeLabelsEnabled( private static final String RM_NODE_LABELS_PREFIX = RM_PREFIX + "node-labels."; + public static final String AM_DEFAULT_NODE_LABEL = + RM_NODE_LABELS_PREFIX + "am.default-node-label-expression"; + public static final String RM_NODE_LABELS_PROVIDER_CONFIG = RM_NODE_LABELS_PREFIX + "provider"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1416665a4fb5d..45b05b7a337be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3452,6 +3452,14 @@ 30000 + + + Overwrites default-node-label-expression only for the ApplicationMaster + container. It is disabled by default. + + yarn.resourcemanager.node-labels.am.default-node-label-expression + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 928eeb19160f1..5bedef4baa387 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -83,6 +83,9 @@ import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture; import org.apache.hadoop.yarn.util.StringHelper; +import static org.apache.commons.lang.StringUtils.isEmpty; +import static org.apache.commons.lang.StringUtils.isNotEmpty; + /** * This class manages the list of applications for the resource manager. */ @@ -106,6 +109,7 @@ public class RMAppManager implements EventHandler, private boolean timelineServiceV2Enabled; private boolean nodeLabelsEnabled; private Set exclusiveEnforcedPartitions; + private String amDefaultNodeLabel; private static final String USER_ID_PREFIX = "userid="; @@ -134,6 +138,8 @@ public RMAppManager(RMContext context, .areNodeLabelsEnabled(rmContext.getYarnConfiguration()); this.exclusiveEnforcedPartitions = YarnConfiguration .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration()); + this.amDefaultNodeLabel = conf + .get(YarnConfiguration.AM_DEFAULT_NODE_LABEL, null); } /** @@ -622,9 +628,12 @@ private List validateAndCreateResourceRequest( } // set label expression for AM ANY request if not set - if (null == anyReq.getNodeLabelExpression()) { - anyReq.setNodeLabelExpression(submissionContext - .getNodeLabelExpression()); + if (isEmpty(anyReq.getNodeLabelExpression())) { + if (isNotEmpty(amDefaultNodeLabel)) { + anyReq.setNodeLabelExpression(amDefaultNodeLabel); + } else { + anyReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression()); + } } // Put ANY request at the front diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 8c73a0d5bc15d..18a184b467df2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -729,6 +729,33 @@ public void testRMAppSubmitAMContainerResourceRequest() throws Exception { app.getAMResourceRequests()); } + @Test + public void testRMAppSubmitAMContainerWithNoLabelByRMDefaultAMNodeLabel() throws Exception { + List reqs = new ArrayList<>(); + ResourceRequest anyReq = ResourceRequest.newInstance( + Priority.newInstance(1), + ResourceRequest.ANY, Resources.createResource(1024), 1, false, null, + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); + reqs.add(anyReq); + asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs)); + asContext.setNodeLabelExpression("fixed"); + + Configuration conf = new Configuration(false); + String defaultAMNodeLabel = "core"; + conf.set(YarnConfiguration.AM_DEFAULT_NODE_LABEL, defaultAMNodeLabel); + + when(mockDefaultQueueInfo.getAccessibleNodeLabels()).thenReturn + (new HashSet() {{ add("core"); }}); + + TestRMAppManager newAppMonitor = createAppManager(rmContext, conf); + newAppMonitor.submitApplication(asContext, "test"); + + RMApp app = rmContext.getRMApps().get(appId); + waitUntilEventProcessed(); + Assert.assertEquals(defaultAMNodeLabel, + app.getAMResourceRequests().get(0).getNodeLabelExpression()); + } + @Test public void testRMAppSubmitResource() throws Exception { asContext.setResource(Resources.createResource(1024)); @@ -836,6 +863,10 @@ public void testRMAppSubmitAMContainerResourceRequestsTwoManyAny() private RMApp testRMAppSubmit() throws Exception { appMonitor.submitApplication(asContext, "test"); + return waitUntilEventProcessed(); + } + + private RMApp waitUntilEventProcessed() throws InterruptedException { RMApp app = rmContext.getRMApps().get(appId); Assert.assertNotNull("app is null", app); Assert.assertEquals("app id doesn't match", appId, app.getApplicationId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md index 32545f3ac42d4..9c39e9950249e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md @@ -170,6 +170,14 @@ Applications can use following Java APIs to specify node label to request * `ResourceRequest.setNodeLabelExpression(..)` to set node label expression for individual resource requests. This can overwrite node label expression set in ApplicationSubmissionContext * Specify `setAMContainerResourceRequest.setNodeLabelExpression` in `ApplicationSubmissionContext` to indicate expected node label for application master container. +__Default AM node-label Configuration__ + +Property | Value +----- | ------ +yarn.resourcemanager.node-labels.am.default-node-label-expression | Overwrites default-node-label-expression only for the ApplicationMaster container. It is disabled by default. + + + Monitoring ----------