From 48401dbc2709c12917e2ba579e9265d049cee424 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 2 Aug 2022 06:48:27 -0700 Subject: [PATCH 1/3] YARN-6972. Adding RM ClusterId in AppInfo. --- .../resourcemanager/webapp/dao/AppInfo.java | 24 +++++++++++++++++++ .../webapp/TestRMWebServicesApps.java | 3 +++ 2 files changed, 27 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 6ebaa06174eff..d683263c24b70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -38,6 +39,8 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -69,6 +72,8 @@ public class AppInfo { protected ApplicationId applicationId; @XmlTransient private String schemePrefix; + @XmlTransient + private SubClusterIdInfo subClusterId; // these are ok for any user to see protected String id; @@ -82,6 +87,7 @@ public class AppInfo { protected String trackingUrl; protected String diagnostics; protected long clusterId; + private String rmClusterId; protected String applicationType; protected String applicationTags = ""; protected int priority; @@ -182,6 +188,16 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { + if(rm != null && rm.getConfig() != null) { + try { + subClusterId = new SubClusterIdInfo( + (YarnConfiguration.getClusterId(rm.getConfig()))); + rmClusterId = + this.getSubClusterIdInfo().toId().toString(); + } catch (HadoopIllegalArgumentException e) { + rmClusterId = null; + } + } this.startedTime = app.getStartTime(); this.launchTime = app.getLaunchTime(); this.finishedTime = app.getFinishTime(); @@ -454,6 +470,14 @@ public long getClusterId() { return this.clusterId; } + public SubClusterIdInfo getSubClusterIdInfo() { + return this.subClusterId; + } + + public String getRmClusterId() { + return this.rmClusterId; + } + public String getApplicationType() { return this.applicationType; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 4c859baf78906..0e40dbf9598ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -103,6 +103,7 @@ protected void configureServlets() { YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); conf.setClass(YarnConfiguration.RM_SCHEDULER, scheduler, ResourceScheduler.class); + conf.set(YarnConfiguration.RM_CLUSTER_ID, "subCluster1"); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); @@ -1825,6 +1826,8 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) } assertEquals("incorrect number of elements", expectedNumberOfElements, info.length()); + assertEquals("rmClusterId is incorrect", "subCluster1", + info.getString("rmClusterId")); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), info.getInt("priority"), From 0e8f91844aeef2d082e03096ac087e8a6fba0820 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 2 Aug 2022 16:08:34 -0700 Subject: [PATCH 2/3] YARN-6952. Fix CheckStyle. --- .../server/resourcemanager/webapp/dao/AppInfo.java | 13 +++++++------ .../webapp/TestRMWebServicesApps.java | 3 +-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index d683263c24b70..481544e477c72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -87,7 +88,7 @@ public class AppInfo { protected String trackingUrl; protected String diagnostics; protected long clusterId; - private String rmClusterId; + protected String rmClusterId; protected String applicationType; protected String applicationTags = ""; protected int priority; @@ -188,13 +189,13 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { - if(rm != null && rm.getConfig() != null) { + if (rm != null && rm.getConfig() != null) { try { - subClusterId = new SubClusterIdInfo( - (YarnConfiguration.getClusterId(rm.getConfig()))); - rmClusterId = - this.getSubClusterIdInfo().toId().toString(); + Configuration yarnConfig = rm.getConfig(); + subClusterId = new SubClusterIdInfo(YarnConfiguration.getClusterId(yarnConfig)); + rmClusterId = this.subClusterId.toId().toString(); } catch (HadoopIllegalArgumentException e) { + subClusterId = null; rmClusterId = null; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 0e40dbf9598ad..78e89d3541407 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1826,8 +1826,7 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) } assertEquals("incorrect number of elements", expectedNumberOfElements, info.length()); - assertEquals("rmClusterId is incorrect", "subCluster1", - info.getString("rmClusterId")); + assertEquals("rmClusterId is incorrect", "subCluster1", info.getString("rmClusterId")); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), info.getInt("priority"), From b60336c4c79b4f582b77fc33e2352bb0760d6666 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 2 Aug 2022 16:15:00 -0700 Subject: [PATCH 3/3] YARN-6952. Fix Junit Test. --- .../server/resourcemanager/webapp/TestRMWebServicesApps.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 78e89d3541407..b304b8a7857b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1806,7 +1806,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 41 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext()