From a84e40f6b2a7e1ff97add17e6560900451627354 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Sun, 8 May 2022 23:05:08 +0200 Subject: [PATCH 1/4] TestRMWebServicesDelegationTokenAuthentication: Call DefaultMetricsSystem.setMiniClusterMode(true) in setup to prevent throwing an exception when registering the metrics source more than once --- ...estRMWebServicesDelegationTokenAuthentication.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java index ea286568340c2..91f3e50bdc4b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java @@ -42,6 +42,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.UserGroupInformation; @@ -109,7 +110,7 @@ public static void setUp() { System.setProperty(SUN_SECURITY_KRB5_RCACHE_KEY, "none"); testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); setupKDC(); - + DefaultMetricsSystem.setMiniClusterMode(true); } catch (Exception e) { assertTrue("Couldn't create MiniKDC", false); } @@ -188,7 +189,13 @@ private void setupAndStartRM() throws Exception { UserGroupInformation.setConfiguration(rmconf); rm = new MockRM(rmconf); rm.start(); - + //TODO remove +// rmconf.writeXml(System.out); +// try { +// rm.start(); +// } catch (Exception e) { +// e.printStackTrace(); +// } } private static void setupKDC() throws Exception { From 95455618c3141e2dcd82c620a6dd4ec598d15b81 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Sun, 8 May 2022 23:06:18 +0200 Subject: [PATCH 2/4] Fix error handling in case ResourceManager's web app can't be started, by saving the webApp's reference to ResourceManager.webApp --- .../org/apache/hadoop/yarn/webapp/WebAppException.java | 10 ++++++++++ .../java/org/apache/hadoop/yarn/webapp/WebApps.java | 2 +- .../yarn/server/resourcemanager/ResourceManager.java | 8 +++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java index dbede94f29589..10d947392192f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java @@ -25,6 +25,11 @@ public class WebAppException extends YarnRuntimeException { private static final long serialVersionUID = 1L; + private WebApp webApp; + + public WebApp getWebApp() { + return webApp; + } public WebAppException(String msg) { super(msg); @@ -35,6 +40,11 @@ public WebAppException(Throwable cause) { } public WebAppException(String msg, Throwable cause) { + this(msg, cause, null); + } + + public WebAppException(String msg, Throwable cause, WebApp webApp) { super(msg, cause); + this.webApp = webApp; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 6bd22b2471986..9fef076196e4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -476,7 +476,7 @@ public WebApp start(WebApp webapp, WebAppContext ui2Context) { LOG.info("Web app " + name + " started at " + httpServer.getConnectorAddress(0).getPort()); } catch (IOException e) { - throw new WebAppException("Error starting http server", e); + throw new WebAppException("Error starting http server", e, webApp); } return webApp; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 3bd6a0fe43a8c..8adcff42a695d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -23,6 +23,7 @@ import com.sun.jersey.spi.container.servlet.ServletContainer; import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics; +import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; @@ -1440,7 +1441,12 @@ protected void startWepApp() { IsResourceManagerActiveServlet.PATH_SPEC, IsResourceManagerActiveServlet.class); - webApp = builder.start(new RMWebApp(this), uiWebAppContext); + try { + webApp = builder.start(new RMWebApp(this), uiWebAppContext); + } catch (WebAppException e) { + webApp = e.getWebApp(); + throw e; + } } private String getWebAppsPath(String appName) { From 6102d0e2c998011741844dc0b75c79af4d516600 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Thu, 12 May 2022 13:37:41 +0200 Subject: [PATCH 3/4] Revert "TestRMWebServicesDelegationTokenAuthentication: Call DefaultMetricsSystem.setMiniClusterMode(true) in setup to prevent throwing an exception when registering the metrics source more than once" This reverts commit a84e40f6b2a7e1ff97add17e6560900451627354. --- ...estRMWebServicesDelegationTokenAuthentication.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java index 91f3e50bdc4b4..ea286568340c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java @@ -42,7 +42,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.UserGroupInformation; @@ -110,7 +109,7 @@ public static void setUp() { System.setProperty(SUN_SECURITY_KRB5_RCACHE_KEY, "none"); testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); setupKDC(); - DefaultMetricsSystem.setMiniClusterMode(true); + } catch (Exception e) { assertTrue("Couldn't create MiniKDC", false); } @@ -189,13 +188,7 @@ private void setupAndStartRM() throws Exception { UserGroupInformation.setConfiguration(rmconf); rm = new MockRM(rmconf); rm.start(); - //TODO remove -// rmconf.writeXml(System.out); -// try { -// rm.start(); -// } catch (Exception e) { -// e.printStackTrace(); -// } + } private static void setupKDC() throws Exception { From 08897ad737a81c394421cbdfd4068b030ab6c413 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Thu, 12 May 2022 20:03:17 +0200 Subject: [PATCH 4/4] Empty-Commit