From 5a16515322237c35a84b8650c1e281aebdebf493 Mon Sep 17 00:00:00 2001 From: luoyuan Date: Mon, 22 Nov 2021 19:01:06 +0800 Subject: [PATCH 1/4] YARN-11012. Add version and process startup time on Router web page --- .../hadoop/yarn/server/router/Router.java | 7 +++ .../yarn/server/router/webapp/NavBlock.java | 1 + .../router/webapp/RouterController.java | 5 ++ .../server/router/webapp/RouterWebApp.java | 1 + .../server/router/webapp/ServerBlock.java | 51 +++++++++++++++++++ .../yarn/server/router/webapp/ServerPage.java | 37 ++++++++++++++ 6 files changed, 102 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerBlock.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerPage.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index fc0b9eeaef3cc..28f5d2a788237 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -29,6 +29,7 @@ import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -83,6 +84,8 @@ public class Router extends CompositeService { private static final String METRICS_NAME = "Router"; + private static long routerStartupTime = Time.monotonicNow(); + public Router() { super(Router.class.getName()); } @@ -171,6 +174,10 @@ public void startWepApp() { webApp = builder.start(new RouterWebApp(this)); } + public static long getRouterStartupTime() { + return routerStartupTime; + } + public static void main(String[] argv) { Configuration conf = new YarnConfiguration(); Thread diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java index 9c39bb7b7a22f..9fae8d002b743 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java @@ -31,6 +31,7 @@ public void render(Block html) { div("#nav"). h3("Cluster"). ul(). + li().a(url("server"), "ServerInfo").__(). li().a(url(""), "About").__(). li().a(url("federation"), "Federation").__(). li().a(url("nodes"), "Nodes").__(). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java index e8cd0de6657f5..a9dc5e7beb9a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java @@ -38,6 +38,11 @@ public void index() { render(AboutPage.class); } + public void server() { + setTitle("About the current router server"); + render(ServerPage.class); + } + public void about() { setTitle("About the Cluster"); render(AboutPage.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java index ba07a1afba466..904afc42fbbd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java @@ -45,6 +45,7 @@ public void setup() { bind(Router.class).toInstance(router); } route("/", RouterController.class); + route("/server", RouterController.class, "server"); route("/cluster", RouterController.class, "about"); route("/about", RouterController.class, "about"); route("/apps", RouterController.class, "apps"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerBlock.java new file mode 100644 index 0000000000000..802b35bd3972e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerBlock.java @@ -0,0 +1,51 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.router.webapp; + +import com.google.inject.Inject; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.util.YarnVersionInfo; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; + +import java.util.Date; + +/** + * Server block for the Router Web UI. + */ +public class ServerBlock extends HtmlBlock { + + @Inject + ServerBlock(Router router, ViewContext ctx) { + super(ctx); + } + + @Override + protected void render(Block html) { + + info("Router Details") + .__("Router started on", + new Date(Router.getRouterStartupTime())) + .__("Router Version", YarnVersionInfo.getVersion()) + .__("Hadoop Version", VersionInfo.getVersion()); + + html.__(InfoBlock.class); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerPage.java new file mode 100644 index 0000000000000..42cbe4ee22d1f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/ServerPage.java @@ -0,0 +1,37 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.router.webapp; + +import org.apache.hadoop.yarn.webapp.SubView; + +/** + * Server page for the Router Web UI. + */ +public class ServerPage extends RouterView { + + @Override + protected void preHead(Page.HTML<__> html) { + commonPreHead(html); + } + + @Override + protected Class content() { + return ServerBlock.class; + } +} From 2dd7f7491187303df2f78ffb3c8015f255636dfa Mon Sep 17 00:00:00 2001 From: luoyuan Date: Fri, 4 Feb 2022 16:37:03 +0800 Subject: [PATCH 2/4] YARN-11012. Add unit test --- .../hadoop/yarn/server/router/TestRouter.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java index 9f0b4c72aac3b..1f90c495770f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java @@ -18,13 +18,17 @@ package org.apache.hadoop.yarn.server.router; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.junit.Assert; import org.junit.Test; @@ -87,4 +91,28 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, } } + @Test + public void testStartTime() { + long start = System.currentTimeMillis(); + YarnConfiguration conf = new YarnConfiguration(); + Router router = new Router(); + router.init(conf); + router.start(); + long end = System.currentTimeMillis(); + + assertTrue(start < Router.getRouterStartupTime()); + assertTrue(end > Router.getRouterStartupTime()); + } + + @Test + public void testVersion() { + YarnConfiguration conf = new YarnConfiguration(); + Router router = new Router(); + router.init(conf); + router.start(); + + assertNotNull(YarnVersionInfo.getVersion()); + assertNotNull(VersionInfo.getVersion()); + } + } From 75717b191ad515bc15a531f17e793001b465a592 Mon Sep 17 00:00:00 2001 From: luoyuan Date: Sat, 5 Feb 2022 17:27:41 +0800 Subject: [PATCH 3/4] YARN-11012. Fix unit test --- .../hadoop/yarn/server/router/TestRouter.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java index 1f90c495770f9..1ea09fa033b4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java @@ -54,6 +54,9 @@ public void testServiceACLRefresh() { conf.set("security.applicationclient.protocol.acl", aclsString); conf.set("security.resourcemanager-administration.protocol.acl", aclsString); + conf.set("yarn.router.clientrm.address", "0.0.0.0:1111"); + conf.set("yarn.router.webapp.address", "0.0.0.0:1112"); + conf.set("yarn.router.rmadmin.address", "0.0.0.0:1113"); Router router = new Router(); router.init(conf); @@ -93,24 +96,11 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, @Test public void testStartTime() { - long start = System.currentTimeMillis(); - YarnConfiguration conf = new YarnConfiguration(); - Router router = new Router(); - router.init(conf); - router.start(); - long end = System.currentTimeMillis(); - - assertTrue(start < Router.getRouterStartupTime()); - assertTrue(end > Router.getRouterStartupTime()); + assertNotNull(Router.getRouterStartupTime()); } @Test public void testVersion() { - YarnConfiguration conf = new YarnConfiguration(); - Router router = new Router(); - router.init(conf); - router.start(); - assertNotNull(YarnVersionInfo.getVersion()); assertNotNull(VersionInfo.getVersion()); } From a120f0fe71581b8663f1061c69927c20ee84a8bc Mon Sep 17 00:00:00 2001 From: luoyuan Date: Sun, 6 Feb 2022 13:55:32 +0800 Subject: [PATCH 4/4] YARN-11012. Fix checkstyle --- .../java/org/apache/hadoop/yarn/server/router/TestRouter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java index 1ea09fa033b4c..a1f047a733de0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java @@ -19,7 +19,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.apache.hadoop.conf.Configuration;