From bb8011a930c42a46729578f5937581b26e76f8b7 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 19 Jul 2022 22:14:59 -0700 Subject: [PATCH 01/10] YARN-6572. Refactoring Router services to use common util classes for pipeline creations. --- .../server/utils/YarnServerBuilderUtils.java | 2 +- .../yarn/server/router/RouterServerUtil.java | 65 ++++++++++++++ .../clientrm/RouterClientRMService.java | 85 +++++-------------- .../router/rmadmin/RouterRMAdminService.java | 76 +++++------------ .../router/webapp/RouterWebServices.java | 74 +++++----------- .../rmadmin/TestRouterRMAdminService.java | 3 +- 6 files changed, 130 insertions(+), 175 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java index 9ee68d12c7797..6a5d22affae8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java @@ -79,7 +79,7 @@ public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId, * * @param applicationId Application ID * @param credentials HDFS Tokens - * @return systemCredentialsForAppsProto SystemCredentialsForAppsProto + * @return systemCredentialsForAppsProto */ public static SystemCredentialsForAppsProto newSystemCredentialsForAppsProto( ApplicationId applicationId, ByteBuffer credentials) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index cc96da62331b9..55896182fde57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -18,13 +18,29 @@ package org.apache.hadoop.yarn.server.router; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.commons.lang3.reflect.MethodUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; +import org.apache.hadoop.yarn.server.router.clientrm.ClientRequestInterceptor; +import org.apache.hadoop.yarn.server.router.rmadmin.RMAdminRequestInterceptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + /** * Common utility methods used by the Router server. * @@ -60,4 +76,53 @@ public static void logAndThrowException(String errMsg, Throwable t) } } + public static R createRequestInterceptorChain(Configuration conf, String pipeLineClassName, + String interceptorClassName, ClientMethod request, Class clazz) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + + List interceptorClassNames = getInterceptorClassNames(conf, + pipeLineClassName, interceptorClassName); + + R pipeline = null; + R current = null; + + for (String className : interceptorClassNames) { + try { + Class interceptorClass = conf.getClassByName(className); + if (clazz.isAssignableFrom(interceptorClass)) { + R interceptorInstance = (R) ReflectionUtils.newInstance(interceptorClass, conf); + if (pipeline == null) { + pipeline = interceptorInstance; + current = interceptorInstance; + continue; + } else { + Method method = clazz.getMethod(request.getMethodName(), request.getTypes()); + method.invoke(current, interceptorInstance); + current = interceptorInstance; + } + } else { + LOG.error("Class: {} not instance of {}.", className, clazz.getCanonicalName()); + throw new YarnRuntimeException("Class: " + className + " not instance of " + + clazz.getCanonicalName()); + } + } catch (ClassNotFoundException e) { + throw new YarnRuntimeException("Could not instantiate RequestInterceptor: " + className, e); + } + } + + return pipeline; + } + + private static List getInterceptorClassNames(Configuration conf, + String pipeLineClass, String interceptorClass) { + String configuredInterceptorClassNames = conf.get(pipeLineClass, interceptorClass); + List interceptorClassNames = new ArrayList(); + Collection tempList = + StringUtils.getStringCollection(configuredInterceptorClassNames); + for (String item : tempList) { + interceptorClassNames.add(item.trim()); + } + return interceptorClassNames; + } + } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 955c48fd953a9..617e8ff03423b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.router.clientrm; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; @@ -110,6 +111,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -147,7 +149,7 @@ public RouterClientRMService() { @Override protected void serviceStart() throws Exception { - LOG.info("Starting Router ClientRMService"); + LOG.info("Starting Router ClientRMService."); Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); UserGroupInformation.setConfiguration(conf); @@ -161,9 +163,7 @@ protected void serviceStart() throws Exception { int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); - this.userPipelineMap = Collections.synchronizedMap( - new LRUCacheHashMap( - maxCacheSize, true)); + this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true)); Configuration serverConf = new Configuration(conf); @@ -181,14 +181,13 @@ protected void serviceStart() throws Exception { } this.server.start(); - LOG.info("Router ClientRMService listening on address: " - + this.server.getListenerAddress()); + LOG.info("Router ClientRMService listening on address: {}.", this.server.getListenerAddress()); super.serviceStart(); } @Override protected void serviceStop() throws Exception { - LOG.info("Stopping Router ClientRMService"); + LOG.info("Stopping Router ClientRMService."); if (this.server != null) { this.server.stop(); } @@ -201,27 +200,6 @@ public Server getServer() { return this.server; } - /** - * Returns the comma separated intercepter class names from the configuration. - * - * @param conf - * @return the intercepter class names as an instance of ArrayList - */ - private List getInterceptorClassNames(Configuration conf) { - String configuredInterceptorClassNames = - conf.get(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS); - - List interceptorClassNames = new ArrayList(); - Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); - for (String item : tempList) { - interceptorClassNames.add(item.trim()); - } - - return interceptorClassNames; - } - @Override public GetNewApplicationResponse getNewApplication( GetNewApplicationRequest request) throws YarnException, IOException { @@ -507,42 +485,25 @@ protected Map getPipelines() { @VisibleForTesting protected ClientRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); + ClientRequestInterceptor pipeline = null; + ClientMethod remoteMethod = null; + try { + remoteMethod = new ClientMethod("setNextInterceptor", + new Class[]{ClientRequestInterceptor.class}, new Object[]{null}); - List interceptorClassNames = getInterceptorClassNames(conf); + pipeline = RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS, + remoteMethod, ClientRequestInterceptor.class); - ClientRequestInterceptor pipeline = null; - ClientRequestInterceptor current = null; - for (String interceptorClassName : interceptorClassNames) { - try { - Class interceptorClass = conf.getClassByName(interceptorClassName); - if (ClientRequestInterceptor.class.isAssignableFrom(interceptorClass)) { - ClientRequestInterceptor interceptorInstance = - (ClientRequestInterceptor) ReflectionUtils - .newInstance(interceptorClass, conf); - if (pipeline == null) { - pipeline = interceptorInstance; - current = interceptorInstance; - continue; - } else { - current.setNextInterceptor(interceptorInstance); - current = interceptorInstance; - } - } else { - throw new YarnRuntimeException( - "Class: " + interceptorClassName + " not instance of " - + ClientRequestInterceptor.class.getCanonicalName()); - } - } catch (ClassNotFoundException e) { + if (pipeline == null) { throw new YarnRuntimeException( - "Could not instantiate ApplicationClientRequestInterceptor: " - + interceptorClassName, - e); + "RequestInterceptor pipeline is not configured in the system."); } - } - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system"); + } catch (IOException | InvocationTargetException | NoSuchMethodException | RuntimeException + | IllegalAccessException ex) { + throw new YarnRuntimeException("RequestInterceptor pipeline is not configured in the system.", + ex); } return pipeline; } @@ -566,14 +527,14 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { // We should init the pipeline instance after it is created and then // add to the map, to ensure thread safe. LOG.info("Initializing request processing pipeline for application " - + "for the user: {}", user); + + "for the user: {}.", user); ClientRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); chainWrapper.init(interceptorChain); } catch (Exception e) { - LOG.error("Init ClientRequestInterceptor error for user: " + user, e); + LOG.error("Init ClientRequestInterceptor error for user: {}.", user, e); throw e; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index e05de7a899488..0dc090dada984 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.router.rmadmin; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; @@ -69,6 +70,9 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; +import org.apache.hadoop.yarn.server.router.RouterServerUtil; +import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; +import org.apache.hadoop.yarn.server.router.clientrm.ClientRequestInterceptor; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -164,27 +168,6 @@ public Server getServer() { return this.server; } - /** - * Returns the comma separated intercepter class names from the configuration. - * - * @param conf - * @return the intercepter class names as an instance of ArrayList - */ - private List getInterceptorClassNames(Configuration conf) { - String configuredInterceptorClassNames = - conf.get(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS); - - List interceptorClassNames = new ArrayList(); - Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); - for (String item : tempList) { - interceptorClassNames.add(item.trim()); - } - - return interceptorClassNames; - } - @VisibleForTesting protected RequestInterceptorChainWrapper getInterceptorChain() throws IOException { @@ -215,43 +198,24 @@ protected Map getPipelines() { @VisibleForTesting protected RMAdminRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); + RMAdminRequestInterceptor pipeline = null; + ClientMethod remoteMethod = null; + try { + remoteMethod = new ClientMethod("setNextInterceptor", + new Class[]{RMAdminRequestInterceptor.class}, new Object[]{null}); - List interceptorClassNames = getInterceptorClassNames(conf); + pipeline = RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS, + remoteMethod, RMAdminRequestInterceptor.class); - RMAdminRequestInterceptor pipeline = null; - RMAdminRequestInterceptor current = null; - for (String interceptorClassName : interceptorClassNames) { - try { - Class interceptorClass = conf.getClassByName(interceptorClassName); - if (RMAdminRequestInterceptor.class - .isAssignableFrom(interceptorClass)) { - RMAdminRequestInterceptor interceptorInstance = - (RMAdminRequestInterceptor) ReflectionUtils - .newInstance(interceptorClass, conf); - if (pipeline == null) { - pipeline = interceptorInstance; - current = interceptorInstance; - continue; - } else { - current.setNextInterceptor(interceptorInstance); - current = interceptorInstance; - } - } else { - throw new YarnRuntimeException( - "Class: " + interceptorClassName + " not instance of " - + RMAdminRequestInterceptor.class.getCanonicalName()); - } - } catch (ClassNotFoundException e) { + if (pipeline == null) { throw new YarnRuntimeException( - "Could not instantiate RMAdminRequestInterceptor: " - + interceptorClassName, - e); + "RequestInterceptor pipeline is not configured in the system."); } - } - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system"); + } catch (IOException | InvocationTargetException | NoSuchMethodException | RuntimeException + | IllegalAccessException ex) { + throw new YarnRuntimeException("Create RequestInterceptor Chain error.", ex); } return pipeline; } @@ -274,14 +238,14 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { try { // We should init the pipeline instance after it is created and then // add to the map, to ensure thread safe. - LOG.info("Initializing request processing pipeline for user: {}", user); + LOG.info("Initializing request processing pipeline for user: {}.", user); RMAdminRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); chainWrapper.init(interceptorChain); } catch (Exception e) { - LOG.error("Init RMAdminRequestInterceptor error for user: " + user, e); + LOG.error("Init RMAdminRequestInterceptor error for user: {}.", user, e); throw e; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index c221b86e98433..a17e4d94db02f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.router.webapp; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -86,6 +87,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.server.router.RouterServerUtil; +import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; +import org.apache.hadoop.yarn.server.router.rmadmin.RMAdminRequestInterceptor; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.LRUCacheHashMap; @@ -140,27 +144,6 @@ public RouterWebServices(final Router router, Configuration conf) { maxCacheSize, true)); } - /** - * Returns the comma separated intercepter class names from the configuration. - * - * @param conf - * @return the intercepter class names as an instance of ArrayList - */ - private List getInterceptorClassNames(Configuration config) { - String configuredInterceptorClassNames = - config.get(YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS); - - List interceptorClassNames = new ArrayList(); - Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); - for (String item : tempList) { - interceptorClassNames.add(item.trim()); - } - - return interceptorClassNames; - } - private void init() { // clear content type response.setContentType(null); @@ -206,43 +189,26 @@ protected Map getPipelines() { */ @VisibleForTesting protected RESTRequestInterceptor createRequestInterceptorChain() { + RESTRequestInterceptor pipeline = null; + ClientMethod remoteMethod = null; + try { + remoteMethod = new ClientMethod("setNextInterceptor", + new Class[]{RMAdminRequestInterceptor.class}, new Object[]{null}); - List interceptorClassNames = getInterceptorClassNames(conf); + pipeline = RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS, + remoteMethod, RESTRequestInterceptor.class); - RESTRequestInterceptor pipeline = null; - RESTRequestInterceptor current = null; - for (String interceptorClassName : interceptorClassNames) { - try { - Class interceptorClass = conf.getClassByName(interceptorClassName); - if (RESTRequestInterceptor.class.isAssignableFrom(interceptorClass)) { - RESTRequestInterceptor interceptorInstance = - (RESTRequestInterceptor) ReflectionUtils - .newInstance(interceptorClass, conf); - if (pipeline == null) { - pipeline = interceptorInstance; - current = interceptorInstance; - continue; - } else { - current.setNextInterceptor(interceptorInstance); - current = interceptorInstance; - } - } else { - throw new YarnRuntimeException( - "Class: " + interceptorClassName + " not instance of " - + RESTRequestInterceptor.class.getCanonicalName()); - } - } catch (ClassNotFoundException e) { + if (pipeline == null) { throw new YarnRuntimeException( - "Could not instantiate RESTRequestInterceptor: " - + interceptorClassName, - e); + "RequestInterceptor pipeline is not configured in the system."); } + } catch (IOException | InvocationTargetException | NoSuchMethodException | RuntimeException + | IllegalAccessException ex) { + throw new YarnRuntimeException("Create RequestInterceptor Chain error.", ex); } - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system"); - } return pipeline; } @@ -264,14 +230,14 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { try { // We should init the pipeline instance after it is created and then // add to the map, to ensure thread safe. - LOG.info("Initializing request processing pipeline for user: {}", user); + LOG.info("Initializing request processing pipeline for user: {}.", user); RESTRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); chainWrapper.init(interceptorChain); } catch (Exception e) { - LOG.error("Init RESTRequestInterceptor error for user: " + user, e); + LOG.error("Init RESTRequestInterceptor error for user: {}", user, e); throw e; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java index 07ef73c3cdb85..867c71fa82e54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; @@ -185,7 +184,7 @@ public void testRouterRMAdminServiceE2E() throws Exception { */ @Test public void testUsersChainMapWithLRUCache() - throws YarnException, IOException, InterruptedException { + throws IOException, InterruptedException { Map pipelines; RequestInterceptorChainWrapper chain; From e9dd485bb9476a731b1d9d1c4bed28d2b77813df Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 20 Jul 2022 01:43:04 -0700 Subject: [PATCH 02/10] YARN-6572. Fix CheckStyle. --- .../yarn/server/router/RouterServerUtil.java | 15 +++++-------- .../clientrm/RouterClientRMService.java | 5 ----- .../router/rmadmin/RouterRMAdminService.java | 14 +++--------- .../router/webapp/RouterWebServices.java | 22 +++++++------------ 4 files changed, 16 insertions(+), 40 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index 55896182fde57..16887aa2c2626 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.router; -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.commons.lang3.reflect.MethodUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -29,12 +27,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; -import org.apache.hadoop.yarn.server.router.clientrm.ClientRequestInterceptor; -import org.apache.hadoop.yarn.server.router.rmadmin.RMAdminRequestInterceptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -90,15 +85,15 @@ public static R createRequestInterceptorChain(Configuration conf, String pip try { Class interceptorClass = conf.getClassByName(className); if (clazz.isAssignableFrom(interceptorClass)) { - R interceptorInstance = (R) ReflectionUtils.newInstance(interceptorClass, conf); + Object interceptorInstance = ReflectionUtils.newInstance(interceptorClass, conf); if (pipeline == null) { - pipeline = interceptorInstance; - current = interceptorInstance; + pipeline = clazz.cast(interceptorInstance); + current = clazz.cast(interceptorInstance); continue; } else { Method method = clazz.getMethod(request.getMethodName(), request.getTypes()); method.invoke(current, interceptorInstance); - current = interceptorInstance; + current = clazz.cast(interceptorInstance); } } else { LOG.error("Class: {} not instance of {}.", className, clazz.getCanonicalName()); @@ -116,7 +111,7 @@ public static R createRequestInterceptorChain(Configuration conf, String pip private static List getInterceptorClassNames(Configuration conf, String pipeLineClass, String interceptorClass) { String configuredInterceptorClassNames = conf.get(pipeLineClass, interceptorClass); - List interceptorClassNames = new ArrayList(); + List interceptorClassNames = new ArrayList<>(); Collection tempList = StringUtils.getStringCollection(configuredInterceptorClassNames); for (String item : tempList) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 617e8ff03423b..4cef4f666d55e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -21,10 +21,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -34,8 +31,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index 0dc090dada984..d3d683152ff98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -21,10 +21,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -35,8 +32,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -72,7 +67,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; -import org.apache.hadoop.yarn.server.router.clientrm.ClientRequestInterceptor; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -125,8 +119,7 @@ protected void serviceStart() throws Exception { conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); this.userPipelineMap = Collections.synchronizedMap( - new LRUCacheHashMap( - maxCacheSize, true)); + new LRUCacheHashMap<>(maxCacheSize, true)); Configuration serverConf = new Configuration(conf); @@ -143,14 +136,13 @@ protected void serviceStart() throws Exception { } this.server.start(); - LOG.info("Router RMAdminService listening on address: " - + this.server.getListenerAddress()); + LOG.info("Router RMAdminService listening on address: {}", this.server.getListenerAddress()); super.serviceStart(); } @Override protected void serviceStop() throws Exception { - LOG.info("Stopping Router RMAdminService"); + LOG.info("Stopping Router RMAdminService."); if (this.server != null) { this.server.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index a17e4d94db02f..743222eed6d38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -20,10 +20,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Set; @@ -49,8 +46,6 @@ import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -89,7 +84,6 @@ import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; -import org.apache.hadoop.yarn.server.router.rmadmin.RMAdminRequestInterceptor; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.LRUCacheHashMap; @@ -139,9 +133,7 @@ public RouterWebServices(final Router router, Configuration conf) { int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); - this.userPipelineMap = Collections.synchronizedMap( - new LRUCacheHashMap( - maxCacheSize, true)); + this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true)); } private void init() { @@ -193,7 +185,7 @@ protected RESTRequestInterceptor createRequestInterceptorChain() { ClientMethod remoteMethod = null; try { remoteMethod = new ClientMethod("setNextInterceptor", - new Class[]{RMAdminRequestInterceptor.class}, new Object[]{null}); + new Class[]{RESTRequestInterceptor.class}, new Object[]{null}); pipeline = RouterServerUtil.createRequestInterceptorChain(conf, YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, @@ -215,7 +207,7 @@ protected RESTRequestInterceptor createRequestInterceptorChain() { /** * Initializes the request intercepter pipeline for the specified user. * - * @param user + * @param user specified user. */ private RequestInterceptorChainWrapper initializePipeline(String user) { synchronized (this.userPipelineMap) { @@ -303,7 +295,7 @@ public ClusterInfo getClusterInfo() { @GET @Path(RMWSConsts.CLUSTER_USER_INFO) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Override public ClusterUserInfo getClusterUserInfo(@Context HttpServletRequest hsr) { init(); @@ -799,10 +791,12 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, @Override public Response listReservation( @QueryParam(RMWSConsts.QUEUE) @DefaultValue(DEFAULT_QUEUE) String queue, - @QueryParam(RMWSConsts.RESERVATION_ID) @DefaultValue(DEFAULT_RESERVATION_ID) String reservationId, + @QueryParam(RMWSConsts.RESERVATION_ID) + @DefaultValue(DEFAULT_RESERVATION_ID) String reservationId, @QueryParam(RMWSConsts.START_TIME) @DefaultValue(DEFAULT_START_TIME) long startTime, @QueryParam(RMWSConsts.END_TIME) @DefaultValue(DEFAULT_END_TIME) long endTime, - @QueryParam(RMWSConsts.INCLUDE_RESOURCE) @DefaultValue(DEFAULT_INCLUDE_RESOURCE) boolean includeResourceAllocations, + @QueryParam(RMWSConsts.INCLUDE_RESOURCE) + @DefaultValue(DEFAULT_INCLUDE_RESOURCE) boolean includeResourceAllocations, @Context HttpServletRequest hsr) throws Exception { init(); RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); From 58555baa13435644be441039e512ac5999543e5d Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 20 Jul 2022 19:50:42 -0700 Subject: [PATCH 03/10] YARN-6572. Refactoring Router services to use common util classes for pipeline creations. --- .../yarn/server/router/RouterServerUtil.java | 26 ++++++++++++++++--- .../clientrm/RouterClientRMService.java | 18 +++---------- .../router/rmadmin/RouterRMAdminService.java | 20 +++----------- .../router/webapp/RouterWebServices.java | 21 +++------------ 4 files changed, 31 insertions(+), 54 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index 16887aa2c2626..fd725b71e1c98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -26,7 +26,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,8 +71,7 @@ public static void logAndThrowException(String errMsg, Throwable t) } public static R createRequestInterceptorChain(Configuration conf, String pipeLineClassName, - String interceptorClassName, ClientMethod request, Class clazz) - throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + String interceptorClassName, Class clazz) { List interceptorClassNames = getInterceptorClassNames(conf, pipeLineClassName, interceptorClassName); @@ -91,7 +89,7 @@ public static R createRequestInterceptorChain(Configuration conf, String pip current = clazz.cast(interceptorInstance); continue; } else { - Method method = clazz.getMethod(request.getMethodName(), request.getTypes()); + Method method = clazz.getMethod("setNextInterceptor", clazz); method.invoke(current, interceptorInstance); current = clazz.cast(interceptorInstance); } @@ -101,10 +99,30 @@ public static R createRequestInterceptorChain(Configuration conf, String pip + clazz.getCanonicalName()); } } catch (ClassNotFoundException e) { + LOG.error("Could not instantiate RequestInterceptor: {}", className, e); throw new YarnRuntimeException("Could not instantiate RequestInterceptor: " + className, e); + } catch (InvocationTargetException e) { + LOG.error("RequestInterceptor {} call setNextInterceptor error.", className, e); + throw new YarnRuntimeException("RequestInterceptor " + className + + " call setNextInterceptor error.", e); + } catch (NoSuchMethodException e) { + LOG.error("RequestInterceptor {} does not contain the method setNextInterceptor.", + className); + throw new YarnRuntimeException("RequestInterceptor " + className + + " does not contain the method setNextInterceptor.", e); + } catch (IllegalAccessException e) { + LOG.error("RequestInterceptor {} call the method setNextInterceptor " + + "does not have access.", className); + throw new YarnRuntimeException("RequestInterceptor " + + className + " call the method setNextInterceptor does not have access.", e); } } + if (pipeline == null) { + throw new YarnRuntimeException( + "RequestInterceptor pipeline is not configured in the system."); + } + return pipeline; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 4cef4f666d55e..71672ca5a3bfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -480,27 +480,15 @@ protected Map getPipelines() { @VisibleForTesting protected ClientRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); - ClientRequestInterceptor pipeline = null; - ClientMethod remoteMethod = null; try { - remoteMethod = new ClientMethod("setNextInterceptor", - new Class[]{ClientRequestInterceptor.class}, new Object[]{null}); - - pipeline = RouterServerUtil.createRequestInterceptorChain(conf, + return RouterServerUtil.createRequestInterceptorChain(conf, YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS, - remoteMethod, ClientRequestInterceptor.class); - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system."); - } - } catch (IOException | InvocationTargetException | NoSuchMethodException | RuntimeException - | IllegalAccessException ex) { + ClientRequestInterceptor.class); + } catch (YarnRuntimeException ex) { throw new YarnRuntimeException("RequestInterceptor pipeline is not configured in the system.", ex); } - return pipeline; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index d3d683152ff98..2775bc47a126f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.router.rmadmin; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; import java.util.Collections; import java.util.Map; @@ -66,7 +65,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.router.RouterServerUtil; -import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -190,26 +188,14 @@ protected Map getPipelines() { @VisibleForTesting protected RMAdminRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); - RMAdminRequestInterceptor pipeline = null; - ClientMethod remoteMethod = null; try { - remoteMethod = new ClientMethod("setNextInterceptor", - new Class[]{RMAdminRequestInterceptor.class}, new Object[]{null}); - - pipeline = RouterServerUtil.createRequestInterceptorChain(conf, + return RouterServerUtil.createRequestInterceptorChain(conf, YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS, - remoteMethod, RMAdminRequestInterceptor.class); - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system."); - } - } catch (IOException | InvocationTargetException | NoSuchMethodException | RuntimeException - | IllegalAccessException ex) { + RMAdminRequestInterceptor.class); + } catch (YarnRuntimeException ex) { throw new YarnRuntimeException("Create RequestInterceptor Chain error.", ex); } - return pipeline; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index 743222eed6d38..a0aad4abb3f93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.router.webapp; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -83,7 +82,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.router.RouterServerUtil; -import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.LRUCacheHashMap; @@ -181,27 +179,14 @@ protected Map getPipelines() { */ @VisibleForTesting protected RESTRequestInterceptor createRequestInterceptorChain() { - RESTRequestInterceptor pipeline = null; - ClientMethod remoteMethod = null; try { - remoteMethod = new ClientMethod("setNextInterceptor", - new Class[]{RESTRequestInterceptor.class}, new Object[]{null}); - - pipeline = RouterServerUtil.createRequestInterceptorChain(conf, + return RouterServerUtil.createRequestInterceptorChain(conf, YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS, - remoteMethod, RESTRequestInterceptor.class); - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system."); - } - } catch (IOException | InvocationTargetException | NoSuchMethodException | RuntimeException - | IllegalAccessException ex) { + RESTRequestInterceptor.class); + } catch (YarnRuntimeException ex) { throw new YarnRuntimeException("Create RequestInterceptor Chain error.", ex); } - - return pipeline; } /** From 34f83e76d08304ca4213e859bc5172c26399b83a Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 20 Jul 2022 19:59:57 -0700 Subject: [PATCH 04/10] YARN-6572. Fix CheckStyle. --- .../yarn/server/router/clientrm/RouterClientRMService.java | 4 ++-- .../yarn/server/router/rmadmin/RouterRMAdminService.java | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 71672ca5a3bfc..f884477f0821a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -509,8 +509,8 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { try { // We should init the pipeline instance after it is created and then // add to the map, to ensure thread safe. - LOG.info("Initializing request processing pipeline for application " - + "for the user: {}.", user); + LOG.info("Initializing request processing pipeline for application for the user: {}.", + user); ClientRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index 2775bc47a126f..149e36bee64a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -102,7 +102,7 @@ public RouterRMAdminService() { @Override protected void serviceStart() throws Exception { - LOG.info("Starting Router RMAdmin Service"); + LOG.info("Starting Router RMAdmin Service."); Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); UserGroupInformation.setConfiguration(conf); @@ -116,8 +116,7 @@ protected void serviceStart() throws Exception { int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); - this.userPipelineMap = Collections.synchronizedMap( - new LRUCacheHashMap<>(maxCacheSize, true)); + this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true)); Configuration serverConf = new Configuration(conf); @@ -134,7 +133,7 @@ protected void serviceStart() throws Exception { } this.server.start(); - LOG.info("Router RMAdminService listening on address: {}", this.server.getListenerAddress()); + LOG.info("Router RMAdminService listening on address: {}.", this.server.getListenerAddress()); super.serviceStart(); } From 1af1df73f24c3babbd3fdae0d8cd9274a0fcf790 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 21 Jul 2022 23:34:10 -0700 Subject: [PATCH 05/10] YARN-6572. Fix CheckStyle. --- .../yarn/server/router/clientrm/RouterClientRMService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index f884477f0821a..2fb327b7a5f4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.router.clientrm; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; import java.util.Collections; import java.util.Map; From 59217e4b672d202cc17e853a9ab9d464138094b0 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 27 Jul 2022 00:45:42 -0700 Subject: [PATCH 06/10] YARN-6572. Fix CheckStyle. --- .../router/clientrm/RouterClientRMService.java | 13 ++++--------- .../server/router/rmadmin/RouterRMAdminService.java | 12 ++++-------- .../server/router/webapp/RouterWebServices.java | 12 ++++-------- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 768d4e1000431..e1fc77da9d25c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -479,15 +479,10 @@ protected Map getPipelines() { @VisibleForTesting protected ClientRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); - try { - return RouterServerUtil.createRequestInterceptorChain(conf, - YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS, - ClientRequestInterceptor.class); - } catch (YarnRuntimeException ex) { - throw new YarnRuntimeException("RequestInterceptor pipeline is not configured in the system.", - ex); - } + return RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS, + ClientRequestInterceptor.class); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index 6eb6eee69f37a..96da41f585c33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -187,14 +187,10 @@ protected Map getPipelines() { @VisibleForTesting protected RMAdminRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); - try { - return RouterServerUtil.createRequestInterceptorChain(conf, - YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS, - RMAdminRequestInterceptor.class); - } catch (YarnRuntimeException ex) { - throw new YarnRuntimeException("Create RequestInterceptor Chain error.", ex); - } + return RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS, + RMAdminRequestInterceptor.class); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index ac3a462a5dfd3..8b9440db90024 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -179,14 +179,10 @@ protected Map getPipelines() { */ @VisibleForTesting protected RESTRequestInterceptor createRequestInterceptorChain() { - try { - return RouterServerUtil.createRequestInterceptorChain(conf, - YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS, - RESTRequestInterceptor.class); - } catch (YarnRuntimeException ex) { - throw new YarnRuntimeException("Create RequestInterceptor Chain error.", ex); - } + return RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS, + RESTRequestInterceptor.class); } /** From eb5f398c11f3b9b732a58b8bc025cdf5882ebe49 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Mon, 1 Aug 2022 16:48:15 -0700 Subject: [PATCH 07/10] YARN-6572. Fix CheckStyle. --- .../yarn/server/router/clientrm/RouterClientRMService.java | 1 - .../hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java | 1 - .../hadoop/yarn/server/router/webapp/RouterWebServices.java | 1 - 3 files changed, 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index e1fc77da9d25c..fad68661d6994 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -103,7 +103,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index 96da41f585c33..56378d4d9fd91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -33,7 +33,6 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index 8b9440db90024..fe5326b38cfb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -47,7 +47,6 @@ import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServiceProtocol; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo; From a7cd76c3bb4b2446a2ed6b4dd2cacd5c770e4704 Mon Sep 17 00:00:00 2001 From: zhujiang02 Date: Wed, 3 Aug 2022 15:42:05 +0800 Subject: [PATCH 08/10] YARN-6572. Conflict Resolution. --- .../apache/hadoop/yarn/server/router/RouterServerUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index fccf3b13e6c7e..836db8006982a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -132,11 +132,12 @@ private static List getInterceptorClassNames(Configuration conf, String configuredInterceptorClassNames = conf.get(pipeLineClass, interceptorClass); List interceptorClassNames = new ArrayList<>(); Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); + StringUtils.getStringCollection(configuredInterceptorClassNames); for (String item : tempList) { interceptorClassNames.add(item.trim()); } return interceptorClassNames; + } /** * Throws an IOException due to an error. @@ -157,5 +158,4 @@ public static void logAndThrowIOException(String errMsg, Throwable t) throw new IOException(errMsg); } } - } \ No newline at end of file From 78dca6452c97ab22eebcf5dc4ab3ac07eec81ebb Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 3 Aug 2022 09:20:54 -0700 Subject: [PATCH 09/10] Revert "YARN-6572. Conflict Resolution." This reverts commit a7cd76c3bb4b2446a2ed6b4dd2cacd5c770e4704. --- .../apache/hadoop/yarn/server/router/RouterServerUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index 836db8006982a..fccf3b13e6c7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -132,12 +132,11 @@ private static List getInterceptorClassNames(Configuration conf, String configuredInterceptorClassNames = conf.get(pipeLineClass, interceptorClass); List interceptorClassNames = new ArrayList<>(); Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); + StringUtils.getStringCollection(configuredInterceptorClassNames); for (String item : tempList) { interceptorClassNames.add(item.trim()); } return interceptorClassNames; - } /** * Throws an IOException due to an error. @@ -158,4 +157,5 @@ public static void logAndThrowIOException(String errMsg, Throwable t) throw new IOException(errMsg); } } + } \ No newline at end of file From 69e9941bcff110b57860a0b2a526f8fdfe2a5c58 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 3 Aug 2022 09:23:35 -0700 Subject: [PATCH 10/10] YARN-6572. Conflict Resolution-v2. --- .../org/apache/hadoop/yarn/server/router/RouterServerUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index fccf3b13e6c7e..70f93a9e6b42c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -137,6 +137,7 @@ private static List getInterceptorClassNames(Configuration conf, interceptorClassNames.add(item.trim()); } return interceptorClassNames; + } /** * Throws an IOException due to an error.