From eeca8dbe0edc363fa239fc555117424972b87436 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 13 Oct 2023 13:50:44 +0800 Subject: [PATCH 1/4] YARN-11592. Add timeout to GPGUtils#invokeRMWebService. --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +++++ .../server/globalpolicygenerator/GPGUtils.java | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 90a8978a228b2..2467f30fb555f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4549,6 +4549,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT; + public static final String GPG_WEBAPP_CONNECT_TIMEOUT = GPG_WEBAPP_PREFIX + "connect-timeout"; + public static final long DEFAULT_GPG_WEBAPP_CONNECT_TIMEOUT = TimeUnit.SECONDS.toMillis(30); + public static final String GPG_WEBAPP_READ_TIMEOUT = GPG_WEBAPP_PREFIX + "read-timeout"; + public static final long DEFAULT_GPG_WEBAPP_READ_TIMEOUT = TimeUnit.SECONDS.toMillis(30); + /** * Connection and Read timeout from the Router to RM. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 02344a51493c6..4f54b5ac99266 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.ws.rs.core.MediaType; @@ -65,7 +66,7 @@ private GPGUtils() { */ public static T invokeRMWebService(String webAddr, String path, final Class returnType, Configuration conf, String selectParam) { - Client client = Client.create(); + Client client = createJerseyClient(conf); T obj; // webAddr stores the form of host:port in subClusterInfo @@ -128,4 +129,15 @@ public static Map createUniformWeights( } return weights; } + + protected static Client createJerseyClient(Configuration conf) { + Client client = Client.create(); + int connectTimeOut = (int) conf.getTimeDuration(YarnConfiguration.GPG_WEBAPP_CONNECT_TIMEOUT, + YarnConfiguration.DEFAULT_GPG_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS); + client.setConnectTimeout(connectTimeOut); + int readTimeout = (int) conf.getTimeDuration(YarnConfiguration.GPG_WEBAPP_READ_TIMEOUT, + YarnConfiguration.DEFAULT_GPG_WEBAPP_READ_TIMEOUT, TimeUnit.MILLISECONDS); + client.setReadTimeout(readTimeout); + return client; + } } From 3f3f91ce98cd23e9a14a7af041e635179617c5a8 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 14 Oct 2023 09:58:28 +0800 Subject: [PATCH 2/4] YARN-11592. Add timeout to GPGUtils#invokeRMWebService. --- .../src/main/resources/yarn-default.xml | 23 +++++++++++++++++++ .../globalpolicygenerator/GPGUtils.java | 4 +++- .../TestGPGPolicyFacade.java | 3 +-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9991e841d74b6..d3e86d258e513 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5706,4 +5706,27 @@ 50000 + + + Set the connect timeout interval, in milliseconds. + A value of 0 means no timeout, otherwise values must be between 1 and Integer#MAX_VALUE. + This ensures that the connection is established within a specified time, + or it triggers a connection timeout exception. + + yarn.federation.gpg.webapp.connect-timeout + 30s + + + + + Set the read timeout interval, in milliseconds. + A value of 0 means no timeout, otherwise values must be between 1 and Integer#MAX_VALUE. + This timeout specifies the maximum time a client should wait for data to be read from a server + once the connection has been established. + If data is not received within the specified time, a read timeout exception is raised. + + yarn.federation.gpg.webapp.read-timeout + 30s + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 4f54b5ac99266..394a92e7f4adb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -31,6 +31,7 @@ import javax.ws.rs.core.MediaType; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -130,7 +131,8 @@ public static Map createUniformWeights( return weights; } - protected static Client createJerseyClient(Configuration conf) { + @VisibleForTesting + public static Client createJerseyClient(Configuration conf) { Client client = Client.create(); int connectTimeOut = (int) conf.getTimeDuration(YarnConfiguration.GPG_WEBAPP_CONNECT_TIMEOUT, YarnConfiguration.DEFAULT_GPG_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java index d20d6c0485f19..d5408dfdafbc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java @@ -42,7 +42,6 @@ import org.junit.Test; import org.mockito.Matchers; -import java.io.IOException; import java.util.HashSet; import java.util.Set; @@ -73,7 +72,7 @@ public TestGPGPolicyFacade() { } @Before - public void setUp() throws IOException, YarnException { + public void setUp() throws YarnException { stateStore = new MemoryFederationStateStore(); stateStore.init(conf); facade.reinitialize(stateStore, conf); From 2bac4178db80535ab2b252690b6bbf535f52d067 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 20 Oct 2023 01:03:22 +0800 Subject: [PATCH 3/4] YARN-11592. Fix CheckStyle. --- .../java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java | 4 ++-- .../hadoop/yarn/server/globalpolicygenerator/GPGUtils.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index 6ed83826dfa58..428f90858cafb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -135,8 +135,8 @@ private DeregisterSubClusterResponse generateAllSubClusterData() { public void testHelp() throws Exception { ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); ByteArrayOutputStream dataErr = new ByteArrayOutputStream(); - System.setOut(new PrintStream(dataOut)); - System.setErr(new PrintStream(dataErr)); + // System.setOut(new PrintStream(dataOut)); + // System.setErr(new PrintStream(dataErr)); String[] args = {"-help"}; rmAdminCLI.run(args); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 394a92e7f4adb..44ab95a77931e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -131,7 +131,6 @@ public static Map createUniformWeights( return weights; } - @VisibleForTesting public static Client createJerseyClient(Configuration conf) { Client client = Client.create(); int connectTimeOut = (int) conf.getTimeDuration(YarnConfiguration.GPG_WEBAPP_CONNECT_TIMEOUT, From c38f58c72e9d4bc15ba8236c956169669a5d0bbd Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 20 Oct 2023 16:16:11 +0800 Subject: [PATCH 4/4] YARN-11592. Fix CheckStyle. --- .../org/apache/hadoop/yarn/client/cli/TestRouterCLI.java | 4 ++-- .../yarn/server/globalpolicygenerator/GPGUtils.java | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index 428f90858cafb..6ed83826dfa58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -135,8 +135,8 @@ private DeregisterSubClusterResponse generateAllSubClusterData() { public void testHelp() throws Exception { ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); ByteArrayOutputStream dataErr = new ByteArrayOutputStream(); - // System.setOut(new PrintStream(dataOut)); - // System.setErr(new PrintStream(dataErr)); + System.setOut(new PrintStream(dataOut)); + System.setErr(new PrintStream(dataErr)); String[] args = {"-help"}; rmAdminCLI.run(args); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 44ab95a77931e..13e1a8a01273f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -31,7 +31,6 @@ import javax.ws.rs.core.MediaType; -import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -131,6 +130,13 @@ public static Map createUniformWeights( return weights; } + /** + * Create JerseyClient based on configuration file. + * We will set the timeout when creating JerseyClient. + * + * @param conf Configuration. + * @return JerseyClient. + */ public static Client createJerseyClient(Configuration conf) { Client client = Client.create(); int connectTimeOut = (int) conf.getTimeDuration(YarnConfiguration.GPG_WEBAPP_CONNECT_TIMEOUT,