From 9cbdcee7ae463e661c42a2626b92e24e113e407f Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 6 Jan 2024 18:21:07 +0800 Subject: [PATCH 1/4] YARN-11634. [Addendum] Speed-up TestTimelineClient. --- .../yarn/client/api/impl/TimelineClientImpl.java | 6 ++++++ .../yarn/client/api/impl/TimelineConnector.java | 13 +++++++++---- .../yarn/client/api/impl/TestTimelineClient.java | 4 ++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 2b9ce4fa8f2ad..45da0f444ba0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -459,4 +459,10 @@ public void putDomain(ApplicationAttemptId appAttemptId, public void setTimelineWriter(TimelineWriter writer) { this.timelineWriter = writer; } + + @Private + @VisibleForTesting + public TimelineConnector getConnector() { + return connector; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index b139bddd101e0..198a5be26b70e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -78,8 +78,8 @@ public class TimelineConnector extends AbstractService { private static final Joiner JOINER = Joiner.on(""); private static final Logger LOG = LoggerFactory.getLogger(TimelineConnector.class); - @VisibleForTesting - public static int DEFAULT_SOCKET_TIMEOUT = 60_000; // 1 minute + + private int socketTimeOut = 60_000; private SSLFactory sslFactory; Client client; @@ -145,14 +145,14 @@ protected void serviceInit(Configuration conf) throws Exception { @Override public HttpURLConnection configure(HttpURLConnection conn) throws IOException { - setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT); + setTimeouts(conn, 60_000); return conn; } }; private ConnectionConfigurator getConnConfigurator(SSLFactory sslFactoryObj) { try { - return initSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, sslFactoryObj); + return initSslConnConfigurator(socketTimeOut, sslFactoryObj); } catch (Exception e) { LOG.debug("Cannot load customized ssl related configuration. " + "Fallback to system-generic settings.", e); @@ -457,4 +457,9 @@ public boolean shouldRetryOn(Exception e) { || e instanceof SocketTimeoutException); } } + + @VisibleForTesting + public void setSocketTimeOut(int socketTimeOut) { + this.socketTimeOut = socketTimeOut; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index cac620f669521..e69c6e5d66f40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -78,7 +78,7 @@ public void setup() { conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.0f); client = createTimelineClient(conf); - TimelineConnector.DEFAULT_SOCKET_TIMEOUT = 10; + client.getConnector().setSocketTimeOut(10); } @AfterEach @@ -89,7 +89,7 @@ public void tearDown() throws Exception { if (isSSLConfigured()) { KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); } - TimelineConnector.DEFAULT_SOCKET_TIMEOUT = 60_000; + client.getConnector().setSocketTimeOut(60_1000); } @Test From 506efb6df641d5797541869452a42bb23397fb58 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 10 Jan 2024 22:49:39 +0800 Subject: [PATCH 2/4] YARN-11634. Fix CheckStyle. --- .../yarn/client/api/impl/TimelineConnector.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index 198a5be26b70e..ec221903141b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -140,15 +140,11 @@ protected void serviceInit(Configuration conf) throws Exception { } } - private static final ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR - = new ConnectionConfigurator() { - @Override - public HttpURLConnection configure(HttpURLConnection conn) - throws IOException { - setTimeouts(conn, 60_000); - return conn; - } - }; + private ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR + = conn -> { + setTimeouts(conn, socketTimeOut); + return conn; + }; private ConnectionConfigurator getConnConfigurator(SSLFactory sslFactoryObj) { try { From aeaa452863fb337ea2971cce4d0918dd77c27315 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 11 Jan 2024 08:52:10 +0800 Subject: [PATCH 3/4] YARN-11634. Fix CheckStyle. --- .../yarn/client/api/impl/TimelineConnector.java | 13 ++++++------- .../yarn/client/api/impl/TestTimelineClient.java | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index ec221903141b5..50674332b5f03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -113,7 +113,7 @@ protected void serviceInit(Configuration conf) throws Exception { sslFactory = getSSLFactory(conf); connConfigurator = getConnConfigurator(sslFactory); } else { - connConfigurator = DEFAULT_TIMEOUT_CONN_CONFIGURATOR; + connConfigurator = default_timeout_conn_configurator; } String defaultAuth = UserGroupInformation.isSecurityEnabled() ? KerberosAuthenticationHandler.TYPE : @@ -140,11 +140,10 @@ protected void serviceInit(Configuration conf) throws Exception { } } - private ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR - = conn -> { - setTimeouts(conn, socketTimeOut); - return conn; - }; + private ConnectionConfigurator default_timeout_conn_configurator = conn -> { + setTimeouts(conn, socketTimeOut); + return conn; + }; private ConnectionConfigurator getConnConfigurator(SSLFactory sslFactoryObj) { try { @@ -152,7 +151,7 @@ private ConnectionConfigurator getConnConfigurator(SSLFactory sslFactoryObj) { } catch (Exception e) { LOG.debug("Cannot load customized ssl related configuration. " + "Fallback to system-generic settings.", e); - return DEFAULT_TIMEOUT_CONN_CONFIGURATOR; + return default_timeout_conn_configurator; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index e69c6e5d66f40..80e425e4853d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -89,7 +89,7 @@ public void tearDown() throws Exception { if (isSSLConfigured()) { KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); } - client.getConnector().setSocketTimeOut(60_1000); + client.getConnector().setSocketTimeOut(60_000); } @Test From 7969b77d6f9c8e72ac340787f69dc4f15f3319a7 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 11 Jan 2024 18:54:59 +0800 Subject: [PATCH 4/4] YARN-11634. Fix CheckStyle. --- .../hadoop/yarn/client/api/impl/TimelineConnector.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index 50674332b5f03..dce877f3cb83e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -113,7 +113,7 @@ protected void serviceInit(Configuration conf) throws Exception { sslFactory = getSSLFactory(conf); connConfigurator = getConnConfigurator(sslFactory); } else { - connConfigurator = default_timeout_conn_configurator; + connConfigurator = defaultTimeoutConnConfigurator; } String defaultAuth = UserGroupInformation.isSecurityEnabled() ? KerberosAuthenticationHandler.TYPE : @@ -140,7 +140,7 @@ protected void serviceInit(Configuration conf) throws Exception { } } - private ConnectionConfigurator default_timeout_conn_configurator = conn -> { + private ConnectionConfigurator defaultTimeoutConnConfigurator = conn -> { setTimeouts(conn, socketTimeOut); return conn; }; @@ -151,7 +151,7 @@ private ConnectionConfigurator getConnConfigurator(SSLFactory sslFactoryObj) { } catch (Exception e) { LOG.debug("Cannot load customized ssl related configuration. " + "Fallback to system-generic settings.", e); - return default_timeout_conn_configurator; + return defaultTimeoutConnConfigurator; } }