From 0f9879016161896a4b6f1d4633d3e790f15f8daa Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Thu, 11 Mar 2021 14:05:51 -0800 Subject: [PATCH 1/4] HADOOP-17578. Improve UGI debug log to help troubleshooting TokenCache related issues. --- .../hadoop/security/UserGroupInformation.java | 8 +++++--- .../security/token/DelegationTokenIssuer.java | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index f5007588036de..aa1247878f8bc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1925,11 +1925,13 @@ public T doAs(PrivilegedExceptionAction action @InterfaceAudience.LimitedPrivate({"HDFS", "KMS"}) @InterfaceStability.Unstable public static void logUserInfo(Logger log, String caption, - UserGroupInformation ugi) throws IOException { + UserGroupInformation ugi) { if (log.isDebugEnabled()) { log.debug(caption + " UGI: " + ugi); - for (Token token : ugi.getTokens()) { - log.debug("+token:" + token); + Map> tokenMap = + ugi.getCredentials().getTokenMap(); + for (Text tokenKey : tokenMap.keySet()) { + log.debug("+token: {} -> {}", tokenKey, tokenMap.get(tokenKey)); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java index 70a53b7166870..66fe6ed6c91ca 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java @@ -21,6 +21,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -32,7 +34,7 @@ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "Yarn"}) @InterfaceStability.Unstable public interface DelegationTokenIssuer { - + Logger LOG = LoggerFactory.getLogger(DelegationTokenIssuer.class); /** * The service name used as the alias for the token in the credential * token map. addDelegationTokens will use this to determine if @@ -88,15 +90,27 @@ static void collectDelegationTokens( final List> tokens) throws IOException { final String serviceName = issuer.getCanonicalServiceName(); // Collect token of the this issuer and then of its embedded children + if (LOG.isDebugEnabled()) { + LOG.debug("Search token for service {} in credentials", serviceName); + } if (serviceName != null) { final Text service = new Text(serviceName); Token token = credentials.getToken(service); if (token == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Token for service {} not found in credentials," + + " try getDelegationToken.", serviceName); + } token = issuer.getDelegationToken(renewer); if (token != null) { tokens.add(token); credentials.addToken(service, token); } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Token for service {} found in credentials," + + "skip getDelegationToken.", serviceName); + } } } // Now collect the tokens from the children. From 61fb9ed1912fd77f83c10e187448d1a252387cd3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Mon, 15 Mar 2021 13:42:03 -0700 Subject: [PATCH 2/4] refactor to trigger jenkins. --- .../org/apache/hadoop/security/UserGroupInformation.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index aa1247878f8bc..c3d7a55a10e6e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1928,10 +1928,9 @@ public static void logUserInfo(Logger log, String caption, UserGroupInformation ugi) { if (log.isDebugEnabled()) { log.debug(caption + " UGI: " + ugi); - Map> tokenMap = - ugi.getCredentials().getTokenMap(); - for (Text tokenKey : tokenMap.keySet()) { - log.debug("+token: {} -> {}", tokenKey, tokenMap.get(tokenKey)); + for (Map.Entry> kv : + ugi.getCredentials().getTokenMap().entrySet()) { + log.debug("+token: {} -> {}", kv.getKey(), kv.getValue()); } } } From 54b0ae4acdbcd460ad7dbf7832a9d09a79a7eb98 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 16 Mar 2021 11:30:34 -0700 Subject: [PATCH 3/4] Fix ut issue. --- .../security/token/DelegationTokenIssuer.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java index 66fe6ed6c91ca..0e3697c28fdba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java @@ -34,7 +34,7 @@ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "Yarn"}) @InterfaceStability.Unstable public interface DelegationTokenIssuer { - Logger LOG = LoggerFactory.getLogger(DelegationTokenIssuer.class); + Logger TOKEN_LOG = LoggerFactory.getLogger(DelegationTokenIssuer.class); /** * The service name used as the alias for the token in the credential * token map. addDelegationTokens will use this to determine if @@ -90,15 +90,15 @@ static void collectDelegationTokens( final List> tokens) throws IOException { final String serviceName = issuer.getCanonicalServiceName(); // Collect token of the this issuer and then of its embedded children - if (LOG.isDebugEnabled()) { - LOG.debug("Search token for service {} in credentials", serviceName); + if (TOKEN_LOG.isDebugEnabled()) { + TOKEN_LOG.debug("Search token for service {} in credentials", serviceName); } if (serviceName != null) { final Text service = new Text(serviceName); Token token = credentials.getToken(service); if (token == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Token for service {} not found in credentials," + + if (TOKEN_LOG.isDebugEnabled()) { + TOKEN_LOG.debug("Token for service {} not found in credentials," + " try getDelegationToken.", serviceName); } token = issuer.getDelegationToken(renewer); @@ -107,8 +107,8 @@ static void collectDelegationTokens( credentials.addToken(service, token); } } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Token for service {} found in credentials," + + if (TOKEN_LOG.isDebugEnabled()) { + TOKEN_LOG.debug("Token for service {} found in credentials," + "skip getDelegationToken.", serviceName); } } From 6655d2238e5762f144e8a97db11e6318250960cb Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 16 Mar 2021 15:10:59 -0700 Subject: [PATCH 4/4] fix checkstyle --- .../apache/hadoop/security/token/DelegationTokenIssuer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java index 0e3697c28fdba..7b0a78bcd3c0d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java @@ -91,7 +91,8 @@ static void collectDelegationTokens( final String serviceName = issuer.getCanonicalServiceName(); // Collect token of the this issuer and then of its embedded children if (TOKEN_LOG.isDebugEnabled()) { - TOKEN_LOG.debug("Search token for service {} in credentials", serviceName); + TOKEN_LOG.debug("Search token for service {} in credentials", + serviceName); } if (serviceName != null) { final Text service = new Text(serviceName);