From 4aad0ae2162df3db394d7cdbe76ba71de365b4f9 Mon Sep 17 00:00:00 2001 From: Ju Clarysse Date: Mon, 14 Mar 2022 08:28:50 +0100 Subject: [PATCH 1/7] fixup --- .../fs/s3a/OIDCTokenCredentialsProvider.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java new file mode 100644 index 0000000000000..3e174e9e22c4a --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java @@ -0,0 +1,78 @@ +package org.apache.hadoop.fs.s3a; + +import org.apache.commons.lang3.StringUtils; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.ProviderUtils; +import org.slf4j.Logger; + +import java.io.IOException; + +/** + * WebIdentityTokenCredentialsProvider supports static configuration + * of OIDC token path, role ARN and role session name. + * + */ +//@InterfaceAudience.Public +//@InterfaceStability.Stable +public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider { + public static final String NAME + = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider"; + + //usually from import static org.apache.hadoop.fs.s3a.Constants.*; + public static final String JWT_PATH = "fs.s3a.jwt.path"; + public static final String ROLE_ARN = "fs.s3a.role.arn"; + public static final String SESSION_NAME = "fs.s3a.session.name"; + + /** Reuse the S3AFileSystem log. */ + private static final Logger LOG = S3AFileSystem.LOG; + + private String jwtPath; + private String roleARN; + private String sessionName; + private IOException lookupIOE; + + public OIDCTokenCredentialsProvider(Configuration conf) { + try { + Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( + conf, S3AFileSystem.class); + this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null); + this.roleARN = S3AUtils.lookupPassword(c, ROLE_ARN, null); + this.sessionName = S3AUtils.lookupPassword(c, SESSION_NAME, null); + } catch (IOException e) { + lookupIOE = e; + } + } + + public AWSCredentials getCredentials() { + if (lookupIOE != null) { + // propagate any initialization problem + throw new CredentialInitializationException(lookupIOE.toString(), + lookupIOE); + } + + LOG.debug("jwtPath {} roleARN {}", jwtPath, roleARN); + + if (!StringUtils.isEmpty(jwtPath) && !StringUtils.isEmpty(roleARN)) { + final AWSCredentialsProvider credentialsProvider = + WebIdentityTokenCredentialsProvider.builder() + .webIdentityTokenFile(jwtPath) + .roleArn(roleARN) + .roleSessionName(sessionName) + .build(); + return credentialsProvider.getCredentials(); + } + else throw new CredentialInitializationException( + "OIDC token path or role ARN is null"); + } + + public void refresh() {} + + @Override + public String toString() { + return getClass().getSimpleName(); + } + +} From d213b2ce9833bb7e482fbdebc76259713334c70f Mon Sep 17 00:00:00 2001 From: Ju Clarysse Date: Mon, 14 Mar 2022 08:32:13 +0100 Subject: [PATCH 2/7] fixup --- .../org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java index 3e174e9e22c4a..55001a14b1cb7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java @@ -21,6 +21,7 @@ public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider"; + //these are the parameters to document and to pass along with the class //usually from import static org.apache.hadoop.fs.s3a.Constants.*; public static final String JWT_PATH = "fs.s3a.jwt.path"; public static final String ROLE_ARN = "fs.s3a.role.arn"; From 2c25b87405c53a5cb9bf5a8d69511d9db29d2c3a Mon Sep 17 00:00:00 2001 From: Ju Clarysse Date: Tue, 15 Mar 2022 12:09:44 +0100 Subject: [PATCH 3/7] fix indent --- .../fs/s3a/OIDCTokenCredentialsProvider.java | 95 +++++++++---------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java index 55001a14b1cb7..727856bec9c53 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java @@ -18,62 +18,61 @@ //@InterfaceAudience.Public //@InterfaceStability.Stable public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider { - public static final String NAME - = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider"; + public static final String NAME + = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider"; - //these are the parameters to document and to pass along with the class - //usually from import static org.apache.hadoop.fs.s3a.Constants.*; - public static final String JWT_PATH = "fs.s3a.jwt.path"; - public static final String ROLE_ARN = "fs.s3a.role.arn"; - public static final String SESSION_NAME = "fs.s3a.session.name"; + //these are the parameters to document and to pass along with the class + //usually from import static org.apache.hadoop.fs.s3a.Constants.*; + public static final String JWT_PATH = "fs.s3a.jwt.path"; + public static final String ROLE_ARN = "fs.s3a.role.arn"; + public static final String SESSION_NAME = "fs.s3a.session.name"; - /** Reuse the S3AFileSystem log. */ - private static final Logger LOG = S3AFileSystem.LOG; + /** Reuse the S3AFileSystem log. */ + private static final Logger LOG = S3AFileSystem.LOG; - private String jwtPath; - private String roleARN; - private String sessionName; - private IOException lookupIOE; + private String jwtPath; + private String roleARN; + private String sessionName; + private IOException lookupIOE; - public OIDCTokenCredentialsProvider(Configuration conf) { - try { - Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( - conf, S3AFileSystem.class); - this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null); - this.roleARN = S3AUtils.lookupPassword(c, ROLE_ARN, null); - this.sessionName = S3AUtils.lookupPassword(c, SESSION_NAME, null); - } catch (IOException e) { - lookupIOE = e; - } - } + public OIDCTokenCredentialsProvider(Configuration conf) { + try { + Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( + conf, S3AFileSystem.class); + this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null); + this.roleARN = S3AUtils.lookupPassword(c, ROLE_ARN, null); + this.sessionName = S3AUtils.lookupPassword(c, SESSION_NAME, null); + } catch (IOException e) { + lookupIOE = e; + } + } - public AWSCredentials getCredentials() { - if (lookupIOE != null) { - // propagate any initialization problem - throw new CredentialInitializationException(lookupIOE.toString(), - lookupIOE); - } + public AWSCredentials getCredentials() { + if (lookupIOE != null) { + // propagate any initialization problem + throw new CredentialInitializationException(lookupIOE.toString(), + lookupIOE); + } - LOG.debug("jwtPath {} roleARN {}", jwtPath, roleARN); + LOG.debug("jwtPath {} roleARN {}", jwtPath, roleARN); - if (!StringUtils.isEmpty(jwtPath) && !StringUtils.isEmpty(roleARN)) { - final AWSCredentialsProvider credentialsProvider = - WebIdentityTokenCredentialsProvider.builder() - .webIdentityTokenFile(jwtPath) - .roleArn(roleARN) - .roleSessionName(sessionName) - .build(); - return credentialsProvider.getCredentials(); - } - else throw new CredentialInitializationException( - "OIDC token path or role ARN is null"); - } + if (!StringUtils.isEmpty(jwtPath) && !StringUtils.isEmpty(roleARN)) { + final AWSCredentialsProvider credentialsProvider = + WebIdentityTokenCredentialsProvider.builder() + .webIdentityTokenFile(jwtPath) + .roleArn(roleARN) + .roleSessionName(sessionName) + .build(); + return credentialsProvider.getCredentials(); + } + else throw new CredentialInitializationException( + "OIDC token path or role ARN is null"); + } - public void refresh() {} + public void refresh() {} - @Override - public String toString() { + @Override + public String toString() { return getClass().getSimpleName(); } - -} +} \ No newline at end of file From f7207421d305b78d47f0b48ab3656b4bede78ac7 Mon Sep 17 00:00:00 2001 From: Ju Clarysse Date: Tue, 15 Mar 2022 12:11:17 +0100 Subject: [PATCH 4/7] fix license --- .../fs/s3a/OIDCTokenCredentialsProvider.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java index 727856bec9c53..d7ba3a3ffd6d7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.s3a; import org.apache.commons.lang3.StringUtils; From 623ce669fd6b9945dfac8050b120985a88b0a632 Mon Sep 17 00:00:00 2001 From: Ju Clarysse Date: Tue, 15 Mar 2022 14:08:52 +0100 Subject: [PATCH 5/7] use external constant class --- .../org/apache/hadoop/fs/s3a/Constants.java | 4 +++ .../fs/s3a/OIDCTokenCredentialsProvider.java | 28 +++++++++++-------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index dd7e425880962..4b445ad6fbb0e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -142,6 +142,10 @@ private Constants() { public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT = SimpleAWSCredentialsProvider.NAME; + /** + * Absolute path to the web identity token file + */ + public static final String JWT_PATH = "fs.s3a.jwt.path"; // the maximum number of tasks cached if all threads are already uploading public static final String MAX_TOTAL_TASKS = "fs.s3a.max.total.tasks"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java index d7ba3a3ffd6d7..f69014a83a491 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java @@ -22,29 +22,33 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.ProviderUtils; import org.slf4j.Logger; import java.io.IOException; +import static org.apache.hadoop.fs.s3a.Constants.*; + /** - * WebIdentityTokenCredentialsProvider supports static configuration - * of OIDC token path, role ARN and role session name. + * Support OpenID Connect (OIDC) token for authenticating with AWS. * + * Please note that users may reference this class name from configuration + * property fs.s3a.aws.credentials.provider. Therefore, changing the class name + * would be a backward-incompatible change. + * + * This credential provider must not fail in creation because that will + * break a chain of credential providers. */ -//@InterfaceAudience.Public -//@InterfaceStability.Stable +@InterfaceAudience.Public +@InterfaceStability.Stable public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider"; - //these are the parameters to document and to pass along with the class - //usually from import static org.apache.hadoop.fs.s3a.Constants.*; - public static final String JWT_PATH = "fs.s3a.jwt.path"; - public static final String ROLE_ARN = "fs.s3a.role.arn"; - public static final String SESSION_NAME = "fs.s3a.session.name"; - /** Reuse the S3AFileSystem log. */ private static final Logger LOG = S3AFileSystem.LOG; @@ -58,8 +62,8 @@ public OIDCTokenCredentialsProvider(Configuration conf) { Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( conf, S3AFileSystem.class); this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null); - this.roleARN = S3AUtils.lookupPassword(c, ROLE_ARN, null); - this.sessionName = S3AUtils.lookupPassword(c, SESSION_NAME, null); + this.roleARN = S3AUtils.lookupPassword(c, ASSUMED_ROLE_ARN, null); + this.sessionName = S3AUtils.lookupPassword(c, ASSUMED_ROLE_SESSION_NAME, null); } catch (IOException e) { lookupIOE = e; } From e624ea1a7cc9224ac2f23ed69506711b9f52bb88 Mon Sep 17 00:00:00 2001 From: Ju Clarysse Date: Tue, 15 Mar 2022 14:29:08 +0100 Subject: [PATCH 6/7] help with logging --- .../hadoop/fs/s3a/OIDCTokenCredentialsProvider.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java index f69014a83a491..03b87bcbbbc1c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java @@ -76,7 +76,7 @@ public AWSCredentials getCredentials() { lookupIOE); } - LOG.debug("jwtPath {} roleARN {}", jwtPath, roleARN); + LOG.debug("jwtPath {} roleARN {} sessionName {}", jwtPath, roleARN, sessionName); if (!StringUtils.isEmpty(jwtPath) && !StringUtils.isEmpty(roleARN)) { final AWSCredentialsProvider credentialsProvider = @@ -95,6 +95,9 @@ public void refresh() {} @Override public String toString() { - return getClass().getSimpleName(); - } + return String.format("%s " + + "jwtPath {%s} roleARN {%s} sessionName {%s}", + getClass().getSimpleName(), + jwtPath, roleARN, sessionName); + } } \ No newline at end of file From 93773618af72386ae99d37540557715cd46ba160 Mon Sep 17 00:00:00 2001 From: Ju Clarysse Date: Tue, 15 Mar 2022 17:54:38 +0100 Subject: [PATCH 7/7] fix indent --- .../fs/s3a/OIDCTokenCredentialsProvider.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java index 03b87bcbbbc1c..1e598447ab00a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java @@ -18,15 +18,17 @@ package org.apache.hadoop.fs.s3a; -import org.apache.commons.lang3.StringUtils; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; +import org.apache.commons.lang3.StringUtils; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.ProviderUtils; + import org.slf4j.Logger; import java.io.IOException; @@ -58,15 +60,15 @@ public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider { private IOException lookupIOE; public OIDCTokenCredentialsProvider(Configuration conf) { - try { - Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( + try { + Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( conf, S3AFileSystem.class); - this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null); - this.roleARN = S3AUtils.lookupPassword(c, ASSUMED_ROLE_ARN, null); - this.sessionName = S3AUtils.lookupPassword(c, ASSUMED_ROLE_SESSION_NAME, null); - } catch (IOException e) { - lookupIOE = e; - } + this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null); + this.roleARN = S3AUtils.lookupPassword(c, ASSUMED_ROLE_ARN, null); + this.sessionName = S3AUtils.lookupPassword(c, ASSUMED_ROLE_SESSION_NAME, null); + } catch (IOException e) { + lookupIOE = e; + } } public AWSCredentials getCredentials() {