From 4725fb11c69a5d277b6ad4ae87074338b46ff34a Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Mon, 14 Jul 2025 11:52:41 +0530 Subject: [PATCH 1/3] HADOOP-19535: S3A : Add WebIdentityTokenFileCredentialsProvider to default S3 credential provider chain --- .../src/main/resources/core-default.xml | 1 + .../hadoop/fs/s3a/AWSCredentialProviderList.java | 12 +++++++++++- .../s3a/auth/CredentialProviderListFactory.java | 16 ++++++++++++++++ .../markdown/tools/hadoop-aws/authentication.md | 1 + .../s3a/adapter/TestV1CredentialsProvider.java | 8 ++++++-- 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 08197a8377e21..8158cec61c0f8 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1452,6 +1452,7 @@ org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, + software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider, org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index d89795c68d3da..06582251753a1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -197,7 +197,17 @@ public AwsCredentials resolveCredentials() { } catch (SdkException e) { lastException = e; LOG.debug("No credentials provided by {}: {}", - provider, e.toString(), e); + provider, e); + } catch (Exception e) { + // convert any other generic exception into SDKException. + // This is required because some credential provider like + // WebIdentityTokenFileCredentialsProvider might throw + // exceptions other than SdkException. + if (e.getMessage() != null) { + lastException = SdkException.create(e.getMessage(), e); + } + LOG.debug("No credentials provided by {}: {}", + provider, e); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index c0daa6a3aecf9..83530c45ec962 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -36,6 +36,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; @@ -81,6 +82,7 @@ public final class CredentialProviderListFactory { STANDARD_AWS_PROVIDERS = Collections.unmodifiableList( Arrays.asList( EnvironmentVariableCredentialsProvider.class, + WebIdentityTokenFileCredentialsProvider.class, IAMInstanceCredentialsProvider.class, SimpleAWSCredentialsProvider.class, TemporaryAWSCredentialsProvider.class)); @@ -109,6 +111,18 @@ public final class CredentialProviderListFactory { public static final String ENVIRONMENT_CREDENTIALS_V2 = EnvironmentVariableCredentialsProvider.class.getName(); + /** + * V1 web identity credential provider: {@value} + */ + public static final String WEB_IDENTITY_CREDENTIALS_V1 = + "com.amazonaws.auth.WebIdentityTokenCredentialsProvider"; + + /** + * V2 web identity credential provider: {@value} + */ + public static final String WEB_IDENTITY_CREDENTIALS_V2 = + WebIdentityTokenFileCredentialsProvider.class.getName(); + /** V1 profile credential provider: {@value}. */ public static final String PROFILE_CREDENTIALS_V1 = "com.amazonaws.auth.profile.ProfileCredentialsProvider"; @@ -184,6 +198,8 @@ private static Map initCredentialProvidersMap() { EC2_IAM_CREDENTIALS_V2); v1v2CredentialProviderMap.put(ENVIRONMENT_CREDENTIALS_V1, ENVIRONMENT_CREDENTIALS_V2); + v1v2CredentialProviderMap.put(WEB_IDENTITY_CREDENTIALS_V1, + WEB_IDENTITY_CREDENTIALS_V2); v1v2CredentialProviderMap.put(PROFILE_CREDENTIALS_V1, PROFILE_CREDENTIALS_V2); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md index af60a48f7da1f..be9b204b73cb9 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md @@ -184,6 +184,7 @@ There are also many in the Amazon SDKs, with the common ones being as follows | `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | AWS Environment Variables | | `software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider` | EC2 Metadata Credentials | | `software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider` | EC2/k8s Metadata Credentials | +| `software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider`| K8s Metadata Credentials | diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java index d4d2a16de1e4c..09d3e257b6adf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -45,6 +46,7 @@ import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ANONYMOUS_CREDENTIALS_V1; import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.EC2_CONTAINER_CREDENTIALS_V1; import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ENVIRONMENT_CREDENTIALS_V1; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.WEB_IDENTITY_CREDENTIALS_V1; import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -72,12 +74,14 @@ public void testV1V2Mapping() throws Exception { Arrays.asList( IAMInstanceCredentialsProvider.class, AnonymousAWSCredentialsProvider.class, - EnvironmentVariableCredentialsProvider.class); + EnvironmentVariableCredentialsProvider.class, + WebIdentityTokenFileCredentialsProvider.class); Configuration conf = createProviderConfiguration(buildClassList( EC2_CONTAINER_CREDENTIALS_V1, ANONYMOUS_CREDENTIALS_V1, - ENVIRONMENT_CREDENTIALS_V1)); + ENVIRONMENT_CREDENTIALS_V1, + WEB_IDENTITY_CREDENTIALS_V1)); AWSCredentialProviderList list1 = createAWSCredentialProviderList( uri1, conf); assertCredentialProviders(expectedClasses, list1); From dbb0e4f4a172fc085c69e4b2d65c7c634d10e128 Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Tue, 15 Jul 2025 07:41:57 +0530 Subject: [PATCH 2/3] Adressing Review comments --- .../src/main/resources/core-default.xml | 1 - .../fs/s3a/AWSCredentialProviderList.java | 2 +- .../auth/CredentialProviderListFactory.java | 16 ---------------- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 18 ++++++++++++++++++ .../s3a/adapter/TestV1CredentialsProvider.java | 8 ++------ 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 8158cec61c0f8..08197a8377e21 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1452,7 +1452,6 @@ org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, - software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider, org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index 06582251753a1..0a11aaa7cde22 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -199,7 +199,7 @@ public AwsCredentials resolveCredentials() { LOG.debug("No credentials provided by {}: {}", provider, e); } catch (Exception e) { - // convert any other generic exception into SDKException. + // convert any other exception into SDKException. // This is required because some credential provider like // WebIdentityTokenFileCredentialsProvider might throw // exceptions other than SdkException. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index 83530c45ec962..c0daa6a3aecf9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -36,7 +36,6 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; @@ -82,7 +81,6 @@ public final class CredentialProviderListFactory { STANDARD_AWS_PROVIDERS = Collections.unmodifiableList( Arrays.asList( EnvironmentVariableCredentialsProvider.class, - WebIdentityTokenFileCredentialsProvider.class, IAMInstanceCredentialsProvider.class, SimpleAWSCredentialsProvider.class, TemporaryAWSCredentialsProvider.class)); @@ -111,18 +109,6 @@ public final class CredentialProviderListFactory { public static final String ENVIRONMENT_CREDENTIALS_V2 = EnvironmentVariableCredentialsProvider.class.getName(); - /** - * V1 web identity credential provider: {@value} - */ - public static final String WEB_IDENTITY_CREDENTIALS_V1 = - "com.amazonaws.auth.WebIdentityTokenCredentialsProvider"; - - /** - * V2 web identity credential provider: {@value} - */ - public static final String WEB_IDENTITY_CREDENTIALS_V2 = - WebIdentityTokenFileCredentialsProvider.class.getName(); - /** V1 profile credential provider: {@value}. */ public static final String PROFILE_CREDENTIALS_V1 = "com.amazonaws.auth.profile.ProfileCredentialsProvider"; @@ -198,8 +184,6 @@ private static Map initCredentialProvidersMap() { EC2_IAM_CREDENTIALS_V2); v1v2CredentialProviderMap.put(ENVIRONMENT_CREDENTIALS_V1, ENVIRONMENT_CREDENTIALS_V2); - v1v2CredentialProviderMap.put(WEB_IDENTITY_CREDENTIALS_V1, - WEB_IDENTITY_CREDENTIALS_V2); v1v2CredentialProviderMap.put(PROFILE_CREDENTIALS_V1, PROFILE_CREDENTIALS_V2); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index fe926dec0c8c9..884d37ce3bebf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -47,6 +47,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -187,6 +188,23 @@ public void testDefaultChain() throws Exception { assertCredentialProviders(expectedClasses, list2); } + @Test + public void testNonSdkExceptionConversion() throws Throwable { + // Create a mock credential provider that throws a non-SDK exception + AwsCredentialsProvider mockProvider = () -> { + throw new RuntimeException("Test credential error"); + }; + + // Create the provider list with our mock provider + AWSCredentialProviderList providerList = + new AWSCredentialProviderList(Collections.singletonList(mockProvider)); + + // Attempt to get credentials, which should trigger the exception + intercept(NoAuthWithAWSException.class, + "No AWS Credentials provided", + () -> providerList.resolveCredentials()); + } + @Test public void testDefaultChainNoURI() throws Exception { Configuration conf = new Configuration(false); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java index 09d3e257b6adf..d4d2a16de1e4c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; -import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -46,7 +45,6 @@ import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ANONYMOUS_CREDENTIALS_V1; import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.EC2_CONTAINER_CREDENTIALS_V1; import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ENVIRONMENT_CREDENTIALS_V1; -import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.WEB_IDENTITY_CREDENTIALS_V1; import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -74,14 +72,12 @@ public void testV1V2Mapping() throws Exception { Arrays.asList( IAMInstanceCredentialsProvider.class, AnonymousAWSCredentialsProvider.class, - EnvironmentVariableCredentialsProvider.class, - WebIdentityTokenFileCredentialsProvider.class); + EnvironmentVariableCredentialsProvider.class); Configuration conf = createProviderConfiguration(buildClassList( EC2_CONTAINER_CREDENTIALS_V1, ANONYMOUS_CREDENTIALS_V1, - ENVIRONMENT_CREDENTIALS_V1, - WEB_IDENTITY_CREDENTIALS_V1)); + ENVIRONMENT_CREDENTIALS_V1)); AWSCredentialProviderList list1 = createAWSCredentialProviderList( uri1, conf); assertCredentialProviders(expectedClasses, list1); From 1ed098441acc10c025cf222e3b8341f02f0ca924 Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Wed, 16 Jul 2025 11:01:03 +0530 Subject: [PATCH 3/3] Addressing PR comments and checkstyle issue --- .../org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java | 4 ++-- .../apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index 0a11aaa7cde22..e2e12b3f58ed6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -197,7 +197,7 @@ public AwsCredentials resolveCredentials() { } catch (SdkException e) { lastException = e; LOG.debug("No credentials provided by {}: {}", - provider, e); + provider, e.toString(), e); } catch (Exception e) { // convert any other exception into SDKException. // This is required because some credential provider like @@ -207,7 +207,7 @@ public AwsCredentials resolveCredentials() { lastException = SdkException.create(e.getMessage(), e); } LOG.debug("No credentials provided by {}: {}", - provider, e); + provider, e.toString(), e); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 884d37ce3bebf..c759043252f31 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -47,7 +47,6 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; -import software.amazon.awssdk.core.exception.SdkException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path;