From cee3b68d92956c0a96cd83f37670e839d3c68fc2 Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Thu, 5 Dec 2024 13:06:05 -0800 Subject: [PATCH 01/10] HADOOP-19384: Add support for ProfileCredentialsProvider This commit adds a wrapper for the AWS ProfileCredentialsProvider. --- .../src/main/resources/core-default.xml | 3 +- .../fs/s3a/ProfileAWSCredentialsProvider.java | 46 +++++++++++++++++++ .../auth/CredentialProviderListFactory.java | 10 ++-- 3 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 8e62550cd239b..8c17cb8c93ef2 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1430,7 +1430,8 @@ org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, - org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider + org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider, + org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider Comma-separated class names of credential provider classes which implement diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java new file mode 100644 index 0000000000000..9efb68788ed29 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java @@ -0,0 +1,46 @@ +package org.apache.hadoop.fs.s3a; + +import org.apache.commons.lang3.SystemUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.profiles.ProfileFile; + +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; + +@InterfaceAudience.Public +@InterfaceStability.Stable +public class ProfileAWSCredentialsProvider implements AwsCredentialsProvider { + public static final String NAME + = "org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider"; + + private ProfileCredentialsProvider pcp; + + private static Path getCredentialsPath() { + String credentialsFile = SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null); + Path path = (credentialsFile == null) ? + FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials") + : FileSystems.getDefault().getPath(credentialsFile); + return path; + } + + public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { + ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); + builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).type(ProfileFile.Type.CREDENTIALS).build()); + pcp = builder.build(); + } + + public ProfileAWSCredentialsProvider(Configuration conf) { + ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); + builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).build()); + pcp = builder.build(); + } + public AwsCredentials resolveCredentials() { + return pcp.resolveCredentials(); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index 941ce741151d5..0805eeeb379d3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -31,6 +31,7 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.apache.hadoop.fs.s3a.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -39,12 +40,6 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; -import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider; -import org.apache.hadoop.fs.s3a.Constants; -import org.apache.hadoop.fs.s3a.S3AUtils; -import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; -import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport; import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.fs.s3native.S3xLoginHelper; @@ -84,7 +79,8 @@ public final class CredentialProviderListFactory { EnvironmentVariableCredentialsProvider.class, IAMInstanceCredentialsProvider.class, SimpleAWSCredentialsProvider.class, - TemporaryAWSCredentialsProvider.class)); + TemporaryAWSCredentialsProvider.class, + ProfileAWSCredentialsProvider.class)); /** V1 credential provider: {@value}. */ public static final String ANONYMOUS_CREDENTIALS_V1 = From 303f92c57b4e9d3e39a4d7a7680c5fadddc354ba Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Thu, 9 Jan 2025 15:00:21 -0800 Subject: [PATCH 02/10] Fix imports in CredentialProviderListFactory --- .../hadoop/fs/s3a/auth/CredentialProviderListFactory.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index 0805eeeb379d3..387f0ab46767c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -31,7 +31,6 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; -import org.apache.hadoop.fs.s3a.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -40,6 +39,13 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; +import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider; +import org.apache.hadoop.fs.s3a.Constants; +import org.apache.hadoop.fs.s3a.S3AUtils; +import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; +import org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider; +import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport; import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.fs.s3native.S3xLoginHelper; From 4f98399ff6bcfd023e607cb6db5b92955362095a Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Tue, 4 Feb 2025 17:56:50 -0800 Subject: [PATCH 03/10] Add ProfileAWSCredentialsProvider test --- .../src/main/resources/core-default.xml | 2 +- .../fs/s3a/ProfileAWSCredentialsProvider.java | 46 --------------- .../auth/CredentialProviderListFactory.java | 1 - .../auth/ProfileAWSCredentialsProvider.java | 56 +++++++++++++++++++ .../fs/s3a/TestS3AAWSCredentialsProvider.java | 35 +++++++++--- 5 files changed, 85 insertions(+), 55 deletions(-) delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 8c17cb8c93ef2..117ec1bab0b4e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1431,7 +1431,6 @@ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider, - org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider Comma-separated class names of credential provider classes which implement @@ -1443,6 +1442,7 @@ token binding it may be used to communicate wih the STS endpoint to request session/role credentials. + org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider is also supported, but is not enabled by default. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java deleted file mode 100644 index 9efb68788ed29..0000000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.apache.hadoop.fs.s3a; - -import org.apache.commons.lang3.SystemUtils; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.profiles.ProfileFile; - -import java.net.URI; -import java.nio.file.FileSystems; -import java.nio.file.Path; - -@InterfaceAudience.Public -@InterfaceStability.Stable -public class ProfileAWSCredentialsProvider implements AwsCredentialsProvider { - public static final String NAME - = "org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider"; - - private ProfileCredentialsProvider pcp; - - private static Path getCredentialsPath() { - String credentialsFile = SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null); - Path path = (credentialsFile == null) ? - FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials") - : FileSystems.getDefault().getPath(credentialsFile); - return path; - } - - public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { - ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); - builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).type(ProfileFile.Type.CREDENTIALS).build()); - pcp = builder.build(); - } - - public ProfileAWSCredentialsProvider(Configuration conf) { - ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); - builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).build()); - pcp = builder.build(); - } - public AwsCredentials resolveCredentials() { - return pcp.resolveCredentials(); - } -} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index 387f0ab46767c..116349c49eac1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -44,7 +44,6 @@ import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; -import org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport; import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java new file mode 100644 index 0000000000000..acdad9fc52480 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java @@ -0,0 +1,56 @@ +package org.apache.hadoop.fs.s3a.auth; + +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.profiles.ProfileFile; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import org.apache.commons.lang3.SystemUtils; +import org.apache.hadoop.conf.Configuration; + +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class ProfileAWSCredentialsProvider extends AbstractAWSCredentialProvider { + public static final String NAME + = "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider"; + public static final String PROFILE_FILE = "fs.s3a.auth.profile.file"; + public static final String PROFILE_NAME = "fs.s3a.auth.profile.name"; + + private final ProfileCredentialsProvider pcp; + + private static Path getCredentialsPath(Configuration conf) { + String credentialsFile = conf.get(PROFILE_FILE, null); + if (credentialsFile == null) { + credentialsFile = SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null); + } + Path path = (credentialsFile == null) ? + FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials") + : FileSystems.getDefault().getPath(credentialsFile); + return path; + } + + private static String getCredentialsName(Configuration conf) { + String profileName = conf.get(PROFILE_NAME, null); + if (profileName == null) { + profileName = SystemUtils.getEnvironmentVariable("AWS_PROFILE", "default"); + } + return profileName; + } + + public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { + super(uri, conf); + ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); + builder.profileName(getCredentialsName(conf)).profileFile(ProfileFile.builder().content(getCredentialsPath(conf)).type(ProfileFile.Type.CREDENTIALS).build()); + pcp = builder.build(); + } + + public AwsCredentials resolveCredentials() { + return pcp.resolveCredentials(); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index d51bc954a6329..d26c1a7e2de33 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -18,8 +18,7 @@ package org.apache.hadoop.fs.s3a; -import java.io.IOException; -import java.io.InterruptedIOException; +import java.io.*; import java.net.URI; import java.nio.file.AccessDeniedException; import java.util.ArrayList; @@ -35,6 +34,7 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.apache.hadoop.fs.s3a.auth.*; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; @@ -47,11 +47,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider; -import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider; -import org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory; -import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; -import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; import org.apache.hadoop.fs.s3a.auth.delegation.CountInvocationsProvider; import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils; @@ -139,6 +134,32 @@ public void testInstantiationChain() throws Throwable { assertCredentialProviders(expectedClasses, list); } + @Test + public void testProfileAWSCredentialsProvider() throws Throwable { + Configuration conf = new Configuration(false); + conf.set(AWS_CREDENTIALS_PROVIDER, ProfileAWSCredentialsProvider.NAME); + try (FileWriter fileWriter = new FileWriter("testcred"); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + bufferedWriter.write("[default]\n" + + "aws_access_key_id = defaultaccesskeyid\n" + + "aws_secret_access_key = defaultsecretkeyid\n"); + bufferedWriter.write("[nondefault]\n" + + "aws_access_key_id = nondefaultaccesskeyid\n" + + "aws_secret_access_key = nondefaultsecretkeyid\n"); + } + conf.set(ProfileAWSCredentialsProvider.PROFILE_FILE, "testcred"); + URI testUri = new URI("s3a://bucket1"); + AWSCredentialProviderList list = createAWSCredentialProviderList(testUri, conf); + assertCredentialProviders(Collections.singletonList(ProfileAWSCredentialsProvider.class), list); + AwsCredentials credentials = list.resolveCredentials(); + assertEquals("defaultaccesskeyid", credentials.accessKeyId()); + assertEquals("defaultsecretkeyid", credentials.secretAccessKey()); + conf.set(ProfileAWSCredentialsProvider.PROFILE_NAME, "nondefault"); + list = createAWSCredentialProviderList(testUri, conf); + credentials = list.resolveCredentials(); + assertEquals("nondefaultaccesskeyid", credentials.accessKeyId()); + assertEquals("nondefaultsecretkeyid", credentials.secretAccessKey()); + } + @Test public void testDefaultChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); From 05b5295cf88b5aa2a5c836e52a7dbc09a8d5a174 Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Wed, 5 Feb 2025 09:50:03 -0800 Subject: [PATCH 04/10] Remove typo from core-default.xml --- .../hadoop-common/src/main/resources/core-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 117ec1bab0b4e..67531d60e39ca 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1430,7 +1430,7 @@ org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, - org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider, + org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider Comma-separated class names of credential provider classes which implement From e62e1b9ac382aec6d39bd16bebc47cec1c5db835 Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Fri, 7 Feb 2025 11:12:47 -0800 Subject: [PATCH 05/10] Remove ProfileAWSCredentialsProvider from STANDARD_AWS_PROVIDERS --- .../hadoop/fs/s3a/auth/CredentialProviderListFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index 116349c49eac1..941ce741151d5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -84,8 +84,7 @@ public final class CredentialProviderListFactory { EnvironmentVariableCredentialsProvider.class, IAMInstanceCredentialsProvider.class, SimpleAWSCredentialsProvider.class, - TemporaryAWSCredentialsProvider.class, - ProfileAWSCredentialsProvider.class)); + TemporaryAWSCredentialsProvider.class)); /** V1 credential provider: {@value}. */ public static final String ANONYMOUS_CREDENTIALS_V1 = From 4da825c94ac72ec4ab8c1b269315cbf125f55cca Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Tue, 11 Feb 2025 14:21:38 -0800 Subject: [PATCH 06/10] Address review comments --- .../auth/ProfileAWSCredentialsProvider.java | 79 ++++++++++++++++--- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 27 +++++-- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java index acdad9fc52480..59772389f1fd7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java @@ -1,44 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.s3a.auth; +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.profiles.ProfileFile; +import org.apache.commons.lang3.SystemUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; - -import org.apache.commons.lang3.SystemUtils; import org.apache.hadoop.conf.Configuration; -import java.net.URI; -import java.nio.file.FileSystems; -import java.nio.file.Path; - @InterfaceAudience.Public @InterfaceStability.Evolving public class ProfileAWSCredentialsProvider extends AbstractAWSCredentialProvider { + private static final Logger LOG = LoggerFactory.getLogger(ProfileAWSCredentialsProvider.class); + public static final String NAME = "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider"; + + /** Conf setting for credentials file path*/ public static final String PROFILE_FILE = "fs.s3a.auth.profile.file"; + + /** Conf setting for profile name*/ public static final String PROFILE_NAME = "fs.s3a.auth.profile.name"; + /** Environment variable for credentials file path*/ + public static final String CREDENTIALS_FILE_ENV = "AWS_SHARED_CREDENTIALS_FILE"; + /** Environment variable for profile name*/ + public static final String PROFILE_ENV = "AWS_PROFILE"; + private final ProfileCredentialsProvider pcp; private static Path getCredentialsPath(Configuration conf) { String credentialsFile = conf.get(PROFILE_FILE, null); if (credentialsFile == null) { - credentialsFile = SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null); + credentialsFile = SystemUtils.getEnvironmentVariable(CREDENTIALS_FILE_ENV, null); + if (credentialsFile != null) { + LOG.info("Fetched credentials file path from environment variable"); + } + } + else { + LOG.info("Fetched credentials file path from conf"); + } + if (credentialsFile == null) { + LOG.info("Using default credentials file path"); + return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws", "credentials"); + } + else { + return FileSystems.getDefault().getPath(credentialsFile); } - Path path = (credentialsFile == null) ? - FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials") - : FileSystems.getDefault().getPath(credentialsFile); - return path; } private static String getCredentialsName(Configuration conf) { String profileName = conf.get(PROFILE_NAME, null); if (profileName == null) { - profileName = SystemUtils.getEnvironmentVariable("AWS_PROFILE", "default"); + profileName = SystemUtils.getEnvironmentVariable(PROFILE_ENV, null); + if (profileName == null) { + profileName = "default"; + LOG.info("Using default profile name"); + } + else { + LOG.info("Fetched profile name from environment variable"); + } + } + else { + LOG.info("Fetched profile name from conf"); } return profileName; } @@ -46,7 +95,11 @@ private static String getCredentialsName(Configuration conf) { public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { super(uri, conf); ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); - builder.profileName(getCredentialsName(conf)).profileFile(ProfileFile.builder().content(getCredentialsPath(conf)).type(ProfileFile.Type.CREDENTIALS).build()); + builder.profileName(getCredentialsName(conf)) + .profileFile(ProfileFile.builder() + .content(getCredentialsPath(conf)) + .type(ProfileFile.Type.CREDENTIALS) + .build()); pcp = builder.build(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index d26c1a7e2de33..8874cd9320785 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -18,7 +18,11 @@ package org.apache.hadoop.fs.s3a; -import java.io.*; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.InterruptedIOException; +import java.io.IOException; import java.net.URI; import java.nio.file.AccessDeniedException; import java.util.ArrayList; @@ -34,7 +38,6 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; -import org.apache.hadoop.fs.s3a.auth.*; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; @@ -47,6 +50,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider; +import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider; +import org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory; +import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; +import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; +import org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.delegation.CountInvocationsProvider; import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils; @@ -138,7 +147,9 @@ public void testInstantiationChain() throws Throwable { public void testProfileAWSCredentialsProvider() throws Throwable { Configuration conf = new Configuration(false); conf.set(AWS_CREDENTIALS_PROVIDER, ProfileAWSCredentialsProvider.NAME); - try (FileWriter fileWriter = new FileWriter("testcred"); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + File tempFile = File.createTempFile("testcred", ".conf", new File("target")); + tempFile.deleteOnExit(); + try (FileWriter fileWriter = new FileWriter(tempFile); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { bufferedWriter.write("[default]\n" + "aws_access_key_id = defaultaccesskeyid\n" + "aws_secret_access_key = defaultsecretkeyid\n"); @@ -146,18 +157,18 @@ public void testProfileAWSCredentialsProvider() throws Throwable { + "aws_access_key_id = nondefaultaccesskeyid\n" + "aws_secret_access_key = nondefaultsecretkeyid\n"); } - conf.set(ProfileAWSCredentialsProvider.PROFILE_FILE, "testcred"); + conf.set(ProfileAWSCredentialsProvider.PROFILE_FILE, tempFile.getAbsolutePath()); URI testUri = new URI("s3a://bucket1"); AWSCredentialProviderList list = createAWSCredentialProviderList(testUri, conf); assertCredentialProviders(Collections.singletonList(ProfileAWSCredentialsProvider.class), list); AwsCredentials credentials = list.resolveCredentials(); - assertEquals("defaultaccesskeyid", credentials.accessKeyId()); - assertEquals("defaultsecretkeyid", credentials.secretAccessKey()); + Assertions.assertThat(credentials.accessKeyId()).isEqualTo("defaultaccesskeyid"); + Assertions.assertThat(credentials.secretAccessKey()).isEqualTo("defaultsecretkeyid"); conf.set(ProfileAWSCredentialsProvider.PROFILE_NAME, "nondefault"); list = createAWSCredentialProviderList(testUri, conf); credentials = list.resolveCredentials(); - assertEquals("nondefaultaccesskeyid", credentials.accessKeyId()); - assertEquals("nondefaultsecretkeyid", credentials.secretAccessKey()); + Assertions.assertThat(credentials.accessKeyId()).isEqualTo("nondefaultaccesskeyid"); + Assertions.assertThat(credentials.secretAccessKey()).isEqualTo("nondefaultsecretkeyid"); } @Test From 52c722f23f03672b760cae9469ab4ab57a8da0c5 Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Wed, 26 Feb 2025 12:50:53 -0800 Subject: [PATCH 07/10] Address more review comments --- .../auth/ProfileAWSCredentialsProvider.java | 12 ++++++------ .../tools/hadoop-aws/authentication.md | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java index 59772389f1fd7..ab15e32cfe692 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java @@ -59,14 +59,14 @@ private static Path getCredentialsPath(Configuration conf) { if (credentialsFile == null) { credentialsFile = SystemUtils.getEnvironmentVariable(CREDENTIALS_FILE_ENV, null); if (credentialsFile != null) { - LOG.info("Fetched credentials file path from environment variable"); + LOG.debug("Fetched credentials file path from environment variable"); } } else { - LOG.info("Fetched credentials file path from conf"); + LOG.debug("Fetched credentials file path from conf"); } if (credentialsFile == null) { - LOG.info("Using default credentials file path"); + LOG.debug("Using default credentials file path"); return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws", "credentials"); } else { @@ -80,14 +80,14 @@ private static String getCredentialsName(Configuration conf) { profileName = SystemUtils.getEnvironmentVariable(PROFILE_ENV, null); if (profileName == null) { profileName = "default"; - LOG.info("Using default profile name"); + LOG.debug("Using default profile name"); } else { - LOG.info("Fetched profile name from environment variable"); + LOG.debug("Fetched profile name from environment variable"); } } else { - LOG.info("Fetched profile name from conf"); + LOG.debug("Fetched profile name from conf"); } return profileName; } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md index 8f22aa9df452e..930418041c9cc 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md @@ -57,6 +57,9 @@ For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade Comma-separated class names of credential provider classes which implement software.amazon.awssdk.auth.credentials.AwsCredentialsProvider. + + org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider is not included in + the chain by default. When S3A delegation tokens are not enabled, this list will be used to directly authenticate with S3 and other AWS services. @@ -171,6 +174,7 @@ There are a number of AWS Credential Providers inside the `hadoop-aws` JAR: | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` | Anonymous Login | | `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider` | [Assumed Role credentials](./assumed_roles.html) | | `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | EC2/k8s instance credentials | +| `org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider` | Session Credentials in profile file | There are also many in the Amazon SDKs, with the common ones being as follows @@ -222,6 +226,21 @@ Note: configuration files MUST be in the `~/.aws/` directory on the local filesystem in all hosts in the cluster. +### Credentials from profile with `ProfileAWSCredentialsProvider`* + +This is a non-default provider that fetches credentials from a profile file, +acting as a Hadoop wrapper around ProfileCredentialsProvider. The profile file and +profile name are both resolved as follows. + +1. If the configuration setting is specified, that takes priority (`fs.s3a.auth.profile.file` + for profile file and `fs.s3a.auth.profile.name` for profile name). +2. If a configuration setting is absent, but the environment variables for + the setting(AWS_SHARED_CREDENTIALS_FILE for profile file and AWS_PROFILE for + profile name) is defined, then the variable is used. +3. If neither configuration setting nor environment variable is present, then + the values default to `~/.aws/credentials` for the profile file, and `default` + for the profile name. + ### Using Session Credentials with `TemporaryAWSCredentialsProvider` [Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) From 3b20f6be4adb9ba3650c8473869eaef51dcff4e9 Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Wed, 12 Mar 2025 13:29:49 -0700 Subject: [PATCH 08/10] Address more review comments --- .../auth/ProfileAWSCredentialsProvider.java | 118 +++++++++--------- .../tools/hadoop-aws/authentication.md | 14 ++- 2 files changed, 68 insertions(+), 64 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java index ab15e32cfe692..097e9c06d5e4a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java @@ -36,74 +36,74 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public class ProfileAWSCredentialsProvider extends AbstractAWSCredentialProvider { - private static final Logger LOG = LoggerFactory.getLogger(ProfileAWSCredentialsProvider.class); + private static final Logger LOG = LoggerFactory.getLogger(ProfileAWSCredentialsProvider.class); - public static final String NAME - = "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider"; + public static final String NAME + = "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider"; - /** Conf setting for credentials file path*/ - public static final String PROFILE_FILE = "fs.s3a.auth.profile.file"; + /** Conf setting for credentials file path*/ + public static final String PROFILE_FILE = "fs.s3a.auth.profile.file"; - /** Conf setting for profile name*/ - public static final String PROFILE_NAME = "fs.s3a.auth.profile.name"; + /** Conf setting for profile name*/ + public static final String PROFILE_NAME = "fs.s3a.auth.profile.name"; - /** Environment variable for credentials file path*/ - public static final String CREDENTIALS_FILE_ENV = "AWS_SHARED_CREDENTIALS_FILE"; - /** Environment variable for profile name*/ - public static final String PROFILE_ENV = "AWS_PROFILE"; + /** Environment variable for credentials file path*/ + public static final String CREDENTIALS_FILE_ENV = "AWS_SHARED_CREDENTIALS_FILE"; + /** Environment variable for profile name*/ + public static final String PROFILE_ENV = "AWS_PROFILE"; - private final ProfileCredentialsProvider pcp; + private final ProfileCredentialsProvider pcp; - private static Path getCredentialsPath(Configuration conf) { - String credentialsFile = conf.get(PROFILE_FILE, null); - if (credentialsFile == null) { - credentialsFile = SystemUtils.getEnvironmentVariable(CREDENTIALS_FILE_ENV, null); - if (credentialsFile != null) { - LOG.debug("Fetched credentials file path from environment variable"); - } - } - else { - LOG.debug("Fetched credentials file path from conf"); - } - if (credentialsFile == null) { - LOG.debug("Using default credentials file path"); - return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws", "credentials"); - } - else { - return FileSystems.getDefault().getPath(credentialsFile); - } + private static Path getCredentialsPath(Configuration conf) { + String credentialsFile = conf.get(PROFILE_FILE, null); + if (credentialsFile == null) { + credentialsFile = SystemUtils.getEnvironmentVariable(CREDENTIALS_FILE_ENV, null); + if (credentialsFile != null) { + LOG.debug("Fetched credentials file path from environment variable"); + } } - - private static String getCredentialsName(Configuration conf) { - String profileName = conf.get(PROFILE_NAME, null); - if (profileName == null) { - profileName = SystemUtils.getEnvironmentVariable(PROFILE_ENV, null); - if (profileName == null) { - profileName = "default"; - LOG.debug("Using default profile name"); - } - else { - LOG.debug("Fetched profile name from environment variable"); - } - } - else { - LOG.debug("Fetched profile name from conf"); - } - return profileName; + else { + LOG.debug("Fetched credentials file path from conf"); } - - public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { - super(uri, conf); - ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); - builder.profileName(getCredentialsName(conf)) - .profileFile(ProfileFile.builder() - .content(getCredentialsPath(conf)) - .type(ProfileFile.Type.CREDENTIALS) - .build()); - pcp = builder.build(); + if (credentialsFile == null) { + LOG.debug("Using default credentials file path"); + return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws", "credentials"); + } + else { + return FileSystems.getDefault().getPath(credentialsFile); } + } - public AwsCredentials resolveCredentials() { - return pcp.resolveCredentials(); + private static String getCredentialsName(Configuration conf) { + String profileName = conf.get(PROFILE_NAME, null); + if (profileName == null) { + profileName = SystemUtils.getEnvironmentVariable(PROFILE_ENV, null); + if (profileName == null) { + profileName = "default"; + LOG.debug("Using default profile name"); + } + else { + LOG.debug("Fetched profile name from environment variable"); + } } + else { + LOG.debug("Fetched profile name from conf"); + } + return profileName; + } + + public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { + super(uri, conf); + ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); + builder.profileName(getCredentialsName(conf)) + .profileFile(ProfileFile.builder() + .content(getCredentialsPath(conf)) + .type(ProfileFile.Type.CREDENTIALS) + .build()); + pcp = builder.build(); + } + + public AwsCredentials resolveCredentials() { + return pcp.resolveCredentials(); + } } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md index 930418041c9cc..af60a48f7da1f 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/authentication.md @@ -57,7 +57,7 @@ For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade Comma-separated class names of credential provider classes which implement software.amazon.awssdk.auth.credentials.AwsCredentialsProvider. - + org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider is not included in the chain by default. @@ -229,18 +229,22 @@ Note: ### Credentials from profile with `ProfileAWSCredentialsProvider`* This is a non-default provider that fetches credentials from a profile file, -acting as a Hadoop wrapper around ProfileCredentialsProvider. The profile file and +acting as a Hadoop wrapper around [ProfileCredentialsProvider](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/ProfileCredentialsProvider.html). The profile file and profile name are both resolved as follows. -1. If the configuration setting is specified, that takes priority (`fs.s3a.auth.profile.file` +1. If the configuration setting is specified, that takes priority ( `fs.s3a.auth.profile.file` for profile file and `fs.s3a.auth.profile.name` for profile name). -2. If a configuration setting is absent, but the environment variables for - the setting(AWS_SHARED_CREDENTIALS_FILE for profile file and AWS_PROFILE for +2. If a configuration setting is absent, but the environment variable for + the setting( `AWS_SHARED_CREDENTIALS_FILE` for profile file and `AWS_PROFILE` for profile name) is defined, then the variable is used. 3. If neither configuration setting nor environment variable is present, then the values default to `~/.aws/credentials` for the profile file, and `default` for the profile name. + +*Important*: This profile file must be on every node in the _cluster_. +If this is not the case, delegation tokens can be used to collect the current credentials and propagate them. + ### Using Session Credentials with `TemporaryAWSCredentialsProvider` [Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) From d83e7fb997c7831fea0e83a6fa8b00566ca97588 Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Tue, 18 Mar 2025 10:08:10 -0700 Subject: [PATCH 09/10] Address checkstyle --- .../auth/ProfileAWSCredentialsProvider.java | 23 ++++++++----------- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 11 +++++---- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java index 097e9c06d5e4a..09eca499b0ea0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java @@ -41,15 +41,15 @@ public class ProfileAWSCredentialsProvider extends AbstractAWSCredentialProvider public static final String NAME = "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider"; - /** Conf setting for credentials file path*/ + /** Conf setting for credentials file path.*/ public static final String PROFILE_FILE = "fs.s3a.auth.profile.file"; - /** Conf setting for profile name*/ + /** Conf setting for profile name.*/ public static final String PROFILE_NAME = "fs.s3a.auth.profile.name"; - /** Environment variable for credentials file path*/ + /** Environment variable for credentials file path.*/ public static final String CREDENTIALS_FILE_ENV = "AWS_SHARED_CREDENTIALS_FILE"; - /** Environment variable for profile name*/ + /** Environment variable for profile name.*/ public static final String PROFILE_ENV = "AWS_PROFILE"; private final ProfileCredentialsProvider pcp; @@ -61,15 +61,14 @@ private static Path getCredentialsPath(Configuration conf) { if (credentialsFile != null) { LOG.debug("Fetched credentials file path from environment variable"); } - } - else { + } else { LOG.debug("Fetched credentials file path from conf"); } if (credentialsFile == null) { LOG.debug("Using default credentials file path"); - return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws", "credentials"); - } - else { + return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(), + ".aws", "credentials"); + } else { return FileSystems.getDefault().getPath(credentialsFile); } } @@ -81,12 +80,10 @@ private static String getCredentialsName(Configuration conf) { if (profileName == null) { profileName = "default"; LOG.debug("Using default profile name"); - } - else { + } else { LOG.debug("Fetched profile name from environment variable"); } - } - else { + } else { LOG.debug("Fetched profile name from conf"); } return profileName; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 8874cd9320785..fa9cbf4cfb726 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -149,13 +149,14 @@ public void testProfileAWSCredentialsProvider() throws Throwable { conf.set(AWS_CREDENTIALS_PROVIDER, ProfileAWSCredentialsProvider.NAME); File tempFile = File.createTempFile("testcred", ".conf", new File("target")); tempFile.deleteOnExit(); - try (FileWriter fileWriter = new FileWriter(tempFile); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + try (FileWriter fileWriter = new FileWriter(tempFile); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { bufferedWriter.write("[default]\n" - + "aws_access_key_id = defaultaccesskeyid\n" - + "aws_secret_access_key = defaultsecretkeyid\n"); + + "aws_access_key_id = defaultaccesskeyid\n" + + "aws_secret_access_key = defaultsecretkeyid\n"); bufferedWriter.write("[nondefault]\n" - + "aws_access_key_id = nondefaultaccesskeyid\n" - + "aws_secret_access_key = nondefaultsecretkeyid\n"); + + "aws_access_key_id = nondefaultaccesskeyid\n" + + "aws_secret_access_key = nondefaultsecretkeyid\n"); } conf.set(ProfileAWSCredentialsProvider.PROFILE_FILE, tempFile.getAbsolutePath()); URI testUri = new URI("s3a://bucket1"); From 825d372deff4b75718a03a6ec766d64009fe8a0a Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Fri, 21 Mar 2025 11:11:54 -0700 Subject: [PATCH 10/10] Address more checkstyle --- .../hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index fa9cbf4cfb726..c2d82624878df 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -152,11 +152,11 @@ public void testProfileAWSCredentialsProvider() throws Throwable { try (FileWriter fileWriter = new FileWriter(tempFile); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { bufferedWriter.write("[default]\n" - + "aws_access_key_id = defaultaccesskeyid\n" - + "aws_secret_access_key = defaultsecretkeyid\n"); + + "aws_access_key_id = defaultaccesskeyid\n" + + "aws_secret_access_key = defaultsecretkeyid\n"); bufferedWriter.write("[nondefault]\n" - + "aws_access_key_id = nondefaultaccesskeyid\n" - + "aws_secret_access_key = nondefaultsecretkeyid\n"); + + "aws_access_key_id = nondefaultaccesskeyid\n" + + "aws_secret_access_key = nondefaultsecretkeyid\n"); } conf.set(ProfileAWSCredentialsProvider.PROFILE_FILE, tempFile.getAbsolutePath()); URI testUri = new URI("s3a://bucket1");