Skip to content

Commit 3e151bd

Browse files
adnanhemaniadnanhemani
committed
PR Revision 1
1 parent 33b3350 commit 3e151bd

File tree

3 files changed

+82
-82
lines changed

3 files changed

+82
-82
lines changed

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -418,12 +418,13 @@ private static Region getS3RegionFromEndpoint(final String endpoint,
418418
return;
419419
}
420420

421-
LOG_S3AG_ENABLED.info("S3 Access Grants plugin is enabled.");
422-
boolean isFallbackEnabled = conf.getBoolean(AWS_S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED, false);
421+
boolean isFallbackEnabled =
422+
conf.getBoolean(AWS_S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED, false);
423423
S3AccessGrantsPlugin accessGrantsPlugin =
424-
S3AccessGrantsPlugin.builder().enableFallback(isFallbackEnabled).build();
424+
S3AccessGrantsPlugin.builder().enableFallback(isFallbackEnabled).build();
425425
builder.addPlugin(accessGrantsPlugin);
426-
LOG_S3AG_ENABLED.info("S3 Access Grants plugin is added to S3 client with fallback: {}", isFallbackEnabled);
426+
LOG_S3AG_ENABLED.info(
427+
"S3 Access Grants plugin is enabled with IAM fallback set to {}", isFallbackEnabled);
427428
}
428429

429430
}

‎hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -617,16 +617,16 @@ java.io.IOException: From option fs.s3a.aws.credentials.provider java.lang.Class
617617
## S3 Authorization Using S3 Access Grants
618618

619619
[S3 Access Grants](https://aws.amazon.com/s3/features/access-grants/) can be used to grant accesses to S3 data using IAM Principals.
620-
In order to enable S3 Access Grants to work with S3A, we enable the
621-
[S3 Access Grants plugin](https://github.com/aws/aws-s3-accessgrants-plugin-java-v2) on all S3 clients,
620+
In order to enable S3 Access Grants, S3A utilizes the
621+
[S3 Access Grants plugin](https://github.com/aws/aws-s3-accessgrants-plugin-java-v2) on all S3 clients,
622622
which is found within the AWS Java SDK bundle (v2.23.19+).
623623

624-
We support both cross-region access (by default) and the
625-
[fallback-to-IAM configuration](https://github.com/aws/aws-s3-accessgrants-plugin-java-v2?tab=readme-ov-file#using-the-plugin)
626-
which allows you to fallback to using your IAM role (and its permission sets directly) to access your S3 data in the case that S3 Access Grants
627-
is unable to authorize your S3 call.
624+
S3A supports both cross-region access (by default) and the
625+
[fallback-to-IAM configuration](https://github.com/aws/aws-s3-accessgrants-plugin-java-v2?tab=readme-ov-file#using-the-plugin)
626+
which allows S3A to fallback to using the IAM role (and its permission sets directly) to access S3 data in the case that S3 Access Grants
627+
is unable to authorize the S3 call.
628628

629-
The following is how you can enable this feature:
629+
The following is how this feature can be enabled:
630630

631631
```xml
632632
<property>
@@ -640,11 +640,11 @@ The following is how you can enable this feature:
640640
</property>
641641
```
642642

643-
Please note that we only enable the [S3 Access Grants plugin](https://github.com/aws/aws-s3-accessgrants-plugin-java-v2) on the S3 clients
644-
as part of this feature. Any usage issues or bug reporting should be done directly at the plugin's
643+
Please note that S3A only enables the [S3 Access Grants plugin](https://github.com/aws/aws-s3-accessgrants-plugin-java-v2) on the S3 clients
644+
as part of this feature. Any usage issues or bug reporting should be done directly at the plugin's
645645
[GitHub repo](https://github.com/aws/aws-s3-accessgrants-plugin-java-v2/issues).
646646

647-
For more details on using S3 Access Grants, please refer to
647+
For more details on using S3 Access Grants, please refer to
648648
[Managing access with S3 Access Grants](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-grants.html).
649649

650650
## <a name="hadoop_credential_providers"></a>Storing secrets with Hadoop Credential Providers

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AccessGrantConfiguration.java‎

Lines changed: 67 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525

2626
import software.amazon.awssdk.awscore.AwsClient;
2727
import software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsIdentityProvider;
28-
import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
2928

3029
import java.io.IOException;
3130
import java.net.URI;
@@ -38,71 +37,71 @@
3837
* Test S3 Access Grants configurations.
3938
*/
4039
public class TestS3AccessGrantConfiguration extends AbstractHadoopTestBase {
41-
/**
42-
* This credential provider will be attached to any client
43-
* that has been configured with the S3 Access Grants plugin.
44-
* {@link software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin}.
45-
*/
46-
public static final String S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS =
47-
S3AccessGrantsIdentityProvider.class.getName();
48-
49-
@Test
50-
public void testS3AccessGrantsEnabled() throws IOException, URISyntaxException {
51-
// Feature is explicitly enabled
52-
AwsClient s3AsyncClient = getAwsClient(createConfig(true), true);
53-
Assert.assertEquals(
54-
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
55-
getCredentialProviderName(s3AsyncClient));
56-
57-
AwsClient s3Client = getAwsClient(createConfig(true), false);
58-
Assert.assertEquals(
59-
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
60-
getCredentialProviderName(s3Client));
61-
}
62-
63-
@Test
64-
public void testS3AccessGrantsDisabled() throws IOException, URISyntaxException {
65-
// Disabled by default
66-
AwsClient s3AsyncDefaultClient = getAwsClient(new Configuration(), true);
67-
Assert.assertNotEquals(
68-
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
69-
getCredentialProviderName(s3AsyncDefaultClient));
70-
71-
AwsClient s3DefaultClient = getAwsClient(new Configuration(), true);
72-
Assert.assertNotEquals(
73-
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
74-
getCredentialProviderName(s3DefaultClient));
75-
76-
// Disabled if explicitly set
77-
AwsClient s3AsyncExplicitlyDisabledClient = getAwsClient(createConfig(false), true);
78-
Assert.assertNotEquals(
79-
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
80-
getCredentialProviderName(s3AsyncExplicitlyDisabledClient));
81-
82-
AwsClient s3ExplicitlyDisabledClient = getAwsClient(createConfig(false), true);
83-
Assert.assertNotEquals(
84-
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
85-
getCredentialProviderName(s3ExplicitlyDisabledClient));
86-
}
87-
88-
private Configuration createConfig(boolean s3agEnabled) {
89-
Configuration conf = new Configuration();
90-
conf.setBoolean(AWS_S3_ACCESS_GRANTS_ENABLED, s3agEnabled);
91-
return conf;
92-
}
93-
94-
private String getCredentialProviderName(AwsClient awsClient) {
95-
return awsClient.serviceClientConfiguration().credentialsProvider().getClass().getName();
96-
}
97-
98-
private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> AwsClient
99-
getAwsClient(Configuration conf, boolean asyncClient) throws IOException, URISyntaxException {
100-
DefaultS3ClientFactory factory = new DefaultS3ClientFactory();
101-
factory.setConf(conf);
102-
S3ClientFactory.S3ClientCreationParameters parameters =
103-
new S3ClientFactory.S3ClientCreationParameters();
104-
URI uri = new URI("any-uri");
105-
return asyncClient ?
106-
factory.createS3AsyncClient(uri, parameters): factory.createS3Client(uri, parameters);
107-
}
40+
/**
41+
* This credential provider will be attached to any client
42+
* that has been configured with the S3 Access Grants plugin.
43+
* {@link software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin}.
44+
*/
45+
public static final String S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS =
46+
S3AccessGrantsIdentityProvider.class.getName();
47+
48+
@Test
49+
public void testS3AccessGrantsEnabled() throws IOException, URISyntaxException {
50+
// Feature is explicitly enabled
51+
AwsClient s3AsyncClient = getAwsClient(createConfig(true), true);
52+
Assert.assertEquals(
53+
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
54+
getCredentialProviderName(s3AsyncClient));
55+
56+
AwsClient s3Client = getAwsClient(createConfig(true), false);
57+
Assert.assertEquals(
58+
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
59+
getCredentialProviderName(s3Client));
60+
}
61+
62+
@Test
63+
public void testS3AccessGrantsDisabled() throws IOException, URISyntaxException {
64+
// Disabled by default
65+
AwsClient s3AsyncDefaultClient = getAwsClient(new Configuration(), true);
66+
Assert.assertNotEquals(
67+
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
68+
getCredentialProviderName(s3AsyncDefaultClient));
69+
70+
AwsClient s3DefaultClient = getAwsClient(new Configuration(), true);
71+
Assert.assertNotEquals(
72+
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
73+
getCredentialProviderName(s3DefaultClient));
74+
75+
// Disabled if explicitly set
76+
AwsClient s3AsyncExplicitlyDisabledClient = getAwsClient(createConfig(false), true);
77+
Assert.assertNotEquals(
78+
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
79+
getCredentialProviderName(s3AsyncExplicitlyDisabledClient));
80+
81+
AwsClient s3ExplicitlyDisabledClient = getAwsClient(createConfig(false), true);
82+
Assert.assertNotEquals(
83+
S3_ACCESS_GRANTS_EXPECTED_CREDENTIAL_PROVIDER_CLASS,
84+
getCredentialProviderName(s3ExplicitlyDisabledClient));
85+
}
86+
87+
private Configuration createConfig(boolean s3agEnabled) {
88+
Configuration conf = new Configuration();
89+
conf.setBoolean(AWS_S3_ACCESS_GRANTS_ENABLED, s3agEnabled);
90+
return conf;
91+
}
92+
93+
private String getCredentialProviderName(AwsClient awsClient) {
94+
return awsClient.serviceClientConfiguration().credentialsProvider().getClass().getName();
95+
}
96+
97+
private AwsClient getAwsClient(Configuration conf, boolean asyncClient)
98+
throws IOException, URISyntaxException {
99+
DefaultS3ClientFactory factory = new DefaultS3ClientFactory();
100+
factory.setConf(conf);
101+
S3ClientFactory.S3ClientCreationParameters parameters =
102+
new S3ClientFactory.S3ClientCreationParameters();
103+
URI uri = new URI("any-uri");
104+
return asyncClient ?
105+
factory.createS3AsyncClient(uri, parameters): factory.createS3Client(uri, parameters);
106+
}
108107
}

0 commit comments

Comments
 (0)