From 76ca4e1367dffa37983993364b7084ae525ead6d Mon Sep 17 00:00:00 2001 From: ashutoshpant <51799223+ashutoshpant@users.noreply.github.com> Date: Mon, 11 Jul 2022 12:02:10 -0400 Subject: [PATCH 1/2] Added extra client attribute to the S3ClientCreationParameters that contains the full s3a path --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 40671e0d334cc..91da04f7b694e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -896,7 +896,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { .withRequestHandlers(auditManager.createRequestHandlers()); s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf) - .createS3Client(getUri(), + .createS3Client(getUri(), name, parameters); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 34674c788901f..822a489131802 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -53,11 +53,12 @@ public interface S3ClientFactory { * Creates a new {@link AmazonS3} client. * * @param uri S3A file system URI + * @param name S3A file system URI that contains the full path * @param parameters parameter object * @return S3 client * @throws IOException IO problem */ - AmazonS3 createS3Client(URI uri, + AmazonS3 createS3Client(URI uri, URI name, S3ClientCreationParameters parameters) throws IOException; /** From 83e0cce7129325027e4e34d192c1a99e3fb4666c Mon Sep 17 00:00:00 2001 From: ashutoshpant <51799223+ashutoshpant@users.noreply.github.com> Date: Mon, 11 Jul 2022 12:12:29 -0400 Subject: [PATCH 2/2] Updated all uses of createS3Client --- .../java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java | 1 + .../java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java | 1 + .../test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java | 2 +- .../s3a/auth/delegation/ITestSessionDelegationInFileystem.java | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index c374ef7397c97..78208ec3a7e54 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -110,6 +110,7 @@ public class DefaultS3ClientFactory extends Configured @Override public AmazonS3 createS3Client( final URI uri, + final URI name, final S3ClientCreationParameters parameters) throws IOException { Configuration conf = getConf(); bucket = uri.getHost(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index 761dd558063ba..2f25aab58187a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -181,6 +181,7 @@ private AmazonS3 createS3Client(Configuration conf, .newStatisticsFromAwsSdk()); AmazonS3 client = factory.createS3Client( new URI("s3a://localhost/"), + new URI("s3a://localhost/path/") parameters); Assertions.assertThat(client.getRegionName()) .describedAs("Client region name") diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java index bd121ba2728eb..f059382438087 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java @@ -34,7 +34,7 @@ public class MockS3ClientFactory implements S3ClientFactory { @Override - public AmazonS3 createS3Client(URI uri, + public AmazonS3 createS3Client(URI uri, URI name, final S3ClientCreationParameters parameters) { AmazonS3 s3 = mock(AmazonS3.class); String bucket = uri.getHost(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index bc223bad457c1..f701a391dab51 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -593,7 +593,7 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()) .withUserAgentSuffix("ITestSessionDelegationInFileystem"); - AmazonS3 s3 = factory.createS3Client(landsat, parameters); + AmazonS3 s3 = factory.createS3Client(landsat, landsat, parameters); return Invoker.once("HEAD", host, () -> s3.getObjectMetadata(host, landsat.getPath().substring(1)));