From a9dbc65f199ddae2004722568926f236634dac4d Mon Sep 17 00:00:00 2001
From: HarshitGupta <harshit.gupta@cloudera.com>
Date: Thu, 30 Nov 2023 12:37:07 +0530
Subject: [PATCH 1/5] HADOOP-19004. S3A: Support Authentication through
 HttpSigner API

Move to the new auth flow based signers for aws. * Implement a new Signer Initialization Chain
* Add a new instantiation method
* Add a new test
* Fix Reflection Code for SignerInitialization

(+steve changes)
* move instantiation code into SignerFactory
* CustomHttpSigner goes into production code for broader testing
* some tuning of property values and names of constants/methods.

Change-Id: I8f34af4bd958d631bcb03d7c073ed1da01cb8205
---
 .../org/apache/hadoop/fs/s3a/Constants.java   |  16 +++
 .../hadoop/fs/s3a/DefaultS3ClientFactory.java |  19 ++-
 .../apache/hadoop/fs/s3a/S3AFileSystem.java   |   6 +
 .../hadoop/fs/s3a/auth/CustomHttpSigner.java  |  68 +++++++++
 .../hadoop/fs/s3a/auth/SignerFactory.java     |  65 +++++++++
 .../hadoop/fs/s3a/impl/AWSClientConfig.java   |   1 +
 .../hadoop/fs/s3a/impl/InternalConstants.java |   6 +
 .../hadoop/fs/s3a/auth/ITestHttpSigner.java   | 135 ++++++++++++++++++
 8 files changed, 315 insertions(+), 1 deletion(-)
 create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java
 create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java

diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index fb4f22cedb9ba..0d07c2955ac09 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1543,4 +1543,20 @@ private Constants() {
    * Value: {@value}.
    */
   public static final boolean S3EXPRESS_CREATE_SESSION_DEFAULT = true;
+
+  /**
+   * Flag to switch to a v2 SDK HTTP signer. Value {@value}.
+   */
+  public static final String HTTP_SIGNER_ENABLED = "fs.s3a.http.signer.enabled";
+
+  /**
+   * Default value of {@link #HTTP_SIGNER_ENABLED}: {@value}.
+   */
+  public static final boolean HTTP_SIGNER_ENABLED_DEFAULT = false;
+
+  /**
+   * Classname of the http signer to use when {@link #HTTP_SIGNER_ENABLED}
+   * is true: {@value}.
+   */
+  public static String HTTP_SIGNER_CLASS_NAME = "fs.s3a.http.signer.class";
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 66e8d60689a8a..6b71e636e628a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -32,7 +32,10 @@
 import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
 import software.amazon.awssdk.core.retry.RetryPolicy;
 import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.auth.spi.scheme.AuthScheme;
+import software.amazon.awssdk.http.auth.spi.signer.HttpSigner;
 import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
@@ -52,10 +55,15 @@
 import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
+import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_CLASS_NAME;
+import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED;
+import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED_DEFAULT;
+import static org.apache.hadoop.fs.s3a.auth.SignerFactory.createHttpSigner;
 import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
 import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS;
 import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AUTH_SCHEME_AWS_SIGV_4;
 
 
 /**
@@ -165,11 +173,19 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> Build
             .pathStyleAccessEnabled(parameters.isPathStyleAccess())
             .build();
 
-    return builder
+    S3BaseClientBuilder s3BaseClientBuilder = builder
         .overrideConfiguration(createClientOverrideConfiguration(parameters, conf))
         .credentialsProvider(parameters.getCredentialSet())
         .disableS3ExpressSessionAuth(!parameters.isExpressCreateSession())
         .serviceConfiguration(serviceConfiguration);
+
+    if (conf.getBoolean(HTTP_SIGNER_ENABLED, HTTP_SIGNER_ENABLED_DEFAULT)) {
+      // use an http signer through an AuthScheme
+      final AuthScheme<AwsCredentialsIdentity> signer =
+          createHttpSigner(conf, AUTH_SCHEME_AWS_SIGV_4, HTTP_SIGNER_CLASS_NAME);
+      builder.putAuthScheme(signer);
+    }
+    return (BuilderT) s3BaseClientBuilder;
   }
 
   /**
@@ -177,6 +193,7 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> Build
    * @param parameters parameter object
    * @param conf configuration object
    * @throws IOException any IOE raised, or translated exception
+   * @throws RuntimeException some failures creating an http signer
    * @return the override configuration
    */
   protected ClientOverrideConfiguration createClientOverrideConfiguration(
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index df7d3f1fb6891..4777b221b26c2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -58,6 +58,8 @@
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CreateSessionRequest;
+import software.amazon.awssdk.services.s3.model.CreateSessionResponse;
 import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
@@ -5758,6 +5760,10 @@ public StoreContext createStoreContext() {
         .build();
   }
 
+  public CreateSessionResponse createSessionInternal(CreateSessionRequest createSessionRequest){
+    return this.s3Client.createSession(createSessionRequest);
+  }
+
   /**
    * Create a marker tools operations binding for this store.
    * Auditing:
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java
new file mode 100644
index 0000000000000..81de0b9f9e4a0
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.auth;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner;
+import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest;
+import software.amazon.awssdk.http.auth.spi.signer.AsyncSignedRequest;
+import software.amazon.awssdk.http.auth.spi.signer.HttpSigner;
+import software.amazon.awssdk.http.auth.spi.signer.SignRequest;
+import software.amazon.awssdk.http.auth.spi.signer.SignedRequest;
+import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
+
+/**
+ * Custom signer that delegates to the AWS V4 signer.
+ * Logs at TRACE the string value of any request.
+ * This is in the production code to support testing the signer plugin mechansim.
+ * To use
+ * <pre>
+ *   fs.s3a.http.signer.enabled = true
+ *   fs.s3a.http.signer.class = org.apache.hadoop.fs.s3a.auth.CustomHttpSigner
+ * <pre>
+ */
+public final class CustomHttpSigner implements HttpSigner<AwsCredentialsIdentity> {
+  private static final Logger LOG = LoggerFactory
+      .getLogger(CustomHttpSigner.class);
+
+  /**
+   * The delegate signer.
+   */
+  private final HttpSigner<AwsCredentialsIdentity> delegateSigner;
+
+  public CustomHttpSigner() {
+    delegateSigner = AwsV4HttpSigner.create();
+  }
+
+  @Override
+  public SignedRequest sign(SignRequest<? extends AwsCredentialsIdentity>
+      request) {
+    LOG.trace("Signing request:{}", request.request());
+    return delegateSigner.sign(request);
+  }
+
+  @Override
+  public CompletableFuture<AsyncSignedRequest> signAsync(final AsyncSignRequest<? extends AwsCredentialsIdentity> request) {
+    LOG.trace("Signing async request:{}", request.request());
+    return delegateSigner.signAsync(request);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index 21c390c07940b..1b6aa7f48f11e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -29,12 +29,20 @@
 import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
 import software.amazon.awssdk.core.signer.NoOpSigner;
 import software.amazon.awssdk.core.signer.Signer;
+import software.amazon.awssdk.http.auth.spi.scheme.AuthScheme;
+import software.amazon.awssdk.http.auth.spi.signer.HttpSigner;
+import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
+import software.amazon.awssdk.identity.spi.IdentityProvider;
+import software.amazon.awssdk.identity.spi.IdentityProviders;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.s3a.S3AUtils;
 import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
 
+import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_CLASS_NAME;
 import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
 import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.apache.hadoop.util.Preconditions.checkState;
 
 /**
  * Signer factory used to register and create signers.
@@ -119,4 +127,61 @@ public static Signer createSigner(String signerType, String configKey) throws IO
 
     return signer;
   }
+
+  /**
+   * Create an auth scheme instance from an ID and a signer.
+   * @param schemeId scheme id
+   * @param signer signer
+   * @return the auth scheme
+   */
+  public static AuthScheme<AwsCredentialsIdentity> createAuthScheme(String schemeId,
+      HttpSigner<AwsCredentialsIdentity> signer) {
+    return new AuthScheme<AwsCredentialsIdentity>() {
+      @Override
+      public String schemeId() {
+        return schemeId;
+      }
+      @Override
+      public IdentityProvider<AwsCredentialsIdentity> identityProvider(IdentityProviders providers) {
+        return providers.identityProvider(AwsCredentialsIdentity.class);
+      }
+      @Override
+      public HttpSigner<AwsCredentialsIdentity> signer() {
+        return signer;
+      }
+    };
+  }
+
+  /**
+   * Create an auth scheme by looking up the signer class in the configuration,
+   * loading and instantiating it.
+   * @param conf configuration
+   * @param scheme scheme to bond to
+   * @param configKey configuration key
+   * @return the auth scheme
+   * @throws InstantiationIOException failure to instantiate
+   * @throws IllegalStateException if the signer class is not defined
+   * @throws RuntimeException other configuration problems
+   */
+  public static AuthScheme<AwsCredentialsIdentity> createHttpSigner(
+      Configuration conf, String scheme, String configKey) throws IOException {
+
+    final Class<? extends HttpSigner> clazz = conf.getClass(HTTP_SIGNER_CLASS_NAME,
+        null, HttpSigner.class);
+    checkState(clazz != null, "No http signer class defined in %s", configKey);
+    LOG.debug("Creating http signer {} from {}", clazz, configKey);
+    try {
+      return createAuthScheme(scheme, clazz.newInstance());
+
+    } catch (InstantiationException | IllegalAccessException e) {
+      throw new InstantiationIOException(
+          InstantiationIOException.Kind.InstantiationFailure,
+          null,
+          clazz.getName(),
+          HTTP_SIGNER_CLASS_NAME,
+          e.toString(),
+          e);
+    }
+  }
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
index 263562fe8a704..f6da9d84e0a77 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
@@ -105,6 +105,7 @@ private AWSClientConfig() {
    * @param awsServiceIdentifier service
    * @return the builder inited with signer, timeouts and UA.
    * @throws IOException failure.
+   * @throws RuntimeException some failures creating an http signer
    */
   public static ClientOverrideConfiguration.Builder createClientConfigBuilder(Configuration conf,
       String awsServiceIdentifier) throws IOException {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
index cd78350a5d024..eab3bb52a3451 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
@@ -286,4 +286,10 @@ private InternalConstants() {
           FS_S3A_CREATE_PERFORMANCE_ENABLED,
           DIRECTORY_OPERATIONS_PURGE_UPLOADS,
           ENABLE_MULTI_DELETE));
+
+  /**
+   * AWS V4 Auth Scheme to use when creating signers.
+   */
+  public static final String AUTH_SCHEME_AWS_SIGV_4 = "aws.auth#sigv4";
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
new file mode 100644
index 0000000000000..b41e6826439b1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
@@ -0,0 +1,135 @@
+package org.apache.hadoop.fs.s3a.auth;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS;
+import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE;
+import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_CLASS_NAME;
+import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_S3;
+import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.createMagicFile;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+
+/**
+ * Test the HTTP signer SPI.
+ * Two different UGIs are created; ths simplifies cleanup.
+ */
+public class ITestHttpSigner extends AbstractS3ATestBase {
+  private static final Logger LOG = LoggerFactory
+      .getLogger(ITestHttpSigner.class);
+
+  private static final String TEST_ID_KEY = "TEST_ID_KEY";
+  private static final String TEST_REGION_KEY = "TEST_REGION_KEY";
+
+  private final UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("user1");
+
+  private final UserGroupInformation ugi2 = UserGroupInformation.createRemoteUser("user2");
+
+  private String regionName;
+
+  private String endpoint;
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    final S3AFileSystem fs = getFileSystem();
+    final Configuration conf = fs.getConf();
+    // determine the endpoint -skipping the test.
+    endpoint = conf.getTrimmed(Constants.ENDPOINT, Constants.CENTRAL_ENDPOINT);
+    LOG.debug("Test endpoint is {}", endpoint);
+    regionName = conf.getTrimmed(Constants.AWS_REGION, "");
+    if (regionName.isEmpty()) {
+      regionName = determineRegion(fs.getBucket());
+    }
+    LOG.debug("Determined region name to be [{}] for bucket [{}]", regionName,
+        fs.getBucket());
+  }
+
+  private String determineRegion(String bucketName) throws IOException {
+    return getS3AInternals().getBucketLocation(bucketName);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    super.teardown();
+    FileSystem.closeAllForUGI(ugi1);
+    FileSystem.closeAllForUGI(ugi2);
+  }
+
+  private Configuration createTestConfig(String identifier) {
+    Configuration conf = createConfiguration();
+
+    removeBaseAndBucketOverrides(conf,
+        CUSTOM_SIGNERS,
+        SIGNING_ALGORITHM_S3,
+        ENABLE_MULTI_DELETE);
+
+    conf.setBoolean(HTTP_SIGNER_ENABLED, true);
+    conf.set(HTTP_SIGNER_CLASS_NAME, CustomHttpSigner.class.getName());
+
+    conf.set(TEST_ID_KEY, identifier);
+    conf.set(TEST_REGION_KEY, regionName);
+
+    // make absolutely sure there is no caching.
+    disableFilesystemCaching(conf);
+
+    return conf;
+  }
+
+  @Test
+  public void testCustomSignerAndInitializer()
+      throws IOException, InterruptedException {
+
+    final Path basePath = path(getMethodName());
+    FileSystem fs1 = runStoreOperationsAndVerify(ugi1,
+        new Path(basePath, "customsignerpath1"), "id1");
+
+    FileSystem fs2 = runStoreOperationsAndVerify(ugi2,
+        new Path(basePath, "customsignerpath2"), "id2");
+  }
+
+  private S3AFileSystem runStoreOperationsAndVerify(UserGroupInformation ugi,
+      Path finalPath, String identifier)
+      throws IOException, InterruptedException {
+    Configuration conf = createTestConfig(identifier);
+    return ugi.doAs((PrivilegedExceptionAction<S3AFileSystem>) () -> {
+      S3AFileSystem fs = (S3AFileSystem)finalPath.getFileSystem(conf);
+
+      fs.mkdirs(finalPath);
+
+      // now do some more operations to make sure all is good.
+      final Path subdir = new Path(finalPath, "year=1970/month=1/day=1");
+      fs.mkdirs(subdir);
+
+      final Path file1 = new Path(subdir, "file1");
+      ContractTestUtils.touch(fs, new Path(subdir, "file1"));
+      fs.listStatus(subdir);
+      fs.delete(file1, false);
+      ContractTestUtils.touch(fs, new Path(subdir, "file1"));
+
+      // create a magic file.
+      createMagicFile(fs, subdir);
+      ContentSummary summary = fs.getContentSummary(finalPath);
+      fs.getS3AInternals().abortMultipartUploads(subdir);
+      fs.rename(subdir, new Path(finalPath, "renamed"));
+      fs.delete(finalPath, true);
+      return fs;
+    });
+  }
+}

From acf1505eabe5772a72e89a232730f0502e90ebdb Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Wed, 6 Dec 2023 13:19:15 +0000
Subject: [PATCH 2/5] HADOOP-19004. HttpSigner: yetus feedback

Change-Id: Ibd1e708e5ff6eb3f7e055153e4d199c945650d2c
---
 .../org/apache/hadoop/fs/s3a/Constants.java   |  2 +-
 .../hadoop/fs/s3a/DefaultS3ClientFactory.java |  1 -
 .../hadoop/fs/s3a/auth/CustomHttpSigner.java  |  6 ++++--
 .../hadoop/fs/s3a/auth/SignerFactory.java     |  7 +++++--
 .../hadoop/fs/s3a/impl/InternalConstants.java |  2 +-
 .../hadoop/fs/s3a/auth/ITestHttpSigner.java   | 21 +++++++++++++++++--
 6 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 0d07c2955ac09..c1c12b5948284 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1558,5 +1558,5 @@ private Constants() {
    * Classname of the http signer to use when {@link #HTTP_SIGNER_ENABLED}
    * is true: {@value}.
    */
-  public static String HTTP_SIGNER_CLASS_NAME = "fs.s3a.http.signer.class";
+  public static final String HTTP_SIGNER_CLASS_NAME = "fs.s3a.http.signer.class";
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 6b71e636e628a..5372261d55e71 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -33,7 +33,6 @@
 import software.amazon.awssdk.core.retry.RetryPolicy;
 import software.amazon.awssdk.http.apache.ApacheHttpClient;
 import software.amazon.awssdk.http.auth.spi.scheme.AuthScheme;
-import software.amazon.awssdk.http.auth.spi.signer.HttpSigner;
 import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
 import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
 import software.amazon.awssdk.regions.Region;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java
index 81de0b9f9e4a0..ba1169a5e5987 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomHttpSigner.java
@@ -38,7 +38,7 @@
  * <pre>
  *   fs.s3a.http.signer.enabled = true
  *   fs.s3a.http.signer.class = org.apache.hadoop.fs.s3a.auth.CustomHttpSigner
- * <pre>
+ * </pre>
  */
 public final class CustomHttpSigner implements HttpSigner<AwsCredentialsIdentity> {
   private static final Logger LOG = LoggerFactory
@@ -61,7 +61,9 @@ public SignedRequest sign(SignRequest<? extends AwsCredentialsIdentity>
   }
 
   @Override
-  public CompletableFuture<AsyncSignedRequest> signAsync(final AsyncSignRequest<? extends AwsCredentialsIdentity> request) {
+  public CompletableFuture<AsyncSignedRequest> signAsync(
+      final AsyncSignRequest<? extends AwsCredentialsIdentity> request) {
+
     LOG.trace("Signing async request:{}", request.request());
     return delegateSigner.signAsync(request);
   }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index 1b6aa7f48f11e..e46fd88e85f89 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -134,15 +134,18 @@ public static Signer createSigner(String signerType, String configKey) throws IO
    * @param signer signer
    * @return the auth scheme
    */
-  public static AuthScheme<AwsCredentialsIdentity> createAuthScheme(String schemeId,
+  public static AuthScheme<AwsCredentialsIdentity> createAuthScheme(
+      String schemeId,
       HttpSigner<AwsCredentialsIdentity> signer) {
+
     return new AuthScheme<AwsCredentialsIdentity>() {
       @Override
       public String schemeId() {
         return schemeId;
       }
       @Override
-      public IdentityProvider<AwsCredentialsIdentity> identityProvider(IdentityProviders providers) {
+      public IdentityProvider<AwsCredentialsIdentity> identityProvider(
+          IdentityProviders providers) {
         return providers.identityProvider(AwsCredentialsIdentity.class);
       }
       @Override
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
index eab3bb52a3451..1148f6fcd4831 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
@@ -288,7 +288,7 @@ private InternalConstants() {
           ENABLE_MULTI_DELETE));
 
   /**
-   * AWS V4 Auth Scheme to use when creating signers.
+   * AWS V4 Auth Scheme to use when creating signers: {@value}.
    */
   public static final String AUTH_SCHEME_AWS_SIGV_4 = "aws.auth#sigv4";
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
index b41e6826439b1..4afddee4c7fbe 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.hadoop.fs.s3a.auth;
 
 import java.io.IOException;
@@ -77,8 +95,7 @@ private Configuration createTestConfig(String identifier) {
 
     removeBaseAndBucketOverrides(conf,
         CUSTOM_SIGNERS,
-        SIGNING_ALGORITHM_S3,
-        ENABLE_MULTI_DELETE);
+        SIGNING_ALGORITHM_S3);
 
     conf.setBoolean(HTTP_SIGNER_ENABLED, true);
     conf.set(HTTP_SIGNER_CLASS_NAME, CustomHttpSigner.class.getName());

From c6542d85c03519dfd9ab4ab1e05baa782f9abfa3 Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Fri, 8 Dec 2023 15:32:16 +0000
Subject: [PATCH 3/5] HADOOP-19004. httpsigner

* Remove unused method in s3afs
* put static imports in DefaultS3ClientFactory in right order

Change-Id: Ib5125d5e1a373835bac5f90efed42d0947cd1ba5
---
 .../java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java | 4 ++--
 .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 ----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 5372261d55e71..05ac5ef921c95 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -57,11 +57,11 @@
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_CLASS_NAME;
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED;
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED_DEFAULT;
-import static org.apache.hadoop.fs.s3a.auth.SignerFactory.createHttpSigner;
-import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
 import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS;
 import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
+import static org.apache.hadoop.fs.s3a.auth.SignerFactory.createHttpSigner;
+import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AUTH_SCHEME_AWS_SIGV_4;
 
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 4777b221b26c2..8c8d84700b368 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -5760,10 +5760,6 @@ public StoreContext createStoreContext() {
         .build();
   }
 
-  public CreateSessionResponse createSessionInternal(CreateSessionRequest createSessionRequest){
-    return this.s3Client.createSession(createSessionRequest);
-  }
-
   /**
    * Create a marker tools operations binding for this store.
    * Auditing:

From e4b37583338e4d7600e4ab9b239b9c28fa9e6d98 Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Thu, 4 Jan 2024 12:12:31 +0000
Subject: [PATCH 4/5] HADOOP-19004: remove unused imports

Change-Id: I5d590b110d8667371eff449154c602e6fd420728
---
 .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 8c8d84700b368..df7d3f1fb6891 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -58,8 +58,6 @@
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
-import software.amazon.awssdk.services.s3.model.CreateSessionRequest;
-import software.amazon.awssdk.services.s3.model.CreateSessionResponse;
 import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;

From 6ccac9c5a30aa321a2ab891a996bd7e47742fe06 Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Thu, 11 Jan 2024 14:13:13 +0000
Subject: [PATCH 5/5] HADOOP-19004. checkstyle

Change-Id: Ifcb9274dcb0124f5dc2f9a5acf6cb0c7b5280bfe
---
 .../test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
index 4afddee4c7fbe..db0aaa6be0eca 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestHttpSigner.java
@@ -36,7 +36,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 
 import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS;
-import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE;
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_CLASS_NAME;
 import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_S3;
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED;