HADOOP-18820. Cut AWS v1 support (apache#5872)

This removes the AWS V1 SDK as a hadoop-aws runtime dependency.

It is still used at compile time so as to build a wrapper class
V1ToV2AwsCredentialProviderAdapter which allows v1 credential provider
to be used for authentication.
All well known credential providers have their classname remapped from
v1 to v2 classes prior to instantiation; this wrapper is not needed
for them.

There is no support for migrating other SDK plugin points
(signing, handlers)

Access to the v2 S3Client class used by an S3A FileSystem
instance is now via a new interface org.apache.hadoop.fs.s3a.S3AInternals;
other low-level operations (getObjectMetadata(Path)) have moved.

Contributed by Steve Loughran

Change-Id: I231fade61b20a206a018d18f5e867f5d3a1fa879

Author: steveloughran

LICENSE-binary

@@ -214,7 +214,6 @@ com.aliyun:aliyun-java-sdk-kms:2.11.0
 com.aliyun:aliyun-java-sdk-ram:3.1.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.0
-com.amazonaws:aws-java-sdk-bundle:1.12.367
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
 com.fasterxml.jackson.core:jackson-annotations:2.12.7
@@ -368,6 +367,8 @@ org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.1
 org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
+software.amazon.awssdk:bundle:jar:2.19.12
+software.amazon.awssdk.crt:aws-crt:0.21.0
 
 
 --------------------------------------------------------------------------------

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1201,61 +1201,31 @@
   <description>AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
 </property>
 
+<property>
+  <name>fs.s3a.session.token</name>
+  <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+    as one of the providers.
+  </description>
+</property>
+
 <property>
   <name>fs.s3a.aws.credentials.provider</name>
   <value>
     org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
     org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
-    com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
+    software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,
     org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
   </value>
   <description>
     Comma-separated class names of credential provider classes which implement
-    com.amazonaws.auth.AWSCredentialsProvider.
+    software.amazon.awssdk.auth.credentials.AwsCredentialsProvider.
 
     When S3A delegation tokens are not enabled, this list will be used
     to directly authenticate with S3 and other AWS services.
     When S3A Delegation tokens are enabled, depending upon the delegation
     token binding it may be used
     to communicate wih the STS endpoint to request session/role
     credentials.
-
-    These are loaded and queried in sequence for a valid set of credentials.
-    Each listed class must implement one of the following means of
-    construction, which are attempted in order:
-    * a public constructor accepting java.net.URI and
-        org.apache.hadoop.conf.Configuration,
-    * a public constructor accepting org.apache.hadoop.conf.Configuration,
-    * a public static method named getInstance that accepts no
-       arguments and returns an instance of
-       com.amazonaws.auth.AWSCredentialsProvider, or
-    * a public default constructor.
-
-    Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
-    anonymous access to a publicly accessible S3 bucket without any credentials.
-    Please note that allowing anonymous access to an S3 bucket compromises
-    security and therefore is unsuitable for most use cases. It can be useful
-    for accessing public data sets without requiring AWS credentials.
-
-    If unspecified, then the default list of credential provider classes,
-    queried in sequence, is:
-    * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks
-       for session login secrets in the Hadoop configuration.
-    * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
-       Uses the values of fs.s3a.access.key and fs.s3a.secret.key.
-    * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
-        configuration of AWS access key ID and secret access key in
-        environment variables named AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
-        and AWS_SESSION_TOKEN as documented in the AWS SDK.
-    * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
-       IAM credentials of any EC2 VM or AWS container in which the process is running.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.session.token</name>
-  <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
-    as one of the providers.
   </description>
 </property>
 
@@ -1353,10 +1323,10 @@
     Note: for job submission to actually collect these tokens,
     Kerberos must be enabled.
 
-    Options are:
+    Bindings available in hadoop-aws are:
     org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding
     org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding
-    and org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
+    org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
   </description>
 </property>
 

hadoop-project/pom.xml

@@ -189,6 +189,7 @@
     <aws-java-sdk.version>1.12.367</aws-java-sdk.version>
     <hsqldb.version>2.7.1</hsqldb.version>
     <aws-java-sdk-v2.version>2.19.12</aws-java-sdk-v2.version>
+    <aws.evenstream.version>1.0.1</aws.evenstream.version>
     <awscrt.version>0.21.0</awscrt.version>
     <frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
     <jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
@@ -1111,18 +1112,29 @@
         <groupId>com.amazonaws</groupId>
         <artifactId>aws-java-sdk-core</artifactId>
         <version>${aws-java-sdk.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>*</groupId>
+            <artifactId>*</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>software.amazon.awssdk</groupId>
         <artifactId>bundle</artifactId>
         <version>${aws-java-sdk-v2.version}</version>
         <exclusions>
           <exclusion>
-            <groupId>io.netty</groupId>
+            <groupId>*</groupId>
             <artifactId>*</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>software.amazon.eventstream</groupId>
+        <artifactId>eventstream</artifactId>
+        <version>${aws.evenstream.version}</version>
+      </dependency>
       <dependency>
         <groupId>software.amazon.awssdk.crt</groupId>
         <artifactId>aws-crt</artifactId>

hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml

@@ -64,6 +64,11 @@
     <Field name="futurePool"/>
     <Bug pattern="IS2_INCONSISTENT_SYNC"/>
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.fs.s3a.S3AFileSystem"/>
+    <Field name="s3AsyncClient"/>
+    <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+  </Match>
   <Match>
     <Class name="org.apache.hadoop.fs.s3a.s3guard.S3GuardTool$BucketInfo"/>
     <Method name="run"/>

hadoop-tools/hadoop-aws/pom.xml

@@ -463,6 +463,16 @@
                     <bannedImport>org.apache.hadoop.mapred.**</bannedImport>
                   </bannedImports>
                 </restrictImports>
+                <restrictImports>
+                  <includeTestCode>false</includeTestCode>
+                  <reason>Restrict AWS v1 imports to adapter code</reason>
+                  <exclusions>
+                    <exclusion>org.apache.hadoop.fs.s3a.adapter.V1ToV2AwsCredentialProviderAdapter</exclusion>
+                  </exclusions>
+                  <bannedImports>
+                    <bannedImport>com.amazonaws.**</bannedImport>
+                  </bannedImports>
+                </restrictImports>
               </rules>
             </configuration>
           </execution>
@@ -483,10 +493,14 @@
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
+
+    <!-- The v1 SDK is used at compilation time for adapter classes in
+         org.apache.hadoop.fs.s3a.adapter. It is not needed at runtime
+         unless a non-standard v1 credential provider is declared. -->
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-core</artifactId>
-      <scope>compile</scope>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
@@ -496,7 +510,11 @@
     <dependency>
       <groupId>software.amazon.awssdk.crt</groupId>
       <artifactId>aws-crt</artifactId>
-      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.eventstream</groupId>
+      <artifactId>eventstream</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.assertj</groupId>

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java

@@ -27,27 +27,21 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.BasicSessionCredentials;
-import org.apache.hadoop.fs.s3a.adapter.V1V2AwsCredentialProviderAdapter;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
 import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Preconditions;
 
 import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
 import software.amazon.awssdk.auth.credentials.AwsCredentials;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
 import software.amazon.awssdk.core.exception.SdkException;
 
 /**
@@ -105,23 +99,8 @@ public AWSCredentialProviderList() {
    * @param providers provider list.
    */
   public AWSCredentialProviderList(
-      Collection<AWSCredentialsProvider> providers) {
-    for (AWSCredentialsProvider provider: providers) {
-      this.providers.add(V1V2AwsCredentialProviderAdapter.adapt(provider));
-    }
-  }
-
-  /**
-   * Create with an initial list of providers.
-   * @param name name for error messages, may be ""
-   * @param providerArgs provider list.
-   */
-  public AWSCredentialProviderList(final String name,
-      final AWSCredentialsProvider... providerArgs) {
-    setName(name);
-    for (AWSCredentialsProvider provider: providerArgs) {
-      this.providers.add(V1V2AwsCredentialProviderAdapter.adapt(provider));
-    }
+      Collection<AwsCredentialsProvider> providers) {
+    this.providers.addAll(providers);
   }
 
   /**
@@ -147,14 +126,6 @@ public void setName(final String name) {
     }
   }
 
-  /**
-   * Add a new provider.
-   * @param provider provider
-   */
-  public void add(AWSCredentialsProvider provider) {
-    providers.add(V1V2AwsCredentialProviderAdapter.adapt(provider));
-  }
-
   /**
    * Add a new SDK V2 provider.
    * @param provider provider
@@ -163,7 +134,6 @@ public void add(AwsCredentialsProvider provider) {
     providers.add(provider);
   }
 
-
   /**
    * Add all providers from another list to this one.
    * @param other the other list.
@@ -173,19 +143,11 @@ public void addAll(AWSCredentialProviderList other) {
   }
 
   /**
-   * This method will get credentials using SDK V2's resolveCredentials and then convert it into
-   * V1 credentials. This required by delegation token binding classes.
-   * @return SDK V1 credentials
+   * Was an implementation of the v1 refresh; now just
+   * a no-op.
    */
-  public AWSCredentials getCredentials() {
-    AwsCredentials credentials = resolveCredentials();
-    if (credentials instanceof AwsSessionCredentials) {
-      return new BasicSessionCredentials(credentials.accessKeyId(),
-          credentials.secretAccessKey(),
-          ((AwsSessionCredentials) credentials).sessionToken());
-    } else {
-      return new BasicAWSCredentials(credentials.accessKeyId(), credentials.secretAccessKey());
-    }
+  @Deprecated
+  public void refresh() {
   }
 
   /**
@@ -256,8 +218,7 @@ public AwsCredentials resolveCredentials() {
    *
    * @return providers
    */
-  @VisibleForTesting
-  List<AwsCredentialsProvider> getProviders() {
+  public List<AwsCredentialsProvider> getProviders() {
     return providers;
   }
 
@@ -289,9 +250,11 @@ public String listProviderNames() {
    */
   @Override
   public String toString() {
-    return "AWSCredentialProviderList[" +
-        name +
-        "refcount= " + refCount.get() + ": [" +
+    return "AWSCredentialProviderList"
+        + " name=" + name
+        + "; refcount= " + refCount.get()
+        + "; size="+ providers.size()
+        + ": [" +
         StringUtils.join(providers, ", ") + ']'
         + (lastProvider != null ? (" last provider: " + lastProvider) : "");
   }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -727,11 +727,19 @@ private Constants() {
   public static final String STREAM_READ_GAUGE_INPUT_POLICY =
       "stream_read_gauge_input_policy";
 
+  /**
+   * S3 Client Factory implementation class: {@value}.
+   * Unstable and incompatible between v1 and v2 SDK versions.
+   */
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
   public static final String S3_CLIENT_FACTORY_IMPL =
       "fs.s3a.s3.client.factory.impl";
 
+  /**
+   * Default factory:
+   * {@code org.apache.hadoop.fs.s3a.DefaultS3ClientFactory}.
+   */
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
   public static final Class<? extends S3ClientFactory>