HADOOP-18889. S3A v2 SDK error translations and troubleshooting docs

Error translation uses reflection to create IOE of the right type.
All IOEs at the bottom of an AWS stack chain are regenerated.

Troubleshooting doc updated with the new classnames;
cutting out stack traces rather than bother recreating them.

Pulled the multipart param stuff, which can be addressed in a separate JIRA.
Where I also think we should move listUploads() to S3AInternals
as it is very internal.
But did tweak ITestS3AMultipartUtils while debugging this...
 leaving that in as better assertions are always good.

+ checkstyles

Change-Id: Iba325cc98a95d11911fe41546f91a912a1d9aca5

Author: steveloughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java

@@ -61,7 +61,7 @@ private MultipartUtils() { }
    *
    * @param storeContext store context
    * @param s3 AmazonS3 client to use.
-   * @param prefix optional key prefix to narrow search.  If null or "/" then whole
+   * @param prefix optional key prefix to narrow search.  If null then whole
    *               bucket will be searched.
    * @param maxKeys maximum batch size to request at a time from S3.
    * @return an iterator of matching uploads
@@ -122,17 +122,9 @@ static class ListingIterator implements
       this.s3 = s3;
       this.requestFactory = storeContext.getRequestFactory();
       this.maxKeys = maxKeys;
+      this.prefix = prefix;
       this.invoker = storeContext.getInvoker();
       this.auditSpan = storeContext.getActiveAuditSpan();
-      String p;
-      if (prefix == null) {
-        p = "/";
-      } else if (!prefix.endsWith("/")) {
-        p = prefix + "/";
-      } else {
-        p = prefix;
-      }
-      this.prefix = p;
 
       // request the first listing.
       requestNextBatch();

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java

@@ -19,12 +19,12 @@
 package org.apache.hadoop.fs.s3a.impl;
 
 import java.io.IOException;
-import java.net.ConnectException;
-import java.net.NoRouteToHostException;
-import java.net.UnknownHostException;
+import java.lang.reflect.Constructor;
 
 import software.amazon.awssdk.awscore.exception.AwsServiceException;
 
+import org.apache.hadoop.fs.PathIOException;
+
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
 
 /**
@@ -40,7 +40,7 @@
  * The existing code las been left in S3AUtils it is to avoid cherry-picking
  * problems on backports.
  */
-public class ErrorTranslation {
+public final class ErrorTranslation {
 
   /**
    * Private constructor for utility class.
@@ -87,7 +87,7 @@ public static IOException maybeExtractNetworkException(String path, Throwable th
 
     // look inside
     Throwable cause = thrown.getCause();
-    while(cause != null && cause.getCause() != null) {
+    while (cause != null && cause.getCause() != null) {
       cause = cause.getCause();
     }
     if (!(cause instanceof IOException)) {
@@ -97,24 +97,41 @@ public static IOException maybeExtractNetworkException(String path, Throwable th
     // the cause can be extracted to an IOE.
     // rather than just return it, we try to preserve the stack trace
     // of the outer exception.
-    IOException ioe = (IOException) cause;
-    // now examine any IOE
-    if (ioe instanceof UnknownHostException) {
-      return (IOException) new UnknownHostException(thrown + ": " + ioe.getMessage())
-          .initCause(thrown);
-    }
-    if (ioe instanceof NoRouteToHostException) {
-      return (IOException) new NoRouteToHostException(thrown + ": " + ioe.getMessage())
-          .initCause(thrown);
-    }
-    if (ioe instanceof ConnectException) {
-      return (IOException) new ConnectException(thrown + ": " + ioe.getMessage())
-          .initCause(thrown);
-    }
+    // as a new instance is created through reflection, the
+    // class of the returned instance will be that of the innermost,
+    // unless no suitable constructor is available.
+    return wrapWithInnerIOE(path, thrown, (IOException) cause);
 
-    // currently not attempting to translate any other exception types.
-    return null;
+  }
 
+  /**
+   * Given an outer and an inner exception, create a new IOE
+   * of the inner type, with the outer exception as the cause.
+   * The message is derived from both.
+   * This only works if the inner exception has a constructor which
+   * takes a string; if not a PathIOException is created.
+   * <p>
+   * See {@code NetUtils}.
+   * @param <T> type of inner exception.
+   * @param path path of the failure.
+   * @param outer outermost exception.
+   * @param inner inner exception.
+   * @return the new exception.
+   */
+  @SuppressWarnings("unchecked")
+  private static <T extends IOException> IOException wrapWithInnerIOE(
+      String path,
+      Throwable outer,
+      T inner) {
+    String msg = outer.toString() + ": " + inner.getMessage();
+    Class<? extends Throwable> clazz = inner.getClass();
+    try {
+      Constructor<? extends Throwable> ctor = clazz.getConstructor(String.class);
+      Throwable t = ctor.newInstance(msg);
+      return (T) (t.initCause(outer));
+    } catch (Throwable e) {
+      return new PathIOException(path, msg, outer);
+    }
   }
 
   /**

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md

@@ -39,7 +39,7 @@ The features which may be unavailable include:
 * Optional Bucket Probes at startup (`fs.s3a.bucket.probe = 0`).
   This is now the default -do not change it.
 * List API to use (`fs.s3a.list.version = 1`)
-* 
+
 ## Configuring s3a to connect to a third party store
 
 
@@ -337,17 +337,16 @@ this makes renaming and deleting significantly slower.
 ```xml
 <configuration>
 
-  
   <property>
     <name>fs.s3a.bucket.gcs-container.access.key</name>
     <value>GOOG1EZ....</value>
   </property>
-
+  
   <property>
     <name>fs.s3a.bucket.gcs-container.secret.key</name>
     <value>SECRETS</value>
   </property>
-  
+
   <property>
     <name>fs.s3a.bucket.gcs-container.endpoint</name>
     <value>https://storage.googleapis.com</value>
@@ -372,18 +371,17 @@ this makes renaming and deleting significantly slower.
     <name>fs.s3a.bucket.gcs-container.select.enabled</name>
     <value>false</value>
   </property>
-  
+
   <property>
     <name>fs.s3a.bucket.gcs-container.path.style.access</name>
     <value>true</value>
   </property>
 
-  
   <property>
     <name>fs.s3a.bucket.gcs-container.endpoint.region</name>
     <value>dummy</value>
   </property>
-  
+
 </configuration>
 ```