HADOOP-17458. S3A to treat "SdkClientException: Data read has a different length than the expected" as EOFException (#3040)

Some network exceptions can raise SdkClientException with message
`Data read has a different length than the expected`.

These should be recoverable.

Contributed by Bogdan Stolojan

Author: bogthe

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java

@@ -132,9 +132,12 @@ public final class S3AUtils {
       S3AEncryptionMethods.SSE_S3.getMethod()
           + " is enabled but an encryption key was set in "
           + SERVER_SIDE_ENCRYPTION_KEY;
-  private static final String EOF_MESSAGE_IN_XML_PARSER
+  public static final String EOF_MESSAGE_IN_XML_PARSER
       = "Failed to sanitize XML document destined for handler class";
 
+  public static final String EOF_READ_DIFFERENT_LENGTH
+      = "Data read has a different length than the expected";
+
   private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s";
 
   /**
@@ -194,7 +197,7 @@ public static IOException translateException(@Nullable String operation,
         // interrupted IO, or a socket exception underneath that class
         return translateInterruptedException(exception, innerCause, message);
       }
-      if (signifiesConnectionBroken(exception)) {
+      if (isMessageTranslatableToEOF(exception)) {
         // call considered an sign of connectivity failure
         return (EOFException)new EOFException(message).initCause(exception);
       }
@@ -415,13 +418,14 @@ public static boolean isThrottleException(Exception ex) {
 
   /**
    * Cue that an AWS exception is likely to be an EOF Exception based
-   * on the message coming back from an XML/JSON parser. This is likely
-   * to be brittle, so only a hint.
+   * on the message coming back from the client. This is likely to be
+   * brittle, so only a hint.
    * @param ex exception
    * @return true if this is believed to be a sign the connection was broken.
    */
-  public static boolean signifiesConnectionBroken(SdkBaseException ex) {
-    return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER);
+  public static boolean isMessageTranslatableToEOF(SdkBaseException ex) {
+    return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER) ||
+            ex.toString().contains(EOF_READ_DIFFERENT_LENGTH);
   }
 
   /**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.s3a;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.net.SocketTimeoutException;
@@ -28,6 +29,7 @@
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.SdkBaseException;
+import com.amazonaws.SdkClientException;
 import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import org.junit.Assert;
@@ -163,6 +165,40 @@ public void test500isStatus500Exception() throws Exception {
         ex);
   }
 
+  @Test
+  public void testExceptionsWithTranslatableMessage() throws Exception {
+    SdkBaseException xmlParsing = new SdkBaseException(EOF_MESSAGE_IN_XML_PARSER);
+    SdkBaseException differentLength = new SdkBaseException(EOF_READ_DIFFERENT_LENGTH);
+
+    verifyTranslated(EOFException.class, xmlParsing);
+    verifyTranslated(EOFException.class, differentLength);
+  }
+
+
+  @Test
+  public void testSdkDifferentLengthExceptionIsTranslatable() throws Throwable {
+    final AtomicInteger counter = new AtomicInteger(0);
+    invoker.retry("test", null, false, () -> {
+      if (counter.incrementAndGet() < ACTIVE_RETRY_LIMIT) {
+        throw new SdkClientException(EOF_READ_DIFFERENT_LENGTH);
+      }
+    });
+
+    assertEquals(ACTIVE_RETRY_LIMIT, counter.get());
+  }
+
+  @Test
+  public void testSdkXmlParsingExceptionIsTranslatable() throws Throwable {
+    final AtomicInteger counter = new AtomicInteger(0);
+    invoker.retry("test", null, false, () -> {
+      if (counter.incrementAndGet() < ACTIVE_RETRY_LIMIT) {
+        throw new SdkClientException(EOF_MESSAGE_IN_XML_PARSER);
+      }
+    });
+
+    assertEquals(ACTIVE_RETRY_LIMIT, counter.get());
+  }
+
   @Test(expected = org.apache.hadoop.net.ConnectTimeoutException.class)
   public void testExtractConnectTimeoutException() throws Throwable {
     throw extractException("", "",