HADOOP-17628. Distcp contract test is really slow with ABFS and S3A; timing out. (#3240)

This patch cuts down the size of directory trees used for
distcp contract tests against object stores, so making
them much faster against distant/slow stores.

On abfs, the test only runs with -Dscale (as was the case for s3a already),
and has the larger scale test timeout.

After every test case, the FileSystem IOStatistics are logged,
to provide information about what IO is taking place and
what it's performance is.

There are some test cases which upload files of 1+ MiB; you can
increase the size of the upload in the option
"scale.test.distcp.file.size.kb" 
Set it to zero and the large file tests are skipped.

Contributed by Steve Loughran.

Author: steveloughran

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java

@@ -948,8 +948,8 @@ public static List<Path> createFiles(final FileSystem fs,
       final int fileCount,
       final int dirCount) throws IOException {
     return createDirsAndFiles(fs, destDir, depth, fileCount, dirCount,
-        new ArrayList<Path>(fileCount),
-        new ArrayList<Path>(dirCount));
+        new ArrayList<>(fileCount),
+        new ArrayList<>(dirCount));
   }
 
   /**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java

@@ -18,17 +18,12 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT_MILLIS;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageStatistics;
-import org.apache.hadoop.fs.s3a.FailureInjectionPolicy;
 import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
 
 /**
@@ -60,41 +55,28 @@ protected Configuration createConfiguration() {
   }
 
   @Override
-  protected S3AContract createContract(Configuration conf) {
-    return new S3AContract(conf);
+  protected boolean shouldUseDirectWrite() {
+    return true;
   }
 
-  /**
-   * Always inject the delay path in, so if the destination is inconsistent,
-   * and uses this key, inconsistency triggered.
-   * @param filepath path string in
-   * @return path on the remote FS for distcp
-   * @throws IOException IO failure
-   */
   @Override
-  protected Path path(final String filepath) throws IOException {
-    Path path = super.path(filepath);
-    return new Path(path, FailureInjectionPolicy.DEFAULT_DELAY_KEY_SUBSTRING);
+  protected S3AContract createContract(Configuration conf) {
+    return new S3AContract(conf);
   }
 
   @Override
-  public void testDirectWrite() throws Exception {
+  public void testDistCpWithIterator() throws Exception {
     final long renames = getRenameOperationCount();
-    super.testDirectWrite();
-    assertEquals("Expected no renames for a direct write distcp", 0L,
-        getRenameOperationCount() - renames);
+    super.testDistCpWithIterator();
+    assertEquals("Expected no renames for a direct write distcp",
+        getRenameOperationCount(),
+         renames);
   }
 
   @Override
   public void testNonDirectWrite() throws Exception {
     final long renames = getRenameOperationCount();
-    try {
-      super.testNonDirectWrite();
-    } catch (FileNotFoundException e) {
-      // We may get this exception when data is written to a DELAY_LISTING_ME
-      // directory causing verification of the distcp success to fail if
-      // S3Guard is not enabled
-    }
+    super.testNonDirectWrite();
     assertEquals("Expected 2 renames for a non-direct write distcp", 2L,
         getRenameOperationCount() - renames);
   }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java

@@ -19,16 +19,24 @@
 package org.apache.hadoop.fs.azurebfs.contract;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
 import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
 import org.junit.Assume;
 
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.assumeScaleTestsEnabled;
+
 /**
  * Contract test for distCp operation.
  */
 public class ITestAbfsFileSystemContractDistCp extends AbstractContractDistCpTest {
   private final ABFSContractTestBinding binding;
 
+  @Override
+  protected int getTestTimeoutMillis() {
+    return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS;
+  }
+
   public ITestAbfsFileSystemContractDistCp() throws Exception {
     binding = new ABFSContractTestBinding();
     Assume.assumeTrue(binding.getAuthType() != AuthType.OAuth);
@@ -38,6 +46,7 @@ public ITestAbfsFileSystemContractDistCp() throws Exception {
   public void setup() throws Exception {
     binding.setup();
     super.setup();
+    assumeScaleTestsEnabled(binding.getRawConfiguration());
   }
 
   @Override