HADOOP-18377. hadoop-aws build to add a -prefetch profile to run all tests with prefetching (#4914)

Contributed by Viraj Jasani

Author: virajjasani

hadoop-tools/hadoop-aws/pom.xml

@@ -52,6 +52,9 @@
     <fs.s3a.directory.marker.audit>false</fs.s3a.directory.marker.audit>
     <!--    marker retention policy -->
     <fs.s3a.directory.marker.retention></fs.s3a.directory.marker.retention>
+
+    <!-- Is prefetch enabled? -->
+    <fs.s3a.prefetch.enabled>unset</fs.s3a.prefetch.enabled>
   </properties>
 
   <profiles>
@@ -126,6 +129,8 @@
                 <!-- Markers-->
                 <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
                 <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+                <!-- Prefetch -->
+                <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
               </systemPropertyVariables>
             </configuration>
           </plugin>
@@ -165,6 +170,8 @@
                     <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
 
                     <test.default.timeout>${test.integration.timeout}</test.default.timeout>
+                    <!-- Prefetch -->
+                    <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
                   </systemPropertyVariables>
                   <!-- Some tests cannot run in parallel.  Tests that cover -->
                   <!-- access to the root directory must run in isolation -->
@@ -214,6 +221,8 @@
                     <!-- Markers-->
                     <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
                     <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+                    <!-- Prefetch -->
+                    <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
                   </systemPropertyVariables>
                   <!-- Do a sequential run for tests that cannot handle -->
                   <!-- parallel execution. -->
@@ -270,6 +279,8 @@
                     <!-- Markers-->
                     <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
                     <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+                    <!-- Prefetch -->
+                    <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
                   </systemPropertyVariables>
                   <forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
                 </configuration>
@@ -331,6 +342,19 @@
       </properties>
     </profile>
 
+    <!-- Turn on prefetching-->
+    <profile>
+      <id>prefetch</id>
+      <activation>
+        <property>
+          <name>prefetch</name>
+        </property>
+      </activation>
+      <properties>
+        <fs.s3a.prefetch.enabled>true</fs.s3a.prefetch.enabled>
+      </properties>
+    </profile>
+
   </profiles>
 
   <build>

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -617,6 +617,20 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with
 a meaningful error ("S3 Select not supported") rather than a generic Bad Request
 exception.
 
+### <a name="enabling-prefetch"></a> Enabling prefetch for all tests
+
+The tests are run with prefetch if the `prefetch` property is set in the
+maven build. This can be combined with the scale tests as well.
+
+```bash
+mvn verify -Dprefetch
+
+mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8
+
+mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8
+```
+
+
 ### Testing Requester Pays
 
 By default, the requester pays tests will look for a bucket that exists on Amazon S3

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java

@@ -575,6 +575,10 @@ public static Configuration prepareTestConfiguration(final Configuration conf) {
         DEFAULT_DIRECTORY_MARKER_POLICY);
     conf.set(DIRECTORY_MARKER_POLICY, directoryRetention);
 
+    boolean prefetchEnabled =
+        getTestPropertyBool(conf, PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT);
+    conf.setBoolean(PREFETCH_ENABLED_KEY, prefetchEnabled);
+
     return conf;
   }