Add method for large test object (landsat) to PublicDatasetTestUtils

Author: dannycjones

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/PublicDatasetTestUtils.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_BUCKET_WITH_MANY_OBJECTS;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_CSVTEST_FILE;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_REQUESTER_PAYS_FILE;
 
 /**
@@ -56,6 +57,12 @@ public class PublicDatasetTestUtils {
   private static final String DEFAULT_BUCKET_WITH_MANY_OBJECTS
       = "s3a://usgs-landsat/collection02/level-1/";
 
+  /**
+   * Default path for the multi MB test file: {@value}.
+   */
+  private static final String DEFAULT_MB_OBJECT
+      = "s3a://landsat-pds/scene_list.gz";
+
   /**
    * Provide a URI for a directory containing many objects.
    *
@@ -70,6 +77,19 @@ public static String getBucketPrefixWithManyObjects(Configuration conf) {
         KEY_BUCKET_WITH_MANY_OBJECTS, DEFAULT_BUCKET_WITH_MANY_OBJECTS);
   }
 
+  /**
+   * Provide a URI to a large (multi MB) object.
+   *
+   * Unless otherwise configured,
+   * this will be {@value DEFAULT_MB_OBJECT}.
+   *
+   * @param conf Hadoop configuration
+   * @return S3A FS URI
+   */
+  public static String getLargeTextObject(Configuration conf) {
+    return fetchFromConfig(conf, KEY_CSVTEST_FILE, DEFAULT_MB_OBJECT);
+  }
+
   /**
    * Provide a URI to an object within a requester pays enabled bucket.
    *

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java

@@ -89,15 +89,14 @@ public interface S3ATestConstants {
    */
   String KEY_CSVTEST_FILE = S3A_SCALE_TEST + "csvfile";
 
-  /**
-   * The landsat bucket: {@value}.
-   */
-  String LANDSAT_BUCKET = "s3a://landsat-pds/";
-
   /**
    * Default path for the multi MB test file: {@value}.
+   *
+   * @deprecated Please use or add an appropriate method
+   *             in {@link PublicDatasetTestUtils}.
    */
-  String DEFAULT_CSVTEST_FILE = LANDSAT_BUCKET + "scene_list.gz";
+  @Deprecated
+  String DEFAULT_CSVTEST_FILE = "s3a://landsat-pds/scene_list.gz";
 
   /**
    * Configuration key for an existing object in a requester pays bucket: {@value}.