S3GuardTool updates to correct ETag or versionId metadata

Author: Ben Roling

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java

@@ -724,7 +724,7 @@ private long importDir(FileStatus status) throws IOException {
       long items = 0;
 
       while (it.hasNext()) {
-        LocatedFileStatus located = it.next();
+        S3LocatedFileStatus located = it.next();
         S3AFileStatus child;
         if (located.isDirectory()) {
           child = DynamoDBMetadataStore.makeDirStatus(located.getPath(),
@@ -736,8 +736,8 @@ private long importDir(FileStatus status) throws IOException {
               located.getPath(),
               located.getBlockSize(),
               located.getOwner(),
-              null,
-              null);
+              located.getETag(),
+              located.getVersionId());
         }
         putParentsIfNotPresent(child);
         getStore().put(new PathMetadata(child));

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md

@@ -1020,6 +1020,14 @@ If this happens, reads of the affected file(s) will result in
 * the file is overwritten (causing an S3Guard metadata update)
 * the S3Guard metadata is pruned
 
+The S3Guard metadata for a file can be corrected with the `s3guard import`
+command as discussed above.  The command can take a file URI instead of a
+bucket URI to correct the metdata for a single file.  For example:
+
+```bash
+hadoop s3guard import [-meta URI] s3a://my-bucket/file-with-bad-metadata
+```
+
 ## Troubleshooting
 
 ### Error: `S3Guard table lacks version marker.`

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java

@@ -37,7 +37,9 @@
 
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.Tristate;
 
 import static org.apache.hadoop.fs.s3a.MultipartTestUtils.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVFile;
@@ -95,6 +97,41 @@ public void testImportCommand() throws Exception {
     // assertTrue(children.isAuthoritative());
   }
 
+  @Test
+  public void testImportCommandRepairsETagAndVersionId() throws Exception {
+    S3AFileSystem fs = getFileSystem();
+    MetadataStore ms = getMetadataStore();
+    Path path = path("test-version-metadata");
+    try (FSDataOutputStream out = fs.create(path)) {
+      out.write(1);
+    }
+    S3AFileStatus originalStatus = (S3AFileStatus) fs.getFileStatus(path);
+
+    // put in bogus ETag and versionId
+    S3AFileStatus bogusStatus = S3AFileStatus.fromFileStatus(originalStatus,
+        Tristate.FALSE, "bogusETag", "bogusVersionId");
+    ms.put(new PathMetadata(bogusStatus));
+
+    // sanity check that bogus status is actually persisted
+    S3AFileStatus retrievedBogusStatus = (S3AFileStatus) fs.getFileStatus(path);
+    assertEquals("bogus ETag was not persisted",
+        "bogusETag", retrievedBogusStatus.getETag());
+    assertEquals("bogus versionId was not persisted",
+        "bogusVersionId", retrievedBogusStatus.getVersionId());
+
+    // execute the import
+    S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
+    cmd.setStore(ms);
+    exec(cmd, "import", path.toString());
+
+    // make sure ETag and versionId were corrected
+    S3AFileStatus updatedStatus = (S3AFileStatus) fs.getFileStatus(path);
+    assertEquals("ETag was not corrected",
+        originalStatus.getETag(), updatedStatus.getETag());
+    assertEquals("VersionId was not corrected",
+        originalStatus.getVersionId(), updatedStatus.getVersionId());
+  }
+
   @Test
   public void testDestroyBucketExistsButNoTable() throws Throwable {
     run(Destroy.NAME,