HDFS-15463. Support validation over a fsimag directory.

Author: szetszwo

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java

@@ -47,7 +47,7 @@
  * {@link org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer}.
  */
 public class FsImageValidation {
-  static final String FS_IMAGE_FILE = "FS_IMAGE_FILE";
+  static final String FS_IMAGE = "FS_IMAGE";
 
   static HdfsConfiguration newHdfsConfiguration() {
     final HdfsConfiguration conf = new HdfsConfiguration();
@@ -61,19 +61,19 @@ static FsImageValidation newInstance(String... args) {
     final String f = Cli.parse(args);
     if (f == null) {
       throw new HadoopIllegalArgumentException(
-          FS_IMAGE_FILE + " is not specified.");
+          FS_IMAGE + " is not specified.");
     }
     return new FsImageValidation(new File(f));
   }
 
-  private final File fsImageFile;
+  private final File fsImage;
 
-  FsImageValidation(File fsImageFile) {
-    this.fsImageFile = fsImageFile;
+  FsImageValidation(File fsImage) {
+    this.fsImage = fsImage;
   }
 
   int checkINodeReference() throws Exception {
-    Cli.println("Check INodeReference for %d: %d", FS_IMAGE_FILE, fsImageFile);
+    Cli.println("Check INodeReference for %s", fsImage);
 
     final TimerTask checkProgress = new TimerTask() {
       @Override
@@ -83,29 +83,39 @@ public void run() {
         Cli.println("%s Progress: %.1f%%", Phase.LOADING_FSIMAGE, 100*percent);
       }
     };
-    final Timer t = new Timer();
-    t.scheduleAtFixedRate(checkProgress, 0, 60_000);
 
     final HdfsConfiguration conf = newHdfsConfiguration();
-    final FSImage fsImage = new FSImage(conf);
-    final FSNamesystem namesystem = new FSNamesystem(conf, fsImage, false);
 
-    final NamespaceInfo namespaceInfo = NNStorage.newNamespaceInfo();
-    namespaceInfo.clusterID = "cluster0";
-    fsImage.getStorage().setStorageInfo(namespaceInfo);
-
-    final FSImageFormat.LoaderDelegator loader
-        = FSImageFormat.newLoader(conf, namesystem);
     INodeReferenceValidation.start();
-    namesystem.writeLock();
-    namesystem.getFSDirectory().writeLock();
-    try {
-      loader.load(fsImageFile, false);
-    } finally {
-      namesystem.getFSDirectory().writeUnlock();
-      namesystem.writeUnlock();
-      t.cancel();
+    final Timer t = new Timer();
+    t.scheduleAtFixedRate(checkProgress, 0, 60_000);
+    if (fsImage.isDirectory()) {
+      Cli.println("Loading %s as a directory.", fsImage);
+      final String dir = fsImage.getCanonicalPath();
+      conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, dir);
+      conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, dir);
+      FSNamesystem.loadFromDisk(conf);
+    } else {
+      Cli.println("Loading %s as a file.", fsImage);
+      final FSImage fsImage = new FSImage(conf);
+      final FSNamesystem namesystem = new FSNamesystem(conf, fsImage, false);
+
+      final NamespaceInfo namespaceInfo = NNStorage.newNamespaceInfo();
+      namespaceInfo.clusterID = "cluster0";
+      fsImage.getStorage().setStorageInfo(namespaceInfo);
+
+      final FSImageFormat.LoaderDelegator loader
+          = FSImageFormat.newLoader(conf, namesystem);
+      namesystem.writeLock();
+      namesystem.getFSDirectory().writeLock();
+      try {
+        loader.load(this.fsImage, false);
+      } finally {
+        namesystem.getFSDirectory().writeUnlock();
+        namesystem.writeUnlock();
+      }
     }
+    t.cancel();
     return INodeReferenceValidation.end();
   }
 
@@ -115,10 +125,9 @@ static class Cli extends Configured implements Tool {
     static {
       final String clazz = FsImageValidation.class.getSimpleName();
       COMMAND = Character.toLowerCase(clazz.charAt(0)) + clazz.substring(1);
-      USAGE = "Usage: hdfs " + COMMAND + " <" + FS_IMAGE_FILE + ">";
+      USAGE = "Usage: hdfs " + COMMAND + " <" + FS_IMAGE + ">";
     }
 
-
     @Override
     public int run(String[] args) throws Exception {
       final FsImageValidation validation = FsImageValidation.newInstance(args);
@@ -130,9 +139,9 @@ public int run(String[] args) throws Exception {
     static String parse(String... args) {
       final String f;
       if (args == null || args.length == 0) {
-        f = System.getenv().get(FS_IMAGE_FILE);
+        f = System.getenv().get(FS_IMAGE);
         if (f != null) {
-          println("Environment variable %s = %s", FS_IMAGE_FILE, f);
+          println("Environment variable %s = %s", FS_IMAGE, f);
         }
       } else if (args.length == 1) {
         f = args[0];
@@ -141,7 +150,7 @@ static String parse(String... args) {
             "args = " + Arrays.toString(args));
       }
 
-      println("%s = %s", FS_IMAGE_FILE, f);
+      println("%s = %s", FS_IMAGE, f);
       return f;
     }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsImageValidation.java

@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.event.Level;
 
@@ -37,10 +38,11 @@ public class TestFsImageValidation {
   public void testINodeReference() throws Exception {
     try {
       final FsImageValidation validation = FsImageValidation.newInstance();
-      validation.checkINodeReference();
+      final int errorCount = validation.checkINodeReference();
+      Assert.assertEquals("Error Count: " + errorCount, 0, errorCount);
     } catch (HadoopIllegalArgumentException e) {
       FsImageValidation.Cli.printError("The environment variable "
-          + FsImageValidation.FS_IMAGE_FILE + " is not set.", e);
+          + FsImageValidation.FS_IMAGE + " is not set.", e);
     }
   }
 }