report dirty metadata in meta table

hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java

@@ -40,7 +40,9 @@
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -92,6 +94,36 @@ public Map<TableName,List<Path>> reportTablesMissingRegions(final List<String> n
     return extraChecker.reportTablesRegions(namespacesOrTables, this::findExtraRegionsInMETA);
   }
 
+  public Map<String, List<byte[]>> reportUndeletedRegions() throws IOException {
+    return HBCKMetaTableAccessor.getUndeletedRegions(this.conn);
+  }
+
+  public Map<String, Integer>
+      removeUndeletedRegion(Map<String, List<byte[]>> reportMap) throws IOException {
+    Table table = conn.getTable(TableName.META_TABLE_NAME);
+    Map<String, Integer> map = new HashMap<>();
+    List<Delete> list = new ArrayList<>();
+    for (Map.Entry<String, List<byte[]>> entry : reportMap.entrySet()) {
+      entry.getValue().forEach(e -> list.add(new Delete(e)));
+      map.put(entry.getKey(), new Integer(list.size()));
+      table.delete(list);
+      list.clear();
+    }
+    table.close();
+    return map;
+  }
+
+  public void deleteRegions(TableName tableName,List<byte[]> rows) throws IOException {
+    Table table = conn.getTable(tableName);
+    List<Delete> list=new ArrayList<>();
+    for (byte[] bytes : rows) {
+      Delete delete=new Delete(bytes);
+      list.add(delete);
+    }
+    table.delete(list);
+    table.close();
+  }
+
   List<Path> findMissingRegionsInMETA(String table) throws IOException {
     InternalMetaChecker<Path> missingChecker = new InternalMetaChecker<>();
     return missingChecker.checkRegionsInMETA(table, (regions, dirs) -> {

hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java

@@ -61,6 +61,7 @@
 import org.apache.hadoop.hbase.filter.RowFilter;
 import org.apache.hadoop.hbase.filter.SubstringComparator;
 import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.util.Bytes;
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
@@ -101,6 +102,8 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
   private static final String SCHEDULE_RECOVERIES = "scheduleRecoveries";
   private static final String RECOVER_UNKNOWN = "recoverUnknown";
   private static final String GENERATE_TABLE_INFO = "generateMissingTableDescriptorFile";
+  private static final String REPORT_UNDELETED_REGIONS_IN_META =
+          "reportUndeletedRegionsInMeta";
   private static final String FIX_META = "fixMeta";
   // TODO update this map in case of the name of a method changes in Hbck interface
   //  in org.apache.hadoop.hbase.client package. Or a new command is added and the hbck command
@@ -273,6 +276,35 @@ Map<TableName, List<String>> extraRegionsInMeta(String[] args)
     return result;
   }
 
+  Map<TableName, List<String>> reportUndeletedRegionsInMeta(String[] args)
+    throws Exception {
+    Options options = new Options();
+    Option fixOption = Option.builder("f").longOpt("fix").build();
+    options.addOption(fixOption);
+    // Parse command-line.
+    CommandLineParser parser = new DefaultParser();
+    CommandLine commandLine;
+    commandLine = parser.parse(options, args, false);
+    boolean fix = commandLine.hasOption(fixOption.getOpt());
+    Map<TableName, List<String>> result = new HashMap<>();
+    try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer =
+      new FsRegionsMetaRecoverer(this.conf)) {
+      Map<String, List<byte[]>> reportMap =
+        fsRegionsMetaRecoverer.reportUndeletedRegions();
+      reportMap.forEach((key, value) ->
+        result.put(TableName.valueOf(key),
+          value.stream().map(Bytes::toString).collect(Collectors.toList())));
+      if (fix) {
+        Map<String, Integer> map =
+          fsRegionsMetaRecoverer.removeUndeletedRegion(reportMap);
+        System.out.println(formatRemovedRegionsMessage(map));
+      }
+    } catch (IOException e) {
+      throw e;
+    }
+    return result;
+  }
+
   private List<String> formatNameSpaceTableParam(String... nameSpaceOrTable) {
     return nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null;
   }
@@ -419,6 +451,8 @@ private static String getCommandUsage() {
     writer.println();
     usageBypass(writer);
     writer.println();
+    usageReportUndeletedRegionsInMeta(writer);
+    writer.println();
     usageExtraRegionsInMeta(writer);
     writer.println();
     usageFilesystem(writer);
@@ -567,6 +601,23 @@ private static void usageReplication(PrintWriter writer) {
     writer.println("   purge if '--fix'.");
   }
 
+  private static void usageReportUndeletedRegionsInMeta(PrintWriter writer){
+    writer.println(" " + REPORT_UNDELETED_REGIONS_IN_META + " [OPTIONS]");
+    writer.println("   Options:");
+    writer.println("    -f, --fix    fix meta by removing all undeleted regions found. ");
+    writer.println("   Reports regions present on hbase:meta, but related tables have been ");
+    writer.println("   deleted on hbase. Needs hbase:meta to be online. ");
+    writer.println("   An example triggering undeleted regions report ");
+    writer.println("     $ HBCK2 " + REPORT_UNDELETED_REGIONS_IN_META);
+    writer.println("   Returns list of undeleted regions for each not found table");
+    writer.println("   If master log continues to print 'TableNotFoundException', or master ");
+    writer.println("   ui report RITs for those table not found regions, or hbck.jsp web page ");
+    writer.println("   report regions but related tables not existing, remove these undeleted ");
+    writer.println("   regions with '--fix' option. ");
+    writer.println("   You should switch active master after remove undeleted regions, then ");
+    writer.println("   those abnormal regions info will disappear. ");
+  }
+
   private static void usageExtraRegionsInMeta(PrintWriter writer) {
     writer.println(" " + EXTRA_REGIONS_IN_META + " <NAMESPACE|"
       + "NAMESPACE:TABLENAME>...");
@@ -975,6 +1026,15 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc
           return EXIT_FAILURE;
         }
         break;
+      case REPORT_UNDELETED_REGIONS_IN_META:
+        try {
+          Map<TableName, List<String>> report =
+            reportUndeletedRegionsInMeta(purgeFirst(commands));
+          System.out.println(formatUndeletedRegionReport(report));
+        }catch (Exception e) {
+          return EXIT_FAILURE;
+        }
+        break;
 
       case GENERATE_TABLE_INFO:
         if(commands.length != 2) {
@@ -987,6 +1047,7 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc
         break;
 
       default:
+        System.out.println("REPORT_UNDELETED_REGIONS_IN_META2");
         showErrorMessage("Unsupported command: " + command);
         return EXIT_FAILURE;
     }
@@ -1008,6 +1069,11 @@ private String formatExtraRegionsReport(Map<TableName,List<String>> report) {
     return formatReportMessage(message, (HashMap)report, s -> s);
   }
 
+  private String formatUndeletedRegionReport(Map<TableName,List<String>> report) {
+    String message = "Regions in Meta but having no valid table, for each table:\n\t";
+    return formatReportMessage(message, (HashMap)report, s -> s);
+  }
+
   private String formatReportMessage(String reportMessage, Map<TableName, List<?>> report,
       Function resolver){
     final StringBuilder builder = new StringBuilder();
@@ -1067,6 +1133,16 @@ private String formatRemovedRegionsMessage(int totalRemoved,
     return finalText.toString();
   }
 
+  private String formatRemovedRegionsMessage(Map<String, Integer> map) {
+    final StringBuilder finalText = new StringBuilder();
+    for (Map.Entry<String, Integer> entry : map.entrySet()) {
+      finalText.append("\n\tRegions that had relate to the not found table '")
+        .append(entry.getKey()).append("' and got removed from Meta: ")
+        .append(entry.getValue()).append("\n");
+    }
+    return finalText.toString();
+  }
+
   private String buildHbck2AssignsCommand(List<String> regions) {
     final StringBuilder builder = new StringBuilder();
     builder.append("assigns ");

hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java

@@ -264,6 +264,40 @@ public static List<RegionInfo> getAllRegions(Connection conn) throws IOException
       });
   }
 
+
+  /**
+   * List all undeleted regions currently in META.
+   * Undeleted regions means table not exist on hbase, but regions still in META.
+   * @param conn a valid, open connection.
+   * @return a Map of all dirty metadata in META.
+   * @throws IOException on any issues related with scanning meta table
+   */
+  public static Map<String, List<byte[]>> getUndeletedRegions(Connection conn)
+          throws IOException {
+    final Map<String, List<byte[]>> undeletedTableRegions = new HashMap<>();
+    final Map<String, TableName> tableNameMap = new HashMap<>();
+    List<TableName> tables = getTables(conn);
+    tables.forEach(tableName -> tableNameMap.put(tableName.getNameAsString(), tableName));
+    Table metaTable = conn.getTable(TableName.META_TABLE_NAME);
+    Scan scan = new Scan();
+    ResultScanner resultScanner = metaTable.getScanner(scan);
+    for (Result result : resultScanner) {
+      byte[] rowBytes = result.getRow();
+      String row = Bytes.toString(rowBytes);
+      String tableName = row.split(",")[0];
+      if (!tableNameMap.containsKey(tableName)) {
+        if (undeletedTableRegions.containsKey(tableName)) {
+          undeletedTableRegions.get(tableName).add(rowBytes);
+        } else {
+          List<byte[]> list = new ArrayList<>();
+          list.add(rowBytes);
+          undeletedTableRegions.put(tableName, list);
+        }
+      }
+    }
+    return undeletedTableRegions;
+  }
+
   /**
    * Scans all "table:state" cell values existing in meta and returns as a map of
    * <code>TableName</code> as key and <code>TableState</code> as the value.

hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java

@@ -72,10 +72,14 @@ public class TestHBCK2 {
   private static final Logger LOG = LoggerFactory.getLogger(TestHBCK2.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final TableName TABLE_NAME = TableName.valueOf(TestHBCK2.class.getSimpleName());
+  private static final TableName DELETED_TABLE_NAME = TableName.
+    valueOf(TestHBCK2.class.getSimpleName() + "-DELETED");
   private static final TableName REGION_STATES_TABLE_NAME = TableName.
     valueOf(TestHBCK2.class.getSimpleName() + "-REGIONS_STATES");
   private final static String ASSIGNS = "assigns";
   private static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta";
+  private static final String REPORT_UNDELETED_REGIONS_IN_META =
+          "reportUndeletedRegionsInMeta";
 
   @Rule
   public TestName testName = new TestName();
@@ -324,6 +328,12 @@ private TableName createTestTable(int totalRegions) throws IOException {
     return tableName;
   }
 
+  private TableName createTestTable(int totalRegions, String name) throws IOException {
+    TableName tableName = TableName.valueOf(name);
+    TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("family1"), totalRegions);
+    return tableName;
+  }
+
   private void testAddMissingRegionsInMetaForTables(int missingRegions, int totalRegions)
     throws Exception {
     TableName tableName = createTestTable(totalRegions);
@@ -610,4 +620,69 @@ private void testReportExtraRegionsInMeta(int extraRegionsInTestTbl,
     assertEquals(expectedTotalExtraRegions, resultingExtraRegions);
   }
 
+  private void deleteTableFamilyRegion() {
+    try {
+      HBCKMetaTableAccessor.MetaScanner<byte[]> scanner =
+        new HBCKMetaTableAccessor.MetaScanner<>();
+      List<byte[]> rows = scanner.scanMeta(TEST_UTIL.getConnection(),
+        scan -> scan.addColumn(HConstants.TABLE_FAMILY,HConstants.TABLE_STATE_QUALIFIER),
+        r -> {
+          byte[] bytes = r.getRow();
+          String row=Bytes.toString(bytes);
+          String tableName = row.split(",")[0];
+          return tableName.equals(DELETED_TABLE_NAME.getNameAsString()) ? bytes : null;
+        });
+      try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer =
+        new FsRegionsMetaRecoverer(TEST_UTIL.getConfiguration())) {
+        fsRegionsMetaRecoverer.deleteRegions(TableName.META_TABLE_NAME, rows);
+      }
+    } catch (IOException e) {
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testReportUndeletedRegionsInMeta() throws Exception {
+    testReportUndeletedRegionsInMeta(5,5);
+  }
+
+  private void testReportUndeletedRegionsInMeta(int undeletedRegions,
+    int expectedUndeletedRegions, String... namespaceOrTable) throws Exception {
+    createTestTable(5, DELETED_TABLE_NAME.getNameAsString());
+    List<RegionInfo> regions = HBCKMetaTableAccessor
+      .getTableRegions(TEST_UTIL.getConnection(), DELETED_TABLE_NAME);
+    regions.subList(0, undeletedRegions).forEach(r -> {
+      deleteRegionDir(DELETED_TABLE_NAME, r.getEncodedName());
+    });
+    deleteTableFamilyRegion();
+    HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+    final Map<TableName, List<String>> report =
+      hbck.reportUndeletedRegionsInMeta(namespaceOrTable);
+    long resultingExtraRegions = report.keySet().stream().mapToLong(nsTbl ->
+      report.get(nsTbl).size()).sum();
+    assertEquals(expectedUndeletedRegions, resultingExtraRegions);
+    String expectedResult = "Regions in Meta but having no valid table, for each table:\n";
+    String result = testFormatUndeleteRegionsInMeta(null);
+    //validates initial execute message
+    assertTrue(result.contains(expectedResult));
+    //validates our test table region is reported as extra
+    expectedResult = "\t" + DELETED_TABLE_NAME.getNameAsString() + "->\n\t\t"
+      + DELETED_TABLE_NAME.getNameAsString();
+    assertTrue(result.contains(expectedResult));
+    //validates remove region with --fix
+    result = testFormatUndeleteRegionsInMeta("-f");
+    expectedResult = "\n\tRegions that had relate to the not found table '"
+      + DELETED_TABLE_NAME.getNameAsString() + "' and got removed from Meta: "
+      + resultingExtraRegions;
+    assertTrue(result.contains(expectedResult));
+  }
+
+  private String testFormatUndeleteRegionsInMeta(String options) throws IOException {
+    if (options == null) {
+      return testRunWithArgs(new String[]{REPORT_UNDELETED_REGIONS_IN_META});
+    } else {
+      return testRunWithArgs(new String[]{REPORT_UNDELETED_REGIONS_IN_META, options});
+    }
+  }
+
 }