From 61a9cd652a47ffa4ba8e214002128b78933d4ace Mon Sep 17 00:00:00 2001 From: mengqi Date: Tue, 1 Dec 2020 14:07:07 +0800 Subject: [PATCH 01/10] remove version check --- .../src/main/java/org/apache/hbase/HBCK2.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 46de2ec8b9..56f7567f1b 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -22,6 +22,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Method; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -60,6 +61,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.mortbay.log.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,7 +137,8 @@ void checkHBCKSupport(ClusterConnection connection, String cmd, String ... suppo getClusterMetrics(EnumSet.of(ClusterMetrics.Option.HBASE_VERSION)).getHBaseVersion(); String [] thresholdVersions = supportedVersions == null || supportedVersions.length == 0? MINIMUM_HBCK2_VERSION: supportedVersions; - boolean supported = Version.check(serverVersion, thresholdVersions); +// boolean supported = Version.check(serverVersion, thresholdVersions); + boolean supported=true; if (!supported) { throw new UnsupportedOperationException(cmd + " not supported on server version=" + serverVersion + "; needs at least a server that matches or exceeds " + @@ -149,8 +152,15 @@ void checkFunctionSupported(ClusterConnection connection, String cmd) throws IOE LOG.info("Skipped {} command version check; 'skip' set", cmd); return; } - List methods = Arrays.asList(connection.getHbck().getClass().getDeclaredMethods()); + LOG.info("Class name: {}",connection.getHbck().getClass().getName()); + URL location = connection.getHbck().getClass().getProtectionDomain().getCodeSource().getLocation(); + LOG.info("CodeSource path: {}" ,connection.getHbck().getClass().getProtectionDomain().getCodeSource()); + List methods = Arrays.asList(connection.getHbck().getClass().getMethods()); List finalCmds = FUNCTION_NAME_MAP.getOrDefault(cmd, Collections.singletonList(cmd)); + for (Method method : methods) { + LOG.info("Method names: {}",method.getName()); + } + LOG.info("FinalCmds {}",finalCmds); boolean supported = methods.stream().anyMatch(method -> finalCmds.contains(method.getName())); if (!supported) { throw new UnsupportedOperationException("This HBase cluster does not support command: " From d0c9b6fefc3eb3d05a7c7073272f36a307ec7255 Mon Sep 17 00:00:00 2001 From: mengqi Date: Tue, 9 Feb 2021 10:39:22 +0800 Subject: [PATCH 02/10] report dirty metadata in meta table --- .../apache/hbase/FsRegionsMetaRecoverer.java | 25 +++++++-- .../src/main/java/org/apache/hbase/HBCK2.java | 53 ++++++++++++++++++- .../apache/hbase/HBCKMetaTableAccessor.java | 48 ++++++++++++++--- 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java index e8a9df70d3..69cedb9620 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java @@ -37,10 +37,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.FSUtils; @@ -92,6 +89,26 @@ public Map> reportTablesMissingRegions(final List n return extraChecker.reportTablesRegions(namespacesOrTables, this::findExtraRegionsInMETA); } + public Map> + reportDirtyMetadata() throws IOException { + return HBCKMetaTableAccessor.getDirtyMetadata(this.conn); + } + + + public void deleteDirtyMetadata(Map> reportMap) throws IOException { + Table table = conn.getTable(TableName.META_TABLE_NAME); + for (Map.Entry> entry : reportMap.entrySet()) { + List list=new ArrayList<>(); + for (byte[] bytes : entry.getValue()) { + Delete delete=new Delete(bytes); + list.add(delete); + } + table.delete(list); + LOG.info("delete dirty table {} metadata",entry.getKey().getNameAsString()); + } + table.close(); + } + List findMissingRegionsInMETA(String table) throws IOException { InternalMetaChecker missingChecker = new InternalMetaChecker<>(); return missingChecker.checkRegionsInMETA(table, (regions, dirs) -> { diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 928135b586..85f557cd59 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -63,9 +63,9 @@ import org.apache.hadoop.hbase.filter.SubstringComparator; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; -import org.mortbay.log.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,6 +102,7 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool { private static final String SET_REGION_STATE = "setRegionState"; private static final String SCHEDULE_RECOVERIES = "scheduleRecoveries"; private static final String GENERATE_TABLE_INFO = "generateMissingTableDescriptorFile"; + private static final String REPORT_DIRTY_METADATA = "reportDirtyMetadata"; private static final String FIX_META = "fixMeta"; // TODO update this map in case of the name of a method changes in Hbck interface // in org.apache.hadoop.hbase.client package. Or a new command is added and the hbck command @@ -280,6 +281,33 @@ Map> extraRegionsInMeta(String[] args) return result; } + Map> reportDirtyRegionsInMeta(String[] args) + throws Exception { + Options options = new Options(); + Option fixOption = Option.builder("f").longOpt("fix").build(); + options.addOption(fixOption); + // Parse command-line. + CommandLineParser parser = new DefaultParser(); + CommandLine commandLine; + commandLine = parser.parse(options, args, false); + boolean fix = commandLine.hasOption(fixOption.getOpt()); + Map> result = new HashMap<>(); + try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = + new FsRegionsMetaRecoverer(this.conf)) { + Map> reportMap = fsRegionsMetaRecoverer.reportDirtyMetadata(); + reportMap.forEach((key, value) -> result.put(key, value.stream().map(Bytes::toString) + .collect(Collectors.toList()))); + if(fix) { + fsRegionsMetaRecoverer.deleteDirtyMetadata(reportMap); + } + } catch (IOException e) { + LOG.error("Error on checking extra regions: ", e); + throw e; + } + + return result; + } + private List formatNameSpaceTableParam(String... nameSpaceOrTable) { return nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null; } @@ -422,6 +450,8 @@ private static String getCommandUsage() { writer.println(); usageBypass(writer); writer.println(); + usageReportDirtyRegionsInMeta(writer); + writer.println(); usageExtraRegionsInMeta(writer); writer.println(); usageFilesystem(writer); @@ -568,6 +598,14 @@ private static void usageReplication(PrintWriter writer) { writer.println(" purge if '--fix'."); } + private static void usageReportDirtyRegionsInMeta(PrintWriter writer){ + writer.println(" " + REPORT_DIRTY_METADATA + " [OPTIONS]"); + writer.println(" Options:"); + writer.println(" -f, --fix fix meta by delete all dirty metadata found."); + writer.println(" Looks for undeleted metadata in meta table. "); + writer.println(" Undeleted metadata means a table has been deleted, but not delete all metadata in meta."); + } + private static void usageExtraRegionsInMeta(PrintWriter writer) { writer.println(" " + EXTRA_REGIONS_IN_META + " ..."); @@ -955,6 +993,14 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc return EXIT_FAILURE; } break; + case REPORT_DIRTY_METADATA: + try { + Map> report = reportDirtyRegionsInMeta(purgeFirst(commands)); + System.out.println(formatDirtyMetadataReport(report)); + }catch (Exception e) { + return EXIT_FAILURE; + } + break; case GENERATE_TABLE_INFO: if(commands.length != 2) { @@ -988,6 +1034,11 @@ private String formatExtraRegionsReport(Map> report) { return formatReportMessage(message, (HashMap)report, s -> s); } + private String formatDirtyMetadataReport(Map> report) { + String message = "Dirty metadata in Meta, for each table:\n\t"; + return formatReportMessage(message, (HashMap)report, s -> s); + } + private String formatReportMessage(String reportMessage, Map> report, Function resolver){ final StringBuilder builder = new StringBuilder(); diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java index 64c2cf71ee..a3bcc8041a 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java @@ -23,13 +23,7 @@ import static org.apache.hadoop.hbase.HConstants.TABLE_STATE_QUALIFIER; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.SortedMap; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.regex.Matcher; @@ -60,6 +54,8 @@ import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; @@ -263,6 +259,44 @@ public static List getAllRegions(Connection conn) throws IOException }); } + + /** + * List all dirty metadata currently in META. + * @param conn a valid, open connection. + * @return a Map of all dirty metadata in META. + * @throws IOException on any issues related with scanning meta table + */ + public static Map> getDirtyMetadata(Connection conn) throws IOException { + Scan scan = new Scan(); + Map> dirtyTableRegions = new HashMap<>(); + List tables = getTables(conn); + Map tableNameMap = new HashMap<>(); + for (TableName tableName : tables) { + tableNameMap.put(tableName.getNameAsString(), tableName); + } + + Table metaTable = conn.getTable(TableName.META_TABLE_NAME); + ResultScanner resultScanner = metaTable.getScanner(scan); + for (Result result : resultScanner) { + List cells = result.listCells(); + for (Cell cell : cells) { + byte[] rowBytes = CellUtil.cloneRow(cell); + String row = Bytes.toString(rowBytes); + String tableName = row.split(",")[0]; + if (!tableNameMap.containsKey(tableName)) { + if (dirtyTableRegions.containsKey(tableNameMap.get(tableName))) { + dirtyTableRegions.get(tableNameMap.get(tableName)).add(rowBytes); + } else { + List list = new ArrayList<>(); + list.add(rowBytes); + dirtyTableRegions.put(tableNameMap.get(tableName), list); + } + } + } + } + return dirtyTableRegions; + } + /** * Scans all "table:state" cell values existing in meta and returns as a map of * TableName as key and TableState as the value. From 60adca4c26f86298d431ed5f9bf2a1a6f910a54c Mon Sep 17 00:00:00 2001 From: mengqi Date: Tue, 9 Feb 2021 10:41:44 +0800 Subject: [PATCH 03/10] recovery version check --- hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 85f557cd59..2e2e3a5553 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -144,8 +144,7 @@ void checkHBCKSupport(ClusterConnection connection, String cmd, String ... suppo getClusterMetrics(EnumSet.of(ClusterMetrics.Option.HBASE_VERSION)).getHBaseVersion(); String [] thresholdVersions = supportedVersions == null || supportedVersions.length == 0? MINIMUM_HBCK2_VERSION: supportedVersions; -// boolean supported = Version.check(serverVersion, thresholdVersions); - boolean supported=true; + boolean supported = Version.check(serverVersion, thresholdVersions); if (!supported) { throw new UnsupportedOperationException(cmd + " not supported on server version=" + serverVersion + "; needs at least a server that matches or exceeds " + From 9e05d907f3941ad80a451991e848489719d172d5 Mon Sep 17 00:00:00 2001 From: mengqi Date: Tue, 9 Feb 2021 10:44:24 +0800 Subject: [PATCH 04/10] recovery checkFunctionSupported --- hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 2e2e3a5553..d99dde79de 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -158,15 +158,8 @@ void checkFunctionSupported(ClusterConnection connection, String cmd) throws IOE LOG.info("Skipped {} command version check; 'skip' set", cmd); return; } - LOG.info("Class name: {}",connection.getHbck().getClass().getName()); - URL location = connection.getHbck().getClass().getProtectionDomain().getCodeSource().getLocation(); - LOG.info("CodeSource path: {}" ,connection.getHbck().getClass().getProtectionDomain().getCodeSource()); - List methods = Arrays.asList(connection.getHbck().getClass().getMethods()); + List methods = Arrays.asList(connection.getHbck().getClass().getDeclaredMethods()); List finalCmds = FUNCTION_NAME_MAP.getOrDefault(cmd, Collections.singletonList(cmd)); - for (Method method : methods) { - LOG.info("Method names: {}",method.getName()); - } - LOG.info("FinalCmds {}",finalCmds); boolean supported = methods.stream().anyMatch(method -> finalCmds.contains(method.getName())); if (!supported) { throw new UnsupportedOperationException("This HBase cluster does not support command: " From 8b26d09dde0cb71dd5cfaa2ca7e5ffa377b99011 Mon Sep 17 00:00:00 2001 From: mengqi Date: Tue, 9 Feb 2021 10:47:57 +0800 Subject: [PATCH 05/10] format code --- .../org/apache/hbase/HBCKMetaTableAccessor.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java index a3bcc8041a..119ba7c28e 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java @@ -267,19 +267,14 @@ public static List getAllRegions(Connection conn) throws IOException * @throws IOException on any issues related with scanning meta table */ public static Map> getDirtyMetadata(Connection conn) throws IOException { - Scan scan = new Scan(); Map> dirtyTableRegions = new HashMap<>(); - List tables = getTables(conn); - Map tableNameMap = new HashMap<>(); - for (TableName tableName : tables) { - tableNameMap.put(tableName.getNameAsString(), tableName); - } - + Map tableNameMap = new HashMap<>(); + getTables(conn).forEach(tableName -> tableNameMap.put(tableName.getNameAsString(), tableName)); Table metaTable = conn.getTable(TableName.META_TABLE_NAME); + Scan scan = new Scan(); ResultScanner resultScanner = metaTable.getScanner(scan); for (Result result : resultScanner) { - List cells = result.listCells(); - for (Cell cell : cells) { + result.listCells().forEach(cell -> { byte[] rowBytes = CellUtil.cloneRow(cell); String row = Bytes.toString(rowBytes); String tableName = row.split(",")[0]; @@ -292,7 +287,7 @@ public static Map> getDirtyMetadata(Connection conn) thr dirtyTableRegions.put(tableNameMap.get(tableName), list); } } - } + }); } return dirtyTableRegions; } From 6a974ae96d9569924efe1950f706db1d5c66c960 Mon Sep 17 00:00:00 2001 From: mengqi Date: Thu, 19 Aug 2021 22:37:47 +0800 Subject: [PATCH 06/10] change dirty region to undeleted region, add undelete region report UTs --- .../apache/hbase/FsRegionsMetaRecoverer.java | 55 ++++---- .../src/main/java/org/apache/hbase/HBCK2.java | 127 ++++++++---------- .../apache/hbase/HBCKMetaTableAccessor.java | 90 +++++-------- .../test/java/org/apache/hbase/TestHBCK2.java | 123 ++++++++++++----- 4 files changed, 209 insertions(+), 186 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java index 69cedb9620..da73c3f00a 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java @@ -17,21 +17,6 @@ */ package org.apache.hbase; -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,10 +25,19 @@ import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.FSUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; + /** * This class implements the inner works required for checking and recovering regions that wrongly * went missing in META, or are left present in META but with no equivalent FS dir. @@ -89,24 +83,35 @@ public Map> reportTablesMissingRegions(final List n return extraChecker.reportTablesRegions(namespacesOrTables, this::findExtraRegionsInMETA); } - public Map> - reportDirtyMetadata() throws IOException { - return HBCKMetaTableAccessor.getDirtyMetadata(this.conn); + public Map> reportUndeletedRegions() throws IOException { + return HBCKMetaTableAccessor.getUndeletedRegions(this.conn); } - - public void deleteDirtyMetadata(Map> reportMap) throws IOException { + public Map removeUndeletedRegion(Map> reportMap) throws IOException { Table table = conn.getTable(TableName.META_TABLE_NAME); - for (Map.Entry> entry : reportMap.entrySet()) { - List list=new ArrayList<>(); + Map map = new HashMap<>(); + for (Map.Entry> entry : reportMap.entrySet()) { + List list = new ArrayList<>(); for (byte[] bytes : entry.getValue()) { - Delete delete=new Delete(bytes); + Delete delete = new Delete(bytes); list.add(delete); } + map.put(entry.getKey(), list.size()); table.delete(list); - LOG.info("delete dirty table {} metadata",entry.getKey().getNameAsString()); } table.close(); + return map; + } + + void deleteRegions(TableName tableName,List rowkeys) throws IOException { + Table table = conn.getTable(tableName); + List list=new ArrayList<>(); + for (byte[] bytes : rowkeys) { + Delete delete=new Delete(bytes); + list.add(delete); + } + table.delete(list); + table.close(); } List findMissingRegionsInMETA(String table) throws IOException { diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index d99dde79de..fdcf4bf616 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -17,66 +17,31 @@ */ package org.apache.hbase; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.lang.reflect.Method; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; - import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.ClusterMetrics; -import org.apache.hadoop.hbase.CompareOperator; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Hbck; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SubstringComparator; import org.apache.hadoop.hbase.master.RegionState; - +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.thirdparty.org.apache.commons.cli.*; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; -import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser; -import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser; -import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; -import org.apache.hbase.thirdparty.org.apache.commons.cli.Option; -import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; -import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import java.io.*; +import java.lang.reflect.Method; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; /** * HBase fixup tool version 2, for hbase-2.0.0+ clusters. @@ -102,7 +67,8 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool { private static final String SET_REGION_STATE = "setRegionState"; private static final String SCHEDULE_RECOVERIES = "scheduleRecoveries"; private static final String GENERATE_TABLE_INFO = "generateMissingTableDescriptorFile"; - private static final String REPORT_DIRTY_METADATA = "reportDirtyMetadata"; + private static final String REPORT_UNDELETED_REGIONS_IN_META = + "reportUndeletedRegionsInMeta"; private static final String FIX_META = "fixMeta"; // TODO update this map in case of the name of a method changes in Hbck interface // in org.apache.hadoop.hbase.client package. Or a new command is added and the hbck command @@ -273,8 +239,8 @@ Map> extraRegionsInMeta(String[] args) return result; } - Map> reportDirtyRegionsInMeta(String[] args) - throws Exception { + Map> reportUndeletedRegionsInMeta(String[] args) + throws Exception { Options options = new Options(); Option fixOption = Option.builder("f").longOpt("fix").build(); options.addOption(fixOption); @@ -285,18 +251,20 @@ Map> reportDirtyRegionsInMeta(String[] args) boolean fix = commandLine.hasOption(fixOption.getOpt()); Map> result = new HashMap<>(); try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = - new FsRegionsMetaRecoverer(this.conf)) { - Map> reportMap = fsRegionsMetaRecoverer.reportDirtyMetadata(); - reportMap.forEach((key, value) -> result.put(key, value.stream().map(Bytes::toString) - .collect(Collectors.toList()))); - if(fix) { - fsRegionsMetaRecoverer.deleteDirtyMetadata(reportMap); + new FsRegionsMetaRecoverer(this.conf)) { + Map> reportMap = + fsRegionsMetaRecoverer.reportUndeletedRegions(); + reportMap.forEach((key, value) -> + result.put(TableName.valueOf(key), + value.stream().map(Bytes::toString).collect(Collectors.toList()))); + if (fix) { + Map map = + fsRegionsMetaRecoverer.removeUndeletedRegion(reportMap); + System.out.println(formatRemovedRegionsMessage(map)); } } catch (IOException e) { - LOG.error("Error on checking extra regions: ", e); throw e; } - return result; } @@ -442,7 +410,7 @@ private static String getCommandUsage() { writer.println(); usageBypass(writer); writer.println(); - usageReportDirtyRegionsInMeta(writer); + usageReportUndeletedRegionsInMeta(writer); writer.println(); usageExtraRegionsInMeta(writer); writer.println(); @@ -590,12 +558,23 @@ private static void usageReplication(PrintWriter writer) { writer.println(" purge if '--fix'."); } - private static void usageReportDirtyRegionsInMeta(PrintWriter writer){ - writer.println(" " + REPORT_DIRTY_METADATA + " [OPTIONS]"); + private static void usageReportUndeletedRegionsInMeta(PrintWriter writer){ + writer.println(" " + REPORT_UNDELETED_REGIONS_IN_META + " [OPTIONS]"); writer.println(" Options:"); - writer.println(" -f, --fix fix meta by delete all dirty metadata found."); - writer.println(" Looks for undeleted metadata in meta table. "); - writer.println(" Undeleted metadata means a table has been deleted, but not delete all metadata in meta."); + writer.println(" -f, --fix fix meta by removing all undeleted regions found."); + writer.println(" Reports regions present on hbase:meta, but related tables have been "); + writer.println(" deleted on hbase. Needs hbase:meta to be online."); + writer.println(" An example triggering undeleted regions report"); + writer.println(" $ HBCK2 " + REPORT_UNDELETED_REGIONS_IN_META); + writer.println(" Returns list of undeleted regions for each not found table"); + writer.println(" for each table on namespaces specified as parameter."); + + writer.println(" If master log continues to print 'TableNotFoundException', or master "); + writer.println(" ui report RITs for those table not found regions, or hbck.jsp web page "); + writer.println(" report regions but related tables not existing, remove those undeleted "); + writer.println(" undeleted regions with '--fix' option. "); + writer.println(" You should switch active master after remove undeleted regions, then "); + writer.println(" those abnormal regions info will disappear. "); } private static void usageExtraRegionsInMeta(PrintWriter writer) { @@ -985,10 +964,11 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc return EXIT_FAILURE; } break; - case REPORT_DIRTY_METADATA: + case REPORT_UNDELETED_REGIONS_IN_META: try { - Map> report = reportDirtyRegionsInMeta(purgeFirst(commands)); - System.out.println(formatDirtyMetadataReport(report)); + Map> report = + reportUndeletedRegionsInMeta(purgeFirst(commands)); + System.out.println(formatUndeletedRegionReport(report)); }catch (Exception e) { return EXIT_FAILURE; } @@ -1005,6 +985,7 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc break; default: + System.out.println("REPORT_UNDELETED_REGIONS_IN_META2"); showErrorMessage("Unsupported command: " + command); return EXIT_FAILURE; } @@ -1026,8 +1007,8 @@ private String formatExtraRegionsReport(Map> report) { return formatReportMessage(message, (HashMap)report, s -> s); } - private String formatDirtyMetadataReport(Map> report) { - String message = "Dirty metadata in Meta, for each table:\n\t"; + private String formatUndeletedRegionReport(Map> report) { + String message = "Regions in Meta but having no valid table, for each table:\n\t"; return formatReportMessage(message, (HashMap)report, s -> s); } @@ -1090,6 +1071,16 @@ private String formatRemovedRegionsMessage(int totalRemoved, return finalText.toString(); } + private String formatRemovedRegionsMessage(Map map) { + final StringBuilder finalText = new StringBuilder(); + for (Map.Entry entry : map.entrySet()) { + finalText.append("\n\tRegions that had relate to the not found table '") + .append(entry.getKey()).append("' and got removed from Meta: ") + .append(entry.getValue()).append("\n"); + } + return finalText.toString(); + } + private String buildHbck2AssignsCommand(List regions) { final StringBuilder builder = new StringBuilder(); builder.append("assigns "); diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java index 119ba7c28e..0e4d16bf3b 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java @@ -17,55 +17,27 @@ */ package org.apache.hbase; -import static org.apache.hadoop.hbase.HConstants.CATALOG_FAMILY; -import static org.apache.hadoop.hbase.HConstants.REGIONINFO_QUALIFIER; -import static org.apache.hadoop.hbase.HConstants.TABLE_FAMILY; -import static org.apache.hadoop.hbase.HConstants.TABLE_STATE_QUALIFIER; - -import java.io.IOException; -import java.util.*; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellBuilder; -import org.apache.hadoop.hbase.CellBuilderFactory; -import org.apache.hadoop.hbase.CellBuilderType; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.RegionLocations; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.RegionInfoBuilder; -import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.RegionReplicaUtil; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.master.RegionState; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; - +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.hadoop.hbase.HConstants.*; /** * hbck's local version of the MetaTableAccessor from the hbase repo @@ -261,35 +233,37 @@ public static List getAllRegions(Connection conn) throws IOException /** - * List all dirty metadata currently in META. + * List all undeleted regions currently in META. + * Undeleted regions means table not exist on hbase, but regions still in META. * @param conn a valid, open connection. * @return a Map of all dirty metadata in META. * @throws IOException on any issues related with scanning meta table */ - public static Map> getDirtyMetadata(Connection conn) throws IOException { - Map> dirtyTableRegions = new HashMap<>(); - Map tableNameMap = new HashMap<>(); - getTables(conn).forEach(tableName -> tableNameMap.put(tableName.getNameAsString(), tableName)); + public static Map> getUndeletedRegions(Connection conn) + throws IOException { + final Map> undeletedTableRegions = new HashMap<>(); + final Map tableNameMap = new HashMap<>(); + List tables = getTables(conn); + tables.forEach(tableName -> tableNameMap.put(tableName.getNameAsString(), tableName)); Table metaTable = conn.getTable(TableName.META_TABLE_NAME); Scan scan = new Scan(); ResultScanner resultScanner = metaTable.getScanner(scan); for (Result result : resultScanner) { - result.listCells().forEach(cell -> { - byte[] rowBytes = CellUtil.cloneRow(cell); - String row = Bytes.toString(rowBytes); - String tableName = row.split(",")[0]; - if (!tableNameMap.containsKey(tableName)) { - if (dirtyTableRegions.containsKey(tableNameMap.get(tableName))) { - dirtyTableRegions.get(tableNameMap.get(tableName)).add(rowBytes); - } else { - List list = new ArrayList<>(); - list.add(rowBytes); - dirtyTableRegions.put(tableNameMap.get(tableName), list); - } + Cell cell =result.listCells().get(0); + byte[] rowBytes = CellUtil.cloneRow(cell); + String row = Bytes.toString(rowBytes); + String tableName = row.split(",")[0]; + if (!tableNameMap.containsKey(tableName)) { + if (undeletedTableRegions.containsKey(tableName)) { + undeletedTableRegions.get(tableName).add(rowBytes); + } else { + List list = new ArrayList<>(); + list.add(rowBytes); + undeletedTableRegions.put(tableName, list); } - }); + } } - return dirtyTableRegions; + return undeletedTableRegions; } /** diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java index 14ab7ddf25..5eec8f388e 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java @@ -17,53 +17,30 @@ */ package org.apache.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.stream.Collectors; - import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Hbck; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Threads; - -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.*; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import static org.apache.hadoop.hbase.HConstants.TABLE_FAMILY; +import static org.apache.hadoop.hbase.HConstants.TABLE_STATE_QUALIFIER; +import static org.junit.Assert.*; + /** * Tests commands. For command-line parsing, see adjacent test. * @see TestHBCKCommandLineParsing @@ -76,6 +53,8 @@ public class TestHBCK2 { valueOf(TestHBCK2.class.getSimpleName() + "-REGIONS_STATES"); private final static String ASSIGNS = "assigns"; private static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta"; + private static final String REPORT_UNDELETED_REGIONS_IN_META = + "reportUndeletedRegionsInMeta"; @Rule public TestName testName = new TestName(); @@ -324,6 +303,12 @@ private TableName createTestTable(int totalRegions) throws IOException { return tableName; } + private TableName createTestTable(int totalRegions, String name) throws IOException { + TableName tableName = TableName.valueOf(name); + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("family1"), totalRegions); + return tableName; + } + private void testAddMissingRegionsInMetaForTables(int missingRegions, int totalRegions) throws Exception { TableName tableName = createTestTable(totalRegions); @@ -610,4 +595,72 @@ private void testReportExtraRegionsInMeta(int extraRegionsInTestTbl, assertEquals(expectedTotalExtraRegions, resultingExtraRegions); } + private void deleteTableFamilyRegion() { + try { + HBCKMetaTableAccessor.MetaScanner scanner = + new HBCKMetaTableAccessor.MetaScanner<>(); + List rowkeys = scanner.scanMeta(TEST_UTIL.getConnection(), + scan -> scan.addColumn(TABLE_FAMILY, TABLE_STATE_QUALIFIER), + r -> { + byte[] bytes = r.getRow(); + String row=Bytes.toString(bytes); + String tableName = row.split(",")[0]; + if(tableName.equals(TABLE_NAME.getNameAsString())){ + return bytes; + } + return null; + }); + try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = + new FsRegionsMetaRecoverer(TEST_UTIL.getConfiguration())) { + fsRegionsMetaRecoverer.deleteRegions(TableName.META_TABLE_NAME,rowkeys); + } + } catch (IOException e) { + fail(e.getMessage()); + } + } + + @Test + public void testReportUndeletedRegionsInMeta() throws Exception { + testReportUndeletedRegionsInMeta(5,5); + } + + private void testReportUndeletedRegionsInMeta(int undeletedRegions, + int expectedUndeletedRegions, String... namespaceOrTable) throws Exception { + List regions = HBCKMetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME); + regions.subList(0, undeletedRegions).forEach(r -> { + deleteRegionDir(TABLE_NAME, r.getEncodedName()); + }); + deleteTableFamilyRegion(); + HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); + final Map> report = + hbck.reportUndeletedRegionsInMeta(namespaceOrTable); + long resultingExtraRegions = report.keySet().stream().mapToLong(nsTbl -> + report.get(nsTbl).size()).sum(); + assertEquals(expectedUndeletedRegions, resultingExtraRegions); + String expectedResult = "Regions in Meta but having no valid table, for each table:\n"; + String result = testFormatUndeleteRegionsInMeta(null); + //validates initial execute message + assertTrue(result.contains(expectedResult)); + //validates our test table region is reported as extra + expectedResult = "\t" + TABLE_NAME.getNameAsString() + "->\n\t\t" + + TABLE_NAME.getNameAsString(); + assertTrue(result.contains(expectedResult)); + //validates remove region with --fix + result = testFormatUndeleteRegionsInMeta("-f"); + expectedResult = "\n\tRegions that had relate to the not found table '" + + TABLE_NAME.getNameAsString() + "' and got removed from Meta: " + + resultingExtraRegions; + assertTrue(result.contains(expectedResult)); + createTestTable(5, TABLE_NAME.getNameAsString()); + } + + private String testFormatUndeleteRegionsInMeta(String options) throws IOException { + if (options == null) { + return testRunWithArgs(new String[]{REPORT_UNDELETED_REGIONS_IN_META}); + } else { + return testRunWithArgs(new String[]{REPORT_UNDELETED_REGIONS_IN_META, options}); + } + } + } From c1a7b394dbddd4c0d9a07bf4d025a2f08803344f Mon Sep 17 00:00:00 2001 From: mengqi Date: Fri, 20 Aug 2021 08:49:01 +0800 Subject: [PATCH 07/10] get row bytes from result --- .../src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java index b228af86f5..3c9366d57b 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java @@ -250,8 +250,7 @@ public static Map> getUndeletedRegions(Connection conn) Scan scan = new Scan(); ResultScanner resultScanner = metaTable.getScanner(scan); for (Result result : resultScanner) { - Cell cell =result.listCells().get(0); - byte[] rowBytes = CellUtil.cloneRow(cell); + byte[] rowBytes = result.getRow(); String row = Bytes.toString(rowBytes); String tableName = row.split(",")[0]; if (!tableNameMap.containsKey(tableName)) { From d4ea548fa35f520aae83fd8a8a12b83947eebd4a Mon Sep 17 00:00:00 2001 From: mengqi Date: Fri, 20 Aug 2021 09:45:04 +0800 Subject: [PATCH 08/10] fix check style and fix TestHBCK2#testReportUndeletedRegionsInMeta --- .../apache/hbase/FsRegionsMetaRecoverer.java | 60 ++++++++------ .../src/main/java/org/apache/hbase/HBCK2.java | 56 ++++++++++--- .../apache/hbase/HBCKMetaTableAccessor.java | 54 +++++++++--- .../test/java/org/apache/hbase/TestHBCK2.java | 82 ++++++++++++------- 4 files changed, 175 insertions(+), 77 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java index da73c3f00a..ae536fd020 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java @@ -17,27 +17,38 @@ */ package org.apache.hbase; +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.FSUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Closeable; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; - /** * This class implements the inner works required for checking and recovering regions that wrongly * went missing in META, or are left present in META but with no equivalent FS dir. @@ -87,30 +98,29 @@ public Map> reportUndeletedRegions() throws IOException { return HBCKMetaTableAccessor.getUndeletedRegions(this.conn); } - public Map removeUndeletedRegion(Map> reportMap) throws IOException { + public Map + removeUndeletedRegion(Map> reportMap) throws IOException { Table table = conn.getTable(TableName.META_TABLE_NAME); Map map = new HashMap<>(); + List list = new ArrayList<>(); for (Map.Entry> entry : reportMap.entrySet()) { - List list = new ArrayList<>(); - for (byte[] bytes : entry.getValue()) { - Delete delete = new Delete(bytes); - list.add(delete); - } - map.put(entry.getKey(), list.size()); + entry.getValue().forEach(e -> list.add(new Delete(e))); + map.put(entry.getKey(), new Integer(list.size())); table.delete(list); + list.clear(); } table.close(); return map; } - void deleteRegions(TableName tableName,List rowkeys) throws IOException { + public void deleteRegions(TableName tableName,List rows) throws IOException { Table table = conn.getTable(tableName); - List list=new ArrayList<>(); - for (byte[] bytes : rowkeys) { - Delete delete=new Delete(bytes); - list.add(delete); - } - table.delete(list); + List list=new ArrayList<>(); + for (byte[] bytes : rows) { + Delete delete=new Delete(bytes); + list.add(delete); + } + table.delete(list); table.close(); } diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 3ecb76134b..b8cd17431c 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -17,31 +17,65 @@ */ package org.apache.hbase; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; + import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.ClusterMetrics; +import org.apache.hadoop.hbase.CompareOperator; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Hbck; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SubstringComparator; import org.apache.hadoop.hbase.master.RegionState; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hbase.thirdparty.org.apache.commons.cli.*; + import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.lang.reflect.Method; -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; +import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; +import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser; +import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser; +import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; +import org.apache.hbase.thirdparty.org.apache.commons.cli.Option; +import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; +import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; /** * HBase fixup tool version 2, for hbase-2.0.0+ clusters. diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java index 3c9366d57b..e7747c51de 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java @@ -17,27 +17,59 @@ */ package org.apache.hbase; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.*; +import static org.apache.hadoop.hbase.HConstants.CATALOG_FAMILY; +import static org.apache.hadoop.hbase.HConstants.REGIONINFO_QUALIFIER; +import static org.apache.hadoop.hbase.HConstants.TABLE_FAMILY; +import static org.apache.hadoop.hbase.HConstants.TABLE_STATE_QUALIFIER; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.SortedMap; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilder; +import org.apache.hadoop.hbase.CellBuilderFactory; +import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; + import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.*; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.apache.hadoop.hbase.HConstants.*; +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** * hbck's local version of the MetaTableAccessor from the hbase repo diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java index 5eec8f388e..df3964a976 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java @@ -17,30 +17,53 @@ */ package org.apache.hbase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Hbck; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Threads; -import org.junit.*; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.stream.Collectors; - -import static org.apache.hadoop.hbase.HConstants.TABLE_FAMILY; -import static org.apache.hadoop.hbase.HConstants.TABLE_STATE_QUALIFIER; -import static org.junit.Assert.*; - /** * Tests commands. For command-line parsing, see adjacent test. * @see TestHBCKCommandLineParsing @@ -49,6 +72,8 @@ public class TestHBCK2 { private static final Logger LOG = LoggerFactory.getLogger(TestHBCK2.class); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final TableName TABLE_NAME = TableName.valueOf(TestHBCK2.class.getSimpleName()); + private static final TableName DELETED_TABLE_NAME = TableName. + valueOf(TestHBCK2.class.getSimpleName() + "-DELETED"); private static final TableName REGION_STATES_TABLE_NAME = TableName. valueOf(TestHBCK2.class.getSimpleName() + "-REGIONS_STATES"); private final static String ASSIGNS = "assigns"; @@ -599,20 +624,17 @@ private void deleteTableFamilyRegion() { try { HBCKMetaTableAccessor.MetaScanner scanner = new HBCKMetaTableAccessor.MetaScanner<>(); - List rowkeys = scanner.scanMeta(TEST_UTIL.getConnection(), - scan -> scan.addColumn(TABLE_FAMILY, TABLE_STATE_QUALIFIER), - r -> { - byte[] bytes = r.getRow(); - String row=Bytes.toString(bytes); - String tableName = row.split(",")[0]; - if(tableName.equals(TABLE_NAME.getNameAsString())){ - return bytes; - } - return null; - }); + List rows = scanner.scanMeta(TEST_UTIL.getConnection(), + scan -> scan.addColumn(HConstants.TABLE_FAMILY,HConstants.TABLE_STATE_QUALIFIER), + r -> { + byte[] bytes = r.getRow(); + String row=Bytes.toString(bytes); + String tableName = row.split(",")[0]; + return tableName.equals(DELETED_TABLE_NAME.getNameAsString()) ? bytes : null; + }); try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = new FsRegionsMetaRecoverer(TEST_UTIL.getConfiguration())) { - fsRegionsMetaRecoverer.deleteRegions(TableName.META_TABLE_NAME,rowkeys); + fsRegionsMetaRecoverer.deleteRegions(TableName.META_TABLE_NAME, rows); } } catch (IOException e) { fail(e.getMessage()); @@ -626,10 +648,11 @@ public void testReportUndeletedRegionsInMeta() throws Exception { private void testReportUndeletedRegionsInMeta(int undeletedRegions, int expectedUndeletedRegions, String... namespaceOrTable) throws Exception { + createTestTable(5, DELETED_TABLE_NAME.getNameAsString()); List regions = HBCKMetaTableAccessor - .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME); + .getTableRegions(TEST_UTIL.getConnection(), DELETED_TABLE_NAME); regions.subList(0, undeletedRegions).forEach(r -> { - deleteRegionDir(TABLE_NAME, r.getEncodedName()); + deleteRegionDir(DELETED_TABLE_NAME, r.getEncodedName()); }); deleteTableFamilyRegion(); HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); @@ -643,16 +666,15 @@ private void testReportUndeletedRegionsInMeta(int undeletedRegions, //validates initial execute message assertTrue(result.contains(expectedResult)); //validates our test table region is reported as extra - expectedResult = "\t" + TABLE_NAME.getNameAsString() + "->\n\t\t" - + TABLE_NAME.getNameAsString(); + expectedResult = "\t" + DELETED_TABLE_NAME.getNameAsString() + "->\n\t\t" + + DELETED_TABLE_NAME.getNameAsString(); assertTrue(result.contains(expectedResult)); //validates remove region with --fix result = testFormatUndeleteRegionsInMeta("-f"); expectedResult = "\n\tRegions that had relate to the not found table '" - + TABLE_NAME.getNameAsString() + "' and got removed from Meta: " + + DELETED_TABLE_NAME.getNameAsString() + "' and got removed from Meta: " + resultingExtraRegions; assertTrue(result.contains(expectedResult)); - createTestTable(5, TABLE_NAME.getNameAsString()); } private String testFormatUndeleteRegionsInMeta(String options) throws IOException { From 121908cff38e2d52dec55966549c9e7071267177 Mon Sep 17 00:00:00 2001 From: mengqi Date: Fri, 20 Aug 2021 10:12:45 +0800 Subject: [PATCH 09/10] remove err describe --- hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index b8cd17431c..85e5fb39e0 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -610,8 +610,6 @@ private static void usageReportUndeletedRegionsInMeta(PrintWriter writer){ writer.println(" An example triggering undeleted regions report"); writer.println(" $ HBCK2 " + REPORT_UNDELETED_REGIONS_IN_META); writer.println(" Returns list of undeleted regions for each not found table"); - writer.println(" for each table on namespaces specified as parameter."); - writer.println(" If master log continues to print 'TableNotFoundException', or master "); writer.println(" ui report RITs for those table not found regions, or hbck.jsp web page "); writer.println(" report regions but related tables not existing, remove those undeleted "); From a7482dc74235bb33aaf68d75e7c3aeeac0a20d73 Mon Sep 17 00:00:00 2001 From: mengqi Date: Fri, 20 Aug 2021 10:21:50 +0800 Subject: [PATCH 10/10] update usageReportUndeletedRegionsInMeta --- hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 85e5fb39e0..819a33a9ea 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -604,16 +604,16 @@ private static void usageReplication(PrintWriter writer) { private static void usageReportUndeletedRegionsInMeta(PrintWriter writer){ writer.println(" " + REPORT_UNDELETED_REGIONS_IN_META + " [OPTIONS]"); writer.println(" Options:"); - writer.println(" -f, --fix fix meta by removing all undeleted regions found."); + writer.println(" -f, --fix fix meta by removing all undeleted regions found. "); writer.println(" Reports regions present on hbase:meta, but related tables have been "); - writer.println(" deleted on hbase. Needs hbase:meta to be online."); - writer.println(" An example triggering undeleted regions report"); + writer.println(" deleted on hbase. Needs hbase:meta to be online. "); + writer.println(" An example triggering undeleted regions report "); writer.println(" $ HBCK2 " + REPORT_UNDELETED_REGIONS_IN_META); writer.println(" Returns list of undeleted regions for each not found table"); writer.println(" If master log continues to print 'TableNotFoundException', or master "); writer.println(" ui report RITs for those table not found regions, or hbck.jsp web page "); - writer.println(" report regions but related tables not existing, remove those undeleted "); - writer.println(" undeleted regions with '--fix' option. "); + writer.println(" report regions but related tables not existing, remove these undeleted "); + writer.println(" regions with '--fix' option. "); writer.println(" You should switch active master after remove undeleted regions, then "); writer.println(" those abnormal regions info will disappear. "); }