From 513b6b421c06910d43fead2e867506669341934e Mon Sep 17 00:00:00 2001 From: Anjan Date: Mon, 4 Jan 2021 21:43:50 +0530 Subject: [PATCH 1/4] HBASE-25445: Use WAL FS instead of master FS --- .../java/org/apache/hadoop/hbase/master/SplitWALManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java index aa91c84cb672..6db094c4e6df 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java @@ -85,8 +85,7 @@ public SplitWALManager(MasterServices master) throws IOException { this.splitWorkerAssigner = new SplitWorkerAssigner(this.master, conf.getInt(HBASE_SPLIT_WAL_MAX_SPLITTER, DEFAULT_HBASE_SPLIT_WAL_MAX_SPLITTER)); this.rootDir = master.getMasterFileSystem().getWALRootDir(); - // TODO: This should be the WAL FS, not the Master FS? - this.fs = master.getMasterFileSystem().getFileSystem(); + this.fs = master.getMasterFileSystem().getWALFileSystem(); this.walArchiveDir = new Path(this.rootDir, HConstants.HREGION_OLDLOGDIR_NAME); } From df2b0272344c072285c71be878650cbd9185a937 Mon Sep 17 00:00:00 2001 From: Anjan Date: Thu, 7 Jan 2021 10:03:12 +0530 Subject: [PATCH 2/4] HBASE-25445: Add UT for splitWALManager archive --- .../hbase/master/TestSplitWALManager.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java index 7edb011f97f4..0e882b8dc52b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java @@ -31,6 +31,15 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure; +import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface; import org.apache.hadoop.hbase.procedure2.Procedure; @@ -43,6 +52,7 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.junit.After; @@ -54,6 +64,8 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category({ MasterTests.class, LargeTests.class }) @@ -63,6 +75,7 @@ public class TestSplitWALManager { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestSplitWALManager.class); + private static final Logger LOG = LoggerFactory.getLogger(TestSplitWALManager.class); private static HBaseTestingUtility TEST_UTIL; private HMaster master; private SplitWALManager splitWALManager; @@ -86,6 +99,59 @@ public void teardown() throws Exception { TEST_UTIL.shutdownMiniCluster(); } + @Test + public void testWALArchiveWithDifferentWalAndRootFS() throws Exception{ + HBaseTestingUtility TEST_UTIL_2 = new HBaseTestingUtility(); + Path dir = TEST_UTIL.getDataTestDirOnTestFS("testWalDir"); + TEST_UTIL_2.getConfiguration().set(CommonFSUtils.HBASE_WAL_DIR, dir.toString()); + CommonFSUtils.setWALRootDir(TEST_UTIL_2.getConfiguration(), dir); + TEST_UTIL_2.startMiniCluster(3); + HMaster TU2_master = TEST_UTIL_2.getHBaseCluster().getMaster(); + LOG.info("The Master FS is pointing to: " + TU2_master.getMasterFileSystem() + .getFileSystem().getUri()); + LOG.info("The WAL FS is pointing to: " + TU2_master.getMasterFileSystem() + .getWALFileSystem().getUri()); + Table TABLE = TEST_UTIL_2.createTable(TABLE_NAME, FAMILY); + TEST_UTIL_2.waitTableAvailable(TABLE_NAME); + Admin admin = TEST_UTIL_2.getAdmin(); + MasterProcedureEnv env = TEST_UTIL_2.getMiniHBaseCluster().getMaster() + .getMasterProcedureExecutor().getEnvironment(); + final ProcedureExecutor executor = TEST_UTIL_2.getMiniHBaseCluster() + .getMaster().getMasterProcedureExecutor(); + List regionInfos = admin.getRegions(TABLE_NAME); + SplitTableRegionProcedure splitProcedure = new SplitTableRegionProcedure( + env, regionInfos.get(0), Bytes.toBytes("row5")); + // Populate some rows in the table + LOG.info("Beginning put data to the table: " + TABLE.toString()); + int rowCount = 5; + for (int i = 0; i < rowCount; i++) { + byte[] row = Bytes.toBytes("row" + i); + Put put = new Put(row); + put.addColumn(FAMILY, FAMILY, FAMILY); + TABLE.put(put); + } + executor.submitProcedure(splitProcedure); + LOG.info("Submitted SplitProcedure."); + TEST_UTIL_2.waitFor(30000, () -> executor.getProcedures().stream() + .filter(p -> p instanceof TransitRegionStateProcedure) + .map(p -> (TransitRegionStateProcedure) p) + .anyMatch(p -> TABLE_NAME.equals(p.getTableName()))); + TEST_UTIL_2.getMiniHBaseCluster().killRegionServer( + TEST_UTIL_2.getMiniHBaseCluster().getRegionServer(0).getServerName()); + TEST_UTIL_2.getMiniHBaseCluster().startRegionServer(); + TEST_UTIL_2.waitUntilNoRegionsInTransition(); + Scan scan = new Scan(); + ResultScanner results = TABLE.getScanner(scan); + int scanRowCount = 0; + Result result = null; + while ((result = results.next()) != null) { + scanRowCount++; + } + Assert.assertEquals("Got " + scanRowCount + " rows when " + rowCount + + " were expected.", rowCount, scanRowCount); + TEST_UTIL_2.shutdownMiniCluster(); + } + @Test public void testAcquireAndRelease() throws Exception { List testProcedures = new ArrayList<>(); From d03125fe4a262959b23c83aae610e527431ae94b Mon Sep 17 00:00:00 2001 From: Anjan Date: Thu, 7 Jan 2021 11:14:57 +0530 Subject: [PATCH 3/4] Rectify LOG line --- .../org/apache/hadoop/hbase/master/TestSplitWALManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java index 0e882b8dc52b..ccbd14e87825 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java @@ -122,7 +122,7 @@ public void testWALArchiveWithDifferentWalAndRootFS() throws Exception{ SplitTableRegionProcedure splitProcedure = new SplitTableRegionProcedure( env, regionInfos.get(0), Bytes.toBytes("row5")); // Populate some rows in the table - LOG.info("Beginning put data to the table: " + TABLE.toString()); + LOG.info("Beginning put data to the table: " + TABLE_NAME.toString()); int rowCount = 5; for (int i = 0; i < rowCount; i++) { byte[] row = Bytes.toBytes("row" + i); From 59ca6d173d3b26b330cda2e2f91ce6d14b847bde Mon Sep 17 00:00:00 2001 From: Anjan Date: Thu, 7 Jan 2021 12:19:48 +0530 Subject: [PATCH 4/4] HBASE-25445: Address feedback; Remove un-needed result variable; Rename local variables --- .../hbase/master/TestSplitWALManager.java | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java index ccbd14e87825..10eda749891d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; @@ -101,22 +100,22 @@ public void teardown() throws Exception { @Test public void testWALArchiveWithDifferentWalAndRootFS() throws Exception{ - HBaseTestingUtility TEST_UTIL_2 = new HBaseTestingUtility(); + HBaseTestingUtility test_util_2 = new HBaseTestingUtility(); Path dir = TEST_UTIL.getDataTestDirOnTestFS("testWalDir"); - TEST_UTIL_2.getConfiguration().set(CommonFSUtils.HBASE_WAL_DIR, dir.toString()); - CommonFSUtils.setWALRootDir(TEST_UTIL_2.getConfiguration(), dir); - TEST_UTIL_2.startMiniCluster(3); - HMaster TU2_master = TEST_UTIL_2.getHBaseCluster().getMaster(); - LOG.info("The Master FS is pointing to: " + TU2_master.getMasterFileSystem() + test_util_2.getConfiguration().set(CommonFSUtils.HBASE_WAL_DIR, dir.toString()); + CommonFSUtils.setWALRootDir(test_util_2.getConfiguration(), dir); + test_util_2.startMiniCluster(3); + HMaster master2 = test_util_2.getHBaseCluster().getMaster(); + LOG.info("The Master FS is pointing to: " + master2.getMasterFileSystem() .getFileSystem().getUri()); - LOG.info("The WAL FS is pointing to: " + TU2_master.getMasterFileSystem() + LOG.info("The WAL FS is pointing to: " + master2.getMasterFileSystem() .getWALFileSystem().getUri()); - Table TABLE = TEST_UTIL_2.createTable(TABLE_NAME, FAMILY); - TEST_UTIL_2.waitTableAvailable(TABLE_NAME); - Admin admin = TEST_UTIL_2.getAdmin(); - MasterProcedureEnv env = TEST_UTIL_2.getMiniHBaseCluster().getMaster() + Table table = test_util_2.createTable(TABLE_NAME, FAMILY); + test_util_2.waitTableAvailable(TABLE_NAME); + Admin admin = test_util_2.getAdmin(); + MasterProcedureEnv env = test_util_2.getMiniHBaseCluster().getMaster() .getMasterProcedureExecutor().getEnvironment(); - final ProcedureExecutor executor = TEST_UTIL_2.getMiniHBaseCluster() + final ProcedureExecutor executor = test_util_2.getMiniHBaseCluster() .getMaster().getMasterProcedureExecutor(); List regionInfos = admin.getRegions(TABLE_NAME); SplitTableRegionProcedure splitProcedure = new SplitTableRegionProcedure( @@ -128,28 +127,27 @@ public void testWALArchiveWithDifferentWalAndRootFS() throws Exception{ byte[] row = Bytes.toBytes("row" + i); Put put = new Put(row); put.addColumn(FAMILY, FAMILY, FAMILY); - TABLE.put(put); + table.put(put); } executor.submitProcedure(splitProcedure); LOG.info("Submitted SplitProcedure."); - TEST_UTIL_2.waitFor(30000, () -> executor.getProcedures().stream() + test_util_2.waitFor(30000, () -> executor.getProcedures().stream() .filter(p -> p instanceof TransitRegionStateProcedure) .map(p -> (TransitRegionStateProcedure) p) .anyMatch(p -> TABLE_NAME.equals(p.getTableName()))); - TEST_UTIL_2.getMiniHBaseCluster().killRegionServer( - TEST_UTIL_2.getMiniHBaseCluster().getRegionServer(0).getServerName()); - TEST_UTIL_2.getMiniHBaseCluster().startRegionServer(); - TEST_UTIL_2.waitUntilNoRegionsInTransition(); + test_util_2.getMiniHBaseCluster().killRegionServer( + test_util_2.getMiniHBaseCluster().getRegionServer(0).getServerName()); + test_util_2.getMiniHBaseCluster().startRegionServer(); + test_util_2.waitUntilNoRegionsInTransition(); Scan scan = new Scan(); - ResultScanner results = TABLE.getScanner(scan); + ResultScanner results = table.getScanner(scan); int scanRowCount = 0; - Result result = null; - while ((result = results.next()) != null) { + while (results.next() != null) { scanRowCount++; } Assert.assertEquals("Got " + scanRowCount + " rows when " + rowCount + " were expected.", rowCount, scanRowCount); - TEST_UTIL_2.shutdownMiniCluster(); + test_util_2.shutdownMiniCluster(); } @Test