From 059fcc4545c2540a0de08b92fc9c16f32a841340 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Tue, 7 Nov 2023 13:13:15 -0500 Subject: [PATCH 1/6] Use existing HMaster Connection in MobFileCleanerChore --- .../org/apache/hadoop/hbase/mob/MobFileCleanerChore.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java index 2c78c6f5ac74..a8146d723fe3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java @@ -36,8 +36,6 @@ import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.master.HMaster; @@ -148,8 +146,7 @@ public void cleanupObsoleteMobFiles(Configuration conf, TableName table) throws // So, if MOB file creation time is greater than this maxTimeToArchive, // this will be skipped and won't be archived. long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive; - try (final Connection conn = ConnectionFactory.createConnection(conf); - final Admin admin = conn.getAdmin();) { + try (final Admin admin = master.getConnection().getAdmin()) { TableDescriptor htd = admin.getDescriptor(table); List list = MobUtils.getMobColumnFamilies(htd); if (list.size() == 0) { From 9a0352c2c055e45e1e17649d7d8ec86b53c32ee0 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Tue, 7 Nov 2023 14:31:00 -0500 Subject: [PATCH 2/6] fix TestMobFileCleanerChore --- .../java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java | 4 ---- .../org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java index a8146d723fe3..894723672eaa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java @@ -92,10 +92,6 @@ private void checkObsoleteConfigurations() { } } - public MobFileCleanerChore() { - this.master = null; - } - @Override protected void chore() { TableDescriptors htds = master.getTableDescriptors(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java index bdc3cce13e4c..126cdcca9e30 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java @@ -92,7 +92,7 @@ public void setUp() throws Exception { HTU.startMiniCluster(); admin = HTU.getAdmin(); - chore = new MobFileCleanerChore(); + chore = new MobFileCleanerChore(HTU.getMiniHBaseCluster().getMaster()); familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); tableDescriptor = HTU.createModifyableTableDescriptor("testMobCompactTable") From 5eb7a02bd5275162192ddb641888f534252623c8 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Tue, 7 Nov 2023 20:14:56 -0500 Subject: [PATCH 3/6] Support more test uses of MobFileCleanerChore --- .../hbase/IntegrationTestMobCompaction.java | 2 +- .../apache/hadoop/hbase/master/HMaster.java | 2 +- .../hadoop/hbase/mob/MobFileCleanerChore.java | 261 +++++++++--------- .../hadoop/hbase/mob/MobStressToolRunner.java | 3 +- .../mob/TestMobCompactionWithDefaults.java | 2 +- .../hbase/mob/TestMobFileCleanerChore.java | 2 +- 6 files changed, 138 insertions(+), 134 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java index f54d815ad4db..af54764cf695 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java @@ -251,7 +251,7 @@ public void run() { try { LOG.info("MOB cleanup chore started ..."); if (chore == null) { - chore = new MobFileCleanerChore(); + chore = new MobFileCleanerChore(admin); } chore.cleanupObsoleteMobFiles(conf, table.getName()); LOG.info("MOB cleanup chore finished"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 3c433f11a689..14e963ea7793 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1491,7 +1491,7 @@ public void updateConfigurationForQuotasObserver(Configuration conf) { conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs); } - private void initMobCleaner() { + private void initMobCleaner() throws IOException { this.mobFileCleanerChore = new MobFileCleanerChore(this); getChoreService().scheduleChore(mobFileCleanerChore); this.mobFileCompactionChore = new MobFileCompactionChore(this); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java index 894723672eaa..3d1ee0ce775f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java @@ -60,9 +60,10 @@ public class MobFileCleanerChore extends ScheduledChore { private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanerChore.class); private final HMaster master; + private final Admin admin; private ExpiredMobFileCleaner cleaner; - public MobFileCleanerChore(HMaster master) { + public MobFileCleanerChore(HMaster master) throws IOException { super(master.getServerName() + "-MobFileCleanerChore", master, master.getConfiguration().getInt(MobConstants.MOB_CLEANER_PERIOD, MobConstants.DEFAULT_MOB_CLEANER_PERIOD), @@ -70,6 +71,7 @@ public MobFileCleanerChore(HMaster master) { MobConstants.DEFAULT_MOB_CLEANER_PERIOD), TimeUnit.SECONDS); this.master = master; + this.admin = master.getConnection().getAdmin(); cleaner = new ExpiredMobFileCleaner(); cleaner.setConf(master.getConfiguration()); checkObsoleteConfigurations(); @@ -92,6 +94,11 @@ private void checkObsoleteConfigurations() { } } + public MobFileCleanerChore(Admin admin) { + this.master = null; + this.admin = admin; + } + @Override protected void chore() { TableDescriptors htds = master.getTableDescriptors(); @@ -142,151 +149,147 @@ public void cleanupObsoleteMobFiles(Configuration conf, TableName table) throws // So, if MOB file creation time is greater than this maxTimeToArchive, // this will be skipped and won't be archived. long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive; - try (final Admin admin = master.getConnection().getAdmin()) { - TableDescriptor htd = admin.getDescriptor(table); - List list = MobUtils.getMobColumnFamilies(htd); - if (list.size() == 0) { - LOG.info("Skipping non-MOB table [{}]", table); - return; - } else { - LOG.info("Only MOB files whose creation time older than {} will be archived, table={}", - maxCreationTimeToArchive, table); - } + TableDescriptor htd = admin.getDescriptor(table); + List list = MobUtils.getMobColumnFamilies(htd); + if (list.size() == 0) { + LOG.info("Skipping non-MOB table [{}]", table); + return; + } else { + LOG.info("Only MOB files whose creation time older than {} will be archived, table={}", + maxCreationTimeToArchive, table); + } - FileSystem fs = FileSystem.get(conf); - Set regionNames = new HashSet<>(); - Path rootDir = CommonFSUtils.getRootDir(conf); - Path tableDir = CommonFSUtils.getTableDir(rootDir, table); - List regionDirs = FSUtils.getRegionDirs(fs, tableDir); + FileSystem fs = FileSystem.get(conf); + Set regionNames = new HashSet<>(); + Path rootDir = CommonFSUtils.getRootDir(conf); + Path tableDir = CommonFSUtils.getTableDir(rootDir, table); + List regionDirs = FSUtils.getRegionDirs(fs, tableDir); - Set allActiveMobFileName = new HashSet(); - for (Path regionPath : regionDirs) { - regionNames.add(regionPath.getName()); - for (ColumnFamilyDescriptor hcd : list) { - String family = hcd.getNameAsString(); - Path storePath = new Path(regionPath, family); - boolean succeed = false; - Set regionMobs = new HashSet(); + Set allActiveMobFileName = new HashSet(); + for (Path regionPath : regionDirs) { + regionNames.add(regionPath.getName()); + for (ColumnFamilyDescriptor hcd : list) { + String family = hcd.getNameAsString(); + Path storePath = new Path(regionPath, family); + boolean succeed = false; + Set regionMobs = new HashSet(); - while (!succeed) { - if (!fs.exists(storePath)) { - String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore" - + " execution, aborting MOB file cleaner chore.", storePath); - throw new IOException(errMsg); + while (!succeed) { + if (!fs.exists(storePath)) { + String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore" + + " execution, aborting MOB file cleaner chore.", storePath); + throw new IOException(errMsg); + } + RemoteIterator rit = fs.listLocatedStatus(storePath); + List storeFiles = new ArrayList(); + // Load list of store files first + while (rit.hasNext()) { + Path p = rit.next().getPath(); + if (fs.isFile(p)) { + storeFiles.add(p); } - RemoteIterator rit = fs.listLocatedStatus(storePath); - List storeFiles = new ArrayList(); - // Load list of store files first - while (rit.hasNext()) { - Path p = rit.next().getPath(); - if (fs.isFile(p)) { - storeFiles.add(p); + } + LOG.info("Found {} store files in: {}", storeFiles.size(), storePath); + Path currentPath = null; + try { + for (Path pp : storeFiles) { + currentPath = pp; + LOG.trace("Store file: {}", pp); + HStoreFile sf = null; + byte[] mobRefData = null; + byte[] bulkloadMarkerData = null; + try { + sf = new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true); + sf.initReader(); + mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS); + bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY); + // close store file to avoid memory leaks + sf.closeStoreFile(true); + } catch (IOException ex) { + // When FileBased SFT is active the store dir can contain corrupted or incomplete + // files. So read errors are expected. We just skip these files. + if (ex instanceof FileNotFoundException) { + throw ex; + } + LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex); + continue; } - } - LOG.info("Found {} store files in: {}", storeFiles.size(), storePath); - Path currentPath = null; - try { - for (Path pp : storeFiles) { - currentPath = pp; - LOG.trace("Store file: {}", pp); - HStoreFile sf = null; - byte[] mobRefData = null; - byte[] bulkloadMarkerData = null; - try { - sf = new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true); - sf.initReader(); - mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS); - bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY); - // close store file to avoid memory leaks - sf.closeStoreFile(true); - } catch (IOException ex) { - // When FileBased SFT is active the store dir can contain corrupted or incomplete - // files. So read errors are expected. We just skip these files. - if (ex instanceof FileNotFoundException) { - throw ex; - } - LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), - ex); + if (mobRefData == null) { + if (bulkloadMarkerData == null) { + LOG.warn("Found old store file with no MOB_FILE_REFS: {} - " + + "can not proceed until all old files will be MOB-compacted.", pp); + return; + } else { + LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp); continue; } - if (mobRefData == null) { - if (bulkloadMarkerData == null) { - LOG.warn("Found old store file with no MOB_FILE_REFS: {} - " - + "can not proceed until all old files will be MOB-compacted.", pp); - return; - } else { - LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp); - continue; - } - } - // file may or may not have MOB references, but was created by the distributed - // mob compaction code. - try { - SetMultimap mobs = - MobUtils.deserializeMobFileRefs(mobRefData).build(); - LOG.debug("Found {} mob references for store={}", mobs.size(), sf); - LOG.trace("Specific mob references found for store={} : {}", sf, mobs); - regionMobs.addAll(mobs.values()); - } catch (RuntimeException exception) { - throw new IOException("failure getting mob references for hfile " + sf, - exception); - } } - } catch (FileNotFoundException e) { - LOG.warn( - "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error", - currentPath, e); - regionMobs.clear(); - continue; + // file may or may not have MOB references, but was created by the distributed + // mob compaction code. + try { + SetMultimap mobs = + MobUtils.deserializeMobFileRefs(mobRefData).build(); + LOG.debug("Found {} mob references for store={}", mobs.size(), sf); + LOG.trace("Specific mob references found for store={} : {}", sf, mobs); + regionMobs.addAll(mobs.values()); + } catch (RuntimeException exception) { + throw new IOException("failure getting mob references for hfile " + sf, exception); + } } - succeed = true; + } catch (FileNotFoundException e) { + LOG.warn( + "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error", + currentPath, e); + regionMobs.clear(); + continue; } + succeed = true; + } - // Add MOB references for current region/family - allActiveMobFileName.addAll(regionMobs); - } // END column families - } // END regions - // Check if number of MOB files too big (over 1M) - if (allActiveMobFileName.size() > 1000000) { - LOG.warn("Found too many active MOB files: {}, table={}, " - + "this may result in high memory pressure.", allActiveMobFileName.size(), table); - } - LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table); - allActiveMobFileName.stream().forEach(LOG::trace); + // Add MOB references for current region/family + allActiveMobFileName.addAll(regionMobs); + } // END column families + } // END regions + // Check if number of MOB files too big (over 1M) + if (allActiveMobFileName.size() > 1000000) { + LOG.warn("Found too many active MOB files: {}, table={}, " + + "this may result in high memory pressure.", allActiveMobFileName.size(), table); + } + LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table); + allActiveMobFileName.stream().forEach(LOG::trace); - // Now scan MOB directories and find MOB files with no references to them - for (ColumnFamilyDescriptor hcd : list) { - List toArchive = new ArrayList(); - String family = hcd.getNameAsString(); - Path dir = MobUtils.getMobFamilyPath(conf, table, family); - RemoteIterator rit = fs.listLocatedStatus(dir); - while (rit.hasNext()) { - LocatedFileStatus lfs = rit.next(); - Path p = lfs.getPath(); - String[] mobParts = p.getName().split("_"); - String regionName = mobParts[mobParts.length - 1]; + // Now scan MOB directories and find MOB files with no references to them + for (ColumnFamilyDescriptor hcd : list) { + List toArchive = new ArrayList(); + String family = hcd.getNameAsString(); + Path dir = MobUtils.getMobFamilyPath(conf, table, family); + RemoteIterator rit = fs.listLocatedStatus(dir); + while (rit.hasNext()) { + LocatedFileStatus lfs = rit.next(); + Path p = lfs.getPath(); + String[] mobParts = p.getName().split("_"); + String regionName = mobParts[mobParts.length - 1]; - if (!regionNames.contains(regionName)) { - // MOB belonged to a region no longer hosted - long creationTime = fs.getFileStatus(p).getModificationTime(); - if (creationTime < maxCreationTimeToArchive) { - LOG.trace("Archiving MOB file {} creation time={}", p, - (fs.getFileStatus(p).getModificationTime())); - toArchive.add(p); - } else { - LOG.trace("Skipping fresh file: {}. Creation time={}", p, - fs.getFileStatus(p).getModificationTime()); - } + if (!regionNames.contains(regionName)) { + // MOB belonged to a region no longer hosted + long creationTime = fs.getFileStatus(p).getModificationTime(); + if (creationTime < maxCreationTimeToArchive) { + LOG.trace("Archiving MOB file {} creation time={}", p, + (fs.getFileStatus(p).getModificationTime())); + toArchive.add(p); } else { - LOG.trace("Keeping MOB file with existing region: {}", p); + LOG.trace("Skipping fresh file: {}. Creation time={}", p, + fs.getFileStatus(p).getModificationTime()); } + } else { + LOG.trace("Keeping MOB file with existing region: {}", p); } - LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(), - table, family); - archiveMobFiles(conf, table, family.getBytes(), toArchive); - LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, - family); } + LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(), + table, family); + archiveMobFiles(conf, table, family.getBytes(), toArchive); + LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, + family); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java index b8d1750f8063..4bc9e386a8af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java @@ -70,7 +70,7 @@ public class MobStressToolRunner { private long count = 500000; private double failureProb = 0.1; private Table table = null; - private MobFileCleanerChore chore = new MobFileCleanerChore(); + private MobFileCleanerChore chore; private static volatile boolean run = true; @@ -84,6 +84,7 @@ public void init(Configuration conf, long numRows) throws IOException { printConf(); Connection conn = ConnectionFactory.createConnection(this.conf); this.admin = conn.getAdmin(); + this.chore = new MobFileCleanerChore(admin); this.familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); this.tableDescriptor = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java index 69ba4ea24b29..673943ac3215 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java @@ -139,7 +139,7 @@ protected void additonalConfigSetup() { public void setUp() throws Exception { htuStart(); admin = HTU.getAdmin(); - cleanerChore = new MobFileCleanerChore(); + cleanerChore = new MobFileCleanerChore(admin); familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); tableDescriptor = HTU.createModifyableTableDescriptor(TestMobUtils.getTableName(test)) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java index 126cdcca9e30..705f637fa05e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java @@ -92,7 +92,7 @@ public void setUp() throws Exception { HTU.startMiniCluster(); admin = HTU.getAdmin(); - chore = new MobFileCleanerChore(HTU.getMiniHBaseCluster().getMaster()); + chore = new MobFileCleanerChore(admin); familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); tableDescriptor = HTU.createModifyableTableDescriptor("testMobCompactTable") From e6d88c72fea9b1964baf93968227dc4ef698ef7e Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sat, 11 Nov 2023 10:39:44 -0500 Subject: [PATCH 4/6] Move bulk of MobFileCleanerChore into MobFileCleanupUtil to simplify tests --- .../hbase/IntegrationTestMobCompaction.java | 14 +- .../apache/hadoop/hbase/master/HMaster.java | 2 +- .../hadoop/hbase/mob/MobFileCleanerChore.java | 221 +--------------- .../hadoop/hbase/mob/MobFileCleanupUtil.java | 241 ++++++++++++++++++ .../hadoop/hbase/mob/MobStressToolRunner.java | 10 +- .../mob/TestMobCompactionWithDefaults.java | 3 - .../hbase/mob/TestMobFileCleanerChore.java | 6 +- 7 files changed, 259 insertions(+), 238 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java index af54764cf695..8e1952a696cd 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.mob.FaultyMobStoreCompactor; import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.mob.MobFileCleanerChore; +import org.apache.hadoop.hbase.mob.MobFileCleanupUtil; import org.apache.hadoop.hbase.mob.MobStoreEngine; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.testclassification.IntegrationTests; @@ -100,7 +100,6 @@ public class IntegrationTestMobCompaction extends IntegrationTestBase { private static ColumnFamilyDescriptor familyDescriptor; private static Admin admin; private static Table table = null; - private static MobFileCleanerChore chore; private static volatile boolean run = true; @@ -249,12 +248,9 @@ static class CleanMobAndArchive implements Runnable { public void run() { while (run) { try { - LOG.info("MOB cleanup chore started ..."); - if (chore == null) { - chore = new MobFileCleanerChore(admin); - } - chore.cleanupObsoleteMobFiles(conf, table.getName()); - LOG.info("MOB cleanup chore finished"); + LOG.info("MOB cleanup started ..."); + MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, table.getName(), admin); + LOG.info("MOB cleanup finished"); Thread.sleep(130000); } catch (Exception e) { @@ -329,7 +325,7 @@ public void testMobCompaction() throws InterruptedException, IOException { LOG.info("Waiting for write thread to finish ..."); writeData.join(); // Cleanup again - chore.cleanupObsoleteMobFiles(conf, table.getName()); + MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, table.getName(), admin); if (util != null) { LOG.info("Archive cleaner started ..."); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 14e963ea7793..3c433f11a689 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1491,7 +1491,7 @@ public void updateConfigurationForQuotasObserver(Configuration conf) { conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs); } - private void initMobCleaner() throws IOException { + private void initMobCleaner() { this.mobFileCleanerChore = new MobFileCleanerChore(this); getChoreService().scheduleChore(mobFileCleanerChore); this.mobFileCompactionChore = new MobFileCompactionChore(this); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java index 3d1ee0ce775f..fda9f1292eb6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java @@ -17,40 +17,20 @@ */ package org.apache.hadoop.hbase.mob; -import java.io.FileNotFoundException; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.TableDescriptors; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.regionserver.BloomType; -import org.apache.hadoop.hbase.regionserver.HStoreFile; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.CommonFSUtils; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap; - /** * The class MobFileCleanerChore for running cleaner regularly to remove the expired and obsolete * (files which have no active references to) mob files. @@ -60,10 +40,9 @@ public class MobFileCleanerChore extends ScheduledChore { private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanerChore.class); private final HMaster master; - private final Admin admin; private ExpiredMobFileCleaner cleaner; - public MobFileCleanerChore(HMaster master) throws IOException { + public MobFileCleanerChore(HMaster master) { super(master.getServerName() + "-MobFileCleanerChore", master, master.getConfiguration().getInt(MobConstants.MOB_CLEANER_PERIOD, MobConstants.DEFAULT_MOB_CLEANER_PERIOD), @@ -71,7 +50,6 @@ public MobFileCleanerChore(HMaster master) throws IOException { MobConstants.DEFAULT_MOB_CLEANER_PERIOD), TimeUnit.SECONDS); this.master = master; - this.admin = master.getConnection().getAdmin(); cleaner = new ExpiredMobFileCleaner(); cleaner.setConf(master.getConfiguration()); checkObsoleteConfigurations(); @@ -94,11 +72,6 @@ private void checkObsoleteConfigurations() { } } - public MobFileCleanerChore(Admin admin) { - this.master = null; - this.admin = admin; - } - @Override protected void chore() { TableDescriptors htds = master.getTableDescriptors(); @@ -124,7 +97,10 @@ protected void chore() { try { // Now clean obsolete files for a table LOG.info("Cleaning obsolete MOB files from table={}", htd.getTableName()); - cleanupObsoleteMobFiles(master.getConfiguration(), htd.getTableName()); + try (final Admin admin = master.getConnection().getAdmin()) { + MobFileCleanupUtil.cleanupObsoleteMobFiles(master.getConfiguration(), htd.getTableName(), + admin); + } LOG.info("Cleaning obsolete MOB files finished for table={}", htd.getTableName()); } catch (IOException e) { LOG.error("Failed to clean the obsolete mob files for table={}", htd.getTableName(), e); @@ -132,191 +108,4 @@ protected void chore() { } } - /** - * Performs housekeeping file cleaning (called by MOB Cleaner chore) - * @param conf configuration - * @param table table name - * @throws IOException exception - */ - public void cleanupObsoleteMobFiles(Configuration conf, TableName table) throws IOException { - - long minAgeToArchive = - conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE); - // We check only those MOB files, which creation time is less - // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap - // gives us full confidence that all corresponding store files will - // exist at the time cleaning procedure begins and will be examined. - // So, if MOB file creation time is greater than this maxTimeToArchive, - // this will be skipped and won't be archived. - long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive; - TableDescriptor htd = admin.getDescriptor(table); - List list = MobUtils.getMobColumnFamilies(htd); - if (list.size() == 0) { - LOG.info("Skipping non-MOB table [{}]", table); - return; - } else { - LOG.info("Only MOB files whose creation time older than {} will be archived, table={}", - maxCreationTimeToArchive, table); - } - - FileSystem fs = FileSystem.get(conf); - Set regionNames = new HashSet<>(); - Path rootDir = CommonFSUtils.getRootDir(conf); - Path tableDir = CommonFSUtils.getTableDir(rootDir, table); - List regionDirs = FSUtils.getRegionDirs(fs, tableDir); - - Set allActiveMobFileName = new HashSet(); - for (Path regionPath : regionDirs) { - regionNames.add(regionPath.getName()); - for (ColumnFamilyDescriptor hcd : list) { - String family = hcd.getNameAsString(); - Path storePath = new Path(regionPath, family); - boolean succeed = false; - Set regionMobs = new HashSet(); - - while (!succeed) { - if (!fs.exists(storePath)) { - String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore" - + " execution, aborting MOB file cleaner chore.", storePath); - throw new IOException(errMsg); - } - RemoteIterator rit = fs.listLocatedStatus(storePath); - List storeFiles = new ArrayList(); - // Load list of store files first - while (rit.hasNext()) { - Path p = rit.next().getPath(); - if (fs.isFile(p)) { - storeFiles.add(p); - } - } - LOG.info("Found {} store files in: {}", storeFiles.size(), storePath); - Path currentPath = null; - try { - for (Path pp : storeFiles) { - currentPath = pp; - LOG.trace("Store file: {}", pp); - HStoreFile sf = null; - byte[] mobRefData = null; - byte[] bulkloadMarkerData = null; - try { - sf = new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true); - sf.initReader(); - mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS); - bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY); - // close store file to avoid memory leaks - sf.closeStoreFile(true); - } catch (IOException ex) { - // When FileBased SFT is active the store dir can contain corrupted or incomplete - // files. So read errors are expected. We just skip these files. - if (ex instanceof FileNotFoundException) { - throw ex; - } - LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex); - continue; - } - if (mobRefData == null) { - if (bulkloadMarkerData == null) { - LOG.warn("Found old store file with no MOB_FILE_REFS: {} - " - + "can not proceed until all old files will be MOB-compacted.", pp); - return; - } else { - LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp); - continue; - } - } - // file may or may not have MOB references, but was created by the distributed - // mob compaction code. - try { - SetMultimap mobs = - MobUtils.deserializeMobFileRefs(mobRefData).build(); - LOG.debug("Found {} mob references for store={}", mobs.size(), sf); - LOG.trace("Specific mob references found for store={} : {}", sf, mobs); - regionMobs.addAll(mobs.values()); - } catch (RuntimeException exception) { - throw new IOException("failure getting mob references for hfile " + sf, exception); - } - } - } catch (FileNotFoundException e) { - LOG.warn( - "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error", - currentPath, e); - regionMobs.clear(); - continue; - } - succeed = true; - } - - // Add MOB references for current region/family - allActiveMobFileName.addAll(regionMobs); - } // END column families - } // END regions - // Check if number of MOB files too big (over 1M) - if (allActiveMobFileName.size() > 1000000) { - LOG.warn("Found too many active MOB files: {}, table={}, " - + "this may result in high memory pressure.", allActiveMobFileName.size(), table); - } - LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table); - allActiveMobFileName.stream().forEach(LOG::trace); - - // Now scan MOB directories and find MOB files with no references to them - for (ColumnFamilyDescriptor hcd : list) { - List toArchive = new ArrayList(); - String family = hcd.getNameAsString(); - Path dir = MobUtils.getMobFamilyPath(conf, table, family); - RemoteIterator rit = fs.listLocatedStatus(dir); - while (rit.hasNext()) { - LocatedFileStatus lfs = rit.next(); - Path p = lfs.getPath(); - String[] mobParts = p.getName().split("_"); - String regionName = mobParts[mobParts.length - 1]; - - if (!regionNames.contains(regionName)) { - // MOB belonged to a region no longer hosted - long creationTime = fs.getFileStatus(p).getModificationTime(); - if (creationTime < maxCreationTimeToArchive) { - LOG.trace("Archiving MOB file {} creation time={}", p, - (fs.getFileStatus(p).getModificationTime())); - toArchive.add(p); - } else { - LOG.trace("Skipping fresh file: {}. Creation time={}", p, - fs.getFileStatus(p).getModificationTime()); - } - } else { - LOG.trace("Keeping MOB file with existing region: {}", p); - } - } - LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(), - table, family); - archiveMobFiles(conf, table, family.getBytes(), toArchive); - LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, - family); - } - } - - /** - * Archives the mob files. - * @param conf The current configuration. - * @param tableName The table name. - * @param family The name of the column family. - * @param storeFiles The files to be archived. - * @throws IOException exception - */ - public void archiveMobFiles(Configuration conf, TableName tableName, byte[] family, - List storeFiles) throws IOException { - - if (storeFiles.size() == 0) { - // nothing to remove - LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}", tableName, - Bytes.toString(family)); - return; - } - Path mobTableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), tableName); - FileSystem fs = storeFiles.get(0).getFileSystem(conf); - - for (Path p : storeFiles) { - LOG.debug("MOB Cleaner is archiving: {}", p); - HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir, - family, p); - } - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java new file mode 100644 index 000000000000..175d7bf5a3b0 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mob; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.HFileArchiver; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap; + +@InterfaceAudience.Private +public class MobFileCleanupUtil { + + private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanupUtil.class); + + /** + * Performs housekeeping file cleaning (called by MOB Cleaner chore) + * @param conf configuration + * @param table table name + * @throws IOException exception + */ + public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Admin admin) + throws IOException { + long minAgeToArchive = + conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE); + // We check only those MOB files, which creation time is less + // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap + // gives us full confidence that all corresponding store files will + // exist at the time cleaning procedure begins and will be examined. + // So, if MOB file creation time is greater than this maxTimeToArchive, + // this will be skipped and won't be archived. + long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive; + TableDescriptor htd = admin.getDescriptor(table); + List list = MobUtils.getMobColumnFamilies(htd); + if (list.size() == 0) { + LOG.info("Skipping non-MOB table [{}]", table); + return; + } else { + LOG.info("Only MOB files whose creation time older than {} will be archived, table={}", + maxCreationTimeToArchive, table); + } + + FileSystem fs = FileSystem.get(conf); + Set regionNames = new HashSet<>(); + Path rootDir = CommonFSUtils.getRootDir(conf); + Path tableDir = CommonFSUtils.getTableDir(rootDir, table); + List regionDirs = FSUtils.getRegionDirs(fs, tableDir); + + Set allActiveMobFileName = new HashSet(); + for (Path regionPath : regionDirs) { + regionNames.add(regionPath.getName()); + for (ColumnFamilyDescriptor hcd : list) { + String family = hcd.getNameAsString(); + Path storePath = new Path(regionPath, family); + boolean succeed = false; + Set regionMobs = new HashSet(); + + while (!succeed) { + if (!fs.exists(storePath)) { + String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore" + + " execution, aborting MOB file cleaner chore.", storePath); + throw new IOException(errMsg); + } + RemoteIterator rit = fs.listLocatedStatus(storePath); + List storeFiles = new ArrayList(); + // Load list of store files first + while (rit.hasNext()) { + Path p = rit.next().getPath(); + if (fs.isFile(p)) { + storeFiles.add(p); + } + } + LOG.info("Found {} store files in: {}", storeFiles.size(), storePath); + Path currentPath = null; + try { + for (Path pp : storeFiles) { + currentPath = pp; + LOG.trace("Store file: {}", pp); + HStoreFile sf = null; + byte[] mobRefData = null; + byte[] bulkloadMarkerData = null; + try { + sf = new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true); + sf.initReader(); + mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS); + bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY); + // close store file to avoid memory leaks + sf.closeStoreFile(true); + } catch (IOException ex) { + // When FileBased SFT is active the store dir can contain corrupted or incomplete + // files. So read errors are expected. We just skip these files. + if (ex instanceof FileNotFoundException) { + throw ex; + } + LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex); + continue; + } + if (mobRefData == null) { + if (bulkloadMarkerData == null) { + LOG.warn("Found old store file with no MOB_FILE_REFS: {} - " + + "can not proceed until all old files will be MOB-compacted.", pp); + return; + } else { + LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp); + continue; + } + } + // file may or may not have MOB references, but was created by the distributed + // mob compaction code. + try { + SetMultimap mobs = + MobUtils.deserializeMobFileRefs(mobRefData).build(); + LOG.debug("Found {} mob references for store={}", mobs.size(), sf); + LOG.trace("Specific mob references found for store={} : {}", sf, mobs); + regionMobs.addAll(mobs.values()); + } catch (RuntimeException exception) { + throw new IOException("failure getting mob references for hfile " + sf, exception); + } + } + } catch (FileNotFoundException e) { + LOG.warn( + "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error", + currentPath, e); + regionMobs.clear(); + continue; + } + succeed = true; + } + + // Add MOB references for current region/family + allActiveMobFileName.addAll(regionMobs); + } // END column families + } // END regions + // Check if number of MOB files too big (over 1M) + if (allActiveMobFileName.size() > 1000000) { + LOG.warn("Found too many active MOB files: {}, table={}, " + + "this may result in high memory pressure.", allActiveMobFileName.size(), table); + } + LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table); + allActiveMobFileName.stream().forEach(LOG::trace); + + // Now scan MOB directories and find MOB files with no references to them + for (ColumnFamilyDescriptor hcd : list) { + List toArchive = new ArrayList(); + String family = hcd.getNameAsString(); + Path dir = MobUtils.getMobFamilyPath(conf, table, family); + RemoteIterator rit = fs.listLocatedStatus(dir); + while (rit.hasNext()) { + LocatedFileStatus lfs = rit.next(); + Path p = lfs.getPath(); + String[] mobParts = p.getName().split("_"); + String regionName = mobParts[mobParts.length - 1]; + + if (!regionNames.contains(regionName)) { + // MOB belonged to a region no longer hosted + long creationTime = fs.getFileStatus(p).getModificationTime(); + if (creationTime < maxCreationTimeToArchive) { + LOG.trace("Archiving MOB file {} creation time={}", p, + (fs.getFileStatus(p).getModificationTime())); + toArchive.add(p); + } else { + LOG.trace("Skipping fresh file: {}. Creation time={}", p, + fs.getFileStatus(p).getModificationTime()); + } + } else { + LOG.trace("Keeping MOB file with existing region: {}", p); + } + } + LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(), + table, family); + archiveMobFiles(conf, table, admin, family.getBytes(), toArchive); + LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, + family); + } + } + + /** + * Archives the mob files. + * @param conf The current configuration. + * @param tableName The table name. + * @param family The name of the column family. + * @param storeFiles The files to be archived. + * @throws IOException exception + */ + private static void archiveMobFiles(Configuration conf, TableName tableName, Admin admin, + byte[] family, List storeFiles) throws IOException { + + if (storeFiles.size() == 0) { + // nothing to remove + LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}", tableName, + Bytes.toString(family)); + return; + } + Path mobTableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), tableName); + FileSystem fs = storeFiles.get(0).getFileSystem(conf); + + for (Path p : storeFiles) { + LOG.debug("MOB Cleaner is archiving: {}", p); + HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir, + family, p); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java index 4bc9e386a8af..ec1a567591cd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java @@ -70,7 +70,6 @@ public class MobStressToolRunner { private long count = 500000; private double failureProb = 0.1; private Table table = null; - private MobFileCleanerChore chore; private static volatile boolean run = true; @@ -84,7 +83,6 @@ public void init(Configuration conf, long numRows) throws IOException { printConf(); Connection conn = ConnectionFactory.createConnection(this.conf); this.admin = conn.getAdmin(); - this.chore = new MobFileCleanerChore(admin); this.familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); this.tableDescriptor = @@ -157,9 +155,9 @@ class CleanMobAndArchive implements Runnable { public void run() { while (run) { try { - LOG.info("MOB cleanup chore started ..."); - chore.cleanupObsoleteMobFiles(conf, table.getName()); - LOG.info("MOB cleanup chore finished"); + LOG.info("MOB cleanup started ..."); + MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, table.getName(), admin); + LOG.info("MOB cleanup finished"); Thread.sleep(130000); } catch (Exception e) { @@ -228,7 +226,7 @@ public void runStressTest() throws InterruptedException, IOException { LOG.info("Waiting for write thread to finish ..."); writeData.join(); // Cleanup again - chore.cleanupObsoleteMobFiles(conf, table.getName()); + MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, table.getName(), admin); getNumberOfMobFiles(conf, table.getName(), new String(fam)); if (HTU != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java index 673943ac3215..3ad6585c4620 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionWithDefaults.java @@ -97,8 +97,6 @@ public class TestMobCompactionWithDefaults { protected int numRegions = 20; protected int rows = 1000; - protected MobFileCleanerChore cleanerChore; - protected Boolean useFileBasedSFT; public TestMobCompactionWithDefaults(Boolean useFileBasedSFT) { @@ -139,7 +137,6 @@ protected void additonalConfigSetup() { public void setUp() throws Exception { htuStart(); admin = HTU.getAdmin(); - cleanerChore = new MobFileCleanerChore(admin); familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); tableDescriptor = HTU.createModifyableTableDescriptor(TestMobUtils.getTableName(test)) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java index 705f637fa05e..7555e77ac026 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java @@ -92,7 +92,7 @@ public void setUp() throws Exception { HTU.startMiniCluster(); admin = HTU.getAdmin(); - chore = new MobFileCleanerChore(admin); + chore = new MobFileCleanerChore(HTU.getHBaseCluster().getMaster()); familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); tableDescriptor = HTU.createModifyableTableDescriptor("testMobCompactTable") @@ -168,7 +168,7 @@ public void testMobFileCleanerChore() throws InterruptedException, IOException { Thread.sleep(minAgeToArchive + 1000); LOG.info("Cleaning up MOB files"); // Cleanup - chore.cleanupObsoleteMobFiles(conf, table.getName()); + MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, table.getName(), admin); // verify that nothing have happened num = getNumberOfMobFiles(conf, table.getName(), new String(fam)); @@ -187,7 +187,7 @@ public void testMobFileCleanerChore() throws InterruptedException, IOException { Thread.sleep(minAgeToArchive + 1000); LOG.info("Cleaning up MOB files"); - chore.cleanupObsoleteMobFiles(conf, table.getName()); + MobFileCleanupUtil.cleanupObsoleteMobFiles(conf, table.getName(), admin); // check that the extra file got deleted num = getNumberOfMobFiles(conf, table.getName(), new String(fam)); From 12bfea2f3c16f4f901533e3066aaf6d9386ffb9b Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sun, 12 Nov 2023 14:41:23 -0500 Subject: [PATCH 5/6] Split part of MobFileCleanupUtil.cleanupObsoleteMobFiles() out for checkstyle rule --- .../hadoop/hbase/mob/MobFileCleanupUtil.java | 65 +++++++++++-------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java index 175d7bf5a3b0..049192624ef3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java @@ -47,10 +47,13 @@ import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap; @InterfaceAudience.Private -public class MobFileCleanupUtil { +public final class MobFileCleanupUtil { private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanupUtil.class); + private MobFileCleanupUtil() { + } + /** * Performs housekeeping file cleaning (called by MOB Cleaner chore) * @param conf configuration @@ -179,37 +182,43 @@ public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, // Now scan MOB directories and find MOB files with no references to them for (ColumnFamilyDescriptor hcd : list) { - List toArchive = new ArrayList(); - String family = hcd.getNameAsString(); - Path dir = MobUtils.getMobFamilyPath(conf, table, family); - RemoteIterator rit = fs.listLocatedStatus(dir); - while (rit.hasNext()) { - LocatedFileStatus lfs = rit.next(); - Path p = lfs.getPath(); - String[] mobParts = p.getName().split("_"); - String regionName = mobParts[mobParts.length - 1]; - - if (!regionNames.contains(regionName)) { - // MOB belonged to a region no longer hosted - long creationTime = fs.getFileStatus(p).getModificationTime(); - if (creationTime < maxCreationTimeToArchive) { - LOG.trace("Archiving MOB file {} creation time={}", p, - (fs.getFileStatus(p).getModificationTime())); - toArchive.add(p); - } else { - LOG.trace("Skipping fresh file: {}. Creation time={}", p, - fs.getFileStatus(p).getModificationTime()); - } + checkColumnFamilyDescriptor(conf, table, fs, admin, hcd, regionNames, + maxCreationTimeToArchive); + } + } + + private static void checkColumnFamilyDescriptor(Configuration conf, TableName table, + FileSystem fs, Admin admin, ColumnFamilyDescriptor hcd, Set regionNames, + long maxCreationTimeToArchive) throws IOException { + List toArchive = new ArrayList(); + String family = hcd.getNameAsString(); + Path dir = MobUtils.getMobFamilyPath(conf, table, family); + RemoteIterator rit = fs.listLocatedStatus(dir); + while (rit.hasNext()) { + LocatedFileStatus lfs = rit.next(); + Path p = lfs.getPath(); + String[] mobParts = p.getName().split("_"); + String regionName = mobParts[mobParts.length - 1]; + + if (!regionNames.contains(regionName)) { + // MOB belonged to a region no longer hosted + long creationTime = fs.getFileStatus(p).getModificationTime(); + if (creationTime < maxCreationTimeToArchive) { + LOG.trace("Archiving MOB file {} creation time={}", p, + (fs.getFileStatus(p).getModificationTime())); + toArchive.add(p); } else { - LOG.trace("Keeping MOB file with existing region: {}", p); + LOG.trace("Skipping fresh file: {}. Creation time={}", p, + fs.getFileStatus(p).getModificationTime()); } + } else { + LOG.trace("Keeping MOB file with existing region: {}", p); } - LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(), - table, family); - archiveMobFiles(conf, table, admin, family.getBytes(), toArchive); - LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, - family); } + LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(), + table, family); + archiveMobFiles(conf, table, admin, family.getBytes(), toArchive); + LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, family); } /** From 47bd4b5d55ec99a916011521ee88abafd7406b75 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sun, 12 Nov 2023 20:17:33 -0500 Subject: [PATCH 6/6] Rename TestMobFileCleanerChore as TestMobFileCleanupUtil --- ...leCleanerChore.java => TestMobFileCleanupUtil.java} | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) rename hbase-server/src/test/java/org/apache/hadoop/hbase/mob/{TestMobFileCleanerChore.java => TestMobFileCleanupUtil.java} (96%) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanupUtil.java similarity index 96% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanupUtil.java index 7555e77ac026..fc9eceb62412 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanupUtil.java @@ -57,11 +57,11 @@ * cleaner chore 7 Verifies that number of MOB files in a mob directory is 1. */ @Category(MediumTests.class) -public class TestMobFileCleanerChore { - private static final Logger LOG = LoggerFactory.getLogger(TestMobFileCleanerChore.class); +public class TestMobFileCleanupUtil { + private static final Logger LOG = LoggerFactory.getLogger(TestMobFileCleanupUtil.class); @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMobFileCleanerChore.class); + HBaseClassTestRule.forClass(TestMobFileCleanupUtil.class); private HBaseTestingUtil HTU; @@ -77,10 +77,9 @@ public class TestMobFileCleanerChore { private ColumnFamilyDescriptor familyDescriptor; private Admin admin; private Table table = null; - private MobFileCleanerChore chore; private long minAgeToArchive = 10000; - public TestMobFileCleanerChore() { + public TestMobFileCleanupUtil() { } @Before @@ -92,7 +91,6 @@ public void setUp() throws Exception { HTU.startMiniCluster(); admin = HTU.getAdmin(); - chore = new MobFileCleanerChore(HTU.getHBaseCluster().getMaster()); familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true) .setMobThreshold(mobLen).setMaxVersions(1).build(); tableDescriptor = HTU.createModifyableTableDescriptor("testMobCompactTable")