HBASE-27826 Refactor code to move creation of Ref files to SFT interface apis

Author: Prathyusha Garre

hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java

@@ -456,7 +456,7 @@ public static String createFromHFileLink(final Configuration conf, final FileSys
    * Create the back reference name
    */
   // package-private for testing
-  static String createBackReferenceName(final String tableNameStr, final String regionName) {
+  public static String createBackReferenceName(final String tableNameStr, final String regionName) {
 
     return regionName + "." + tableNameStr.replace(TableName.NAMESPACE_DELIM, '=');
   }

hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java

@@ -195,7 +195,7 @@ public static Reference convert(final FSProtos.Reference r) {
    * delimiter, pb reads to EOF which may not be what you want).
    * @return This instance serialized as a delimited protobuf w/ a magic pb prefix.
    */
-  byte[] toByteArray() throws IOException {
+  public byte[] toByteArray() throws IOException {
     return ProtobufUtil.prependPBMagic(convert().toByteArray());
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java

@@ -612,7 +612,7 @@ private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg
           // to read the hfiles.
           storeFileInfo.setConf(storeConfiguration);
           Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
-            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
+            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), tracker);
           mergedFiles.add(refFile);
         }
       }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java

@@ -666,8 +666,9 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
     // table dir. In case of failure, the proc would go through this again, already existing
     // region dirs and split files would just be ignored, new split files should get created.
     int nbFiles = 0;
-    final Map<String, Collection<StoreFileInfo>> files =
-      new HashMap<String, Collection<StoreFileInfo>>(htd.getColumnFamilyCount());
+    final Map<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>> files =
+      new HashMap<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>>(
+        htd.getColumnFamilyCount());
     for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
       String family = cfd.getNameAsString();
       StoreFileTracker tracker =
@@ -690,7 +691,7 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
         }
         if (filteredSfis == null) {
           filteredSfis = new ArrayList<StoreFileInfo>(sfis.size());
-          files.put(family, filteredSfis);
+          files.put(family, new Pair(filteredSfis, tracker));
         }
         filteredSfis.add(sfi);
         nbFiles++;
@@ -713,10 +714,11 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
     final List<Future<Pair<Path, Path>>> futures = new ArrayList<Future<Pair<Path, Path>>>(nbFiles);
 
     // Split each store file.
-    for (Map.Entry<String, Collection<StoreFileInfo>> e : files.entrySet()) {
+    for (Map.Entry<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>> e : files
+      .entrySet()) {
       byte[] familyName = Bytes.toBytes(e.getKey());
       final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName);
-      final Collection<StoreFileInfo> storeFiles = e.getValue();
+      final Collection<StoreFileInfo> storeFiles = e.getValue().getFirst();
       if (storeFiles != null && storeFiles.size() > 0) {
         final Configuration storeConfiguration =
           StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
@@ -727,8 +729,9 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
           // is running in a regionserver's Store context, or we might not be able
           // to read the hfiles.
           storeFileInfo.setConf(storeConfiguration);
-          StoreFileSplitter sfs = new StoreFileSplitter(regionFs, familyName,
-            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
+          StoreFileSplitter sfs =
+            new StoreFileSplitter(regionFs, e.getValue().getSecond(), familyName,
+              new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
           futures.add(threadPool.submit(sfs));
         }
       }
@@ -794,19 +797,19 @@ private void assertSplitResultFilesCount(final FileSystem fs,
     }
   }
 
-  private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
-    throws IOException {
+  private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, StoreFileTracker tracker,
+    byte[] family, HStoreFile sf) throws IOException {
     if (LOG.isDebugEnabled()) {
       LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath()
         + " for region: " + getParentRegion().getShortNameToLog());
     }
 
     final byte[] splitRow = getSplitRow();
     final String familyName = Bytes.toString(family);
-    final Path path_first =
-      regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, false, splitPolicy);
-    final Path path_second =
-      regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, true, splitPolicy);
+    final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
+      false, splitPolicy, tracker);
+    final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
+      true, splitPolicy, tracker);
     if (LOG.isDebugEnabled()) {
       LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath()
         + " for region: " + getParentRegion().getShortNameToLog());
@@ -822,22 +825,25 @@ private class StoreFileSplitter implements Callable<Pair<Path, Path>> {
     private final HRegionFileSystem regionFs;
     private final byte[] family;
     private final HStoreFile sf;
+    private final StoreFileTracker tracker;
 
     /**
      * Constructor that takes what it needs to split
      * @param regionFs the file system
      * @param family   Family that contains the store file
      * @param sf       which file
      */
-    public StoreFileSplitter(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) {
+    public StoreFileSplitter(HRegionFileSystem regionFs, StoreFileTracker tracker, byte[] family,
+      HStoreFile sf) {
       this.regionFs = regionFs;
       this.sf = sf;
       this.family = family;
+      this.tracker = tracker;
     }
 
     @Override
     public Pair<Path, Path> call() throws IOException {
-      return splitStoreFile(regionFs, family, sf);
+      return splitStoreFile(regionFs, tracker, family, sf);
     }
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java

@@ -35,6 +35,7 @@
 import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.ScheduledChore;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Get;
@@ -50,6 +51,8 @@
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CommonFSUtils;
 import org.apache.hadoop.hbase.util.Pair;
@@ -422,7 +425,7 @@ private static Pair<Boolean, Boolean> checkRegionReferences(MasterServices servi
     try {
       HRegionFileSystem regionFs = HRegionFileSystem
         .openRegionFromFileSystem(services.getConfiguration(), fs, tabledir, region, true);
-      boolean references = regionFs.hasReferences(tableDescriptor);
+      boolean references = hasReferences(services.getConfiguration(), regionFs, tableDescriptor);
       return new Pair<>(Boolean.TRUE, references);
     } catch (IOException e) {
       LOG.error("Error trying to determine if region {} has references, assuming it does",
@@ -431,6 +434,17 @@ private static Pair<Boolean, Boolean> checkRegionReferences(MasterServices servi
     }
   }
 
+  private static boolean hasReferences(Configuration conf, HRegionFileSystem regionFs,
+    TableDescriptor htd) throws IOException {
+    for (ColumnFamilyDescriptor family : htd.getColumnFamilies()) {
+      StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, family, regionFs, false);
+      if (sft.hasReferences(family.getNameAsString())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   private void updateAssignmentManagerMetrics() {
     services.getAssignmentManager().getAssignmentManagerMetrics()
       .updateHoles(lastReport.getHoles().size());

hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.regionserver.HStoreFile;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
@@ -41,10 +42,12 @@ public CachedMobFile(HStoreFile sf) {
   }
 
   public static CachedMobFile create(FileSystem fs, Path path, Configuration conf,
-    CacheConfig cacheConf) throws IOException {
-    // XXX: primaryReplica is only used for constructing the key of block cache so it is not a
-    // critical problem if we pass the wrong value, so here we always pass true. Need to fix later.
-    HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
+    CacheConfig cacheConf, StoreFileTracker sft) throws IOException {
+    // XXX: primaryReplica is only used for constructing the key of block cache so
+    // it is not a
+    // critical problem if we pass the wrong value, so here we always pass true.
+    // Need to fix later.
+    HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft);
     return new CachedMobFile(sf);
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java

@@ -56,17 +56,17 @@ public class ExpiredMobFileCleaner extends Configured implements Tool {
    * @param tableName The current table name.
    * @param family    The current family.
    */
-  public void cleanExpiredMobFiles(String tableName, ColumnFamilyDescriptor family)
+  public void cleanExpiredMobFiles(TableDescriptor htd, ColumnFamilyDescriptor family)
     throws IOException {
     Configuration conf = getConf();
-    TableName tn = TableName.valueOf(tableName);
+    String tableName = htd.getTableName().getNameAsString();
     FileSystem fs = FileSystem.get(conf);
     LOG.info("Cleaning the expired MOB files of " + family.getNameAsString() + " in " + tableName);
     // disable the block cache.
     Configuration copyOfConf = new Configuration(conf);
     copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
     CacheConfig cacheConfig = new CacheConfig(copyOfConf);
-    MobUtils.cleanExpiredMobFiles(fs, conf, tn, family, cacheConfig,
+    MobUtils.cleanExpiredMobFiles(fs, conf, htd, family, cacheConfig,
       EnvironmentEdgeManager.currentTime());
   }
 
@@ -105,7 +105,7 @@ public int run(String[] args) throws Exception {
         throw new IOException(
           "The minVersions of the column family is not 0, could not be handled by this cleaner");
       }
-      cleanExpiredMobFiles(tableName, family);
+      cleanExpiredMobFiles(htd, family);
       return 0;
     } finally {
       admin.close();

hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java

@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.regionserver.HStoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
@@ -133,11 +134,13 @@ public void close() throws IOException {
    * @param cacheConf The CacheConfig.
    * @return An instance of the MobFile.
    */
-  public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf)
-    throws IOException {
-    // XXX: primaryReplica is only used for constructing the key of block cache so it is not a
-    // critical problem if we pass the wrong value, so here we always pass true. Need to fix later.
-    HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
+  public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf,
+    StoreFileTracker sft) throws IOException {
+    // XXX: primaryReplica is only used for constructing the key of block cache so
+    // it is not a
+    // critical problem if we pass the wrong value, so here we always pass true.
+    // Need to fix later.
+    HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft);
     return new MobFile(sf);
   }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java

@@ -33,6 +33,9 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.regionserver.StoreContext;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
 import org.apache.hadoop.hbase.util.IdLock;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
@@ -198,9 +201,11 @@ public void evictFile(String fileName) {
    * @param cacheConf The current MobCacheConfig
    * @return A opened mob file.
    */
-  public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
+  public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf,
+    StoreContext storeContext) throws IOException {
+    StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext);
     if (!isCacheEnabled) {
-      MobFile mobFile = MobFile.create(fs, path, conf, cacheConf);
+      MobFile mobFile = MobFile.create(fs, path, conf, cacheConf, sft);
       mobFile.open();
       return mobFile;
     } else {
@@ -214,7 +219,7 @@ public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws
             if (map.size() > mobFileMaxCacheSize) {
               evict();
             }
-            cached = CachedMobFile.create(fs, path, conf, cacheConf);
+            cached = CachedMobFile.create(fs, path, conf, cacheConf, sft);
             cached.open();
             map.put(fileName, cached);
             miss.increment();

hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java

@@ -87,7 +87,7 @@ protected void chore() {
       for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
         if (hcd.isMobEnabled() && hcd.getMinVersions() == 0) {
           try {
-            cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd);
+            cleaner.cleanExpiredMobFiles(htd, hcd);
           } catch (IOException e) {
             LOG.error("Failed to clean the expired mob files table={} family={}",
               htd.getTableName().getNameAsString(), hcd.getNameAsString(), e);