Skip to content

HBASE-28969 Move HFileLink file creations to SFT (#6459) #7377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: branch-2.6
Choose a base branch
from
+364 −203
Draft

HBASE-28969 Move HFileLink file creations to SFT (#6459) #7377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 0 additions & 170 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
Expand Down Expand Up @@ -290,175 +289,6 @@ public static String createHFileLinkName(final TableName tableName, final String
return s;
}

/**
* Create a new HFileLink
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param hfileRegionInfo - Linked HFile Region Info
* @param hfileName - Linked HFile name
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
public static String create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName)
throws IOException {
return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true);
}

/**
* Create a new HFileLink
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param hfileRegionInfo - Linked HFile Region Info
* @param hfileName - Linked HFile name
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
public static String create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName,
final boolean createBackRef) throws IOException {
TableName linkedTable = hfileRegionInfo.getTable();
String linkedRegion = hfileRegionInfo.getEncodedName();
return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef);
}

/**
* Create a new HFileLink
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param linkedTable - Linked Table Name
* @param linkedRegion - Linked Region Name
* @param hfileName - Linked HFile name
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
public static String create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
final String hfileName) throws IOException {
return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true);
}

/**
* Create a new HFileLink. In the event of link creation failure, this method throws an
* IOException, so that the calling upper laying can decide on how to proceed with this.
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param linkedTable - Linked Table Name
* @param linkedRegion - Linked Region Name
* @param hfileName - Linked HFile name
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
public static String create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
final String hfileName, final boolean createBackRef) throws IOException {
String familyName = dstFamilyPath.getName();
String regionName = dstFamilyPath.getParent().getName();
String tableName =
CommonFSUtils.getTableName(dstFamilyPath.getParent().getParent()).getNameAsString();

return create(conf, fs, dstFamilyPath, familyName, tableName, regionName, linkedTable,
linkedRegion, hfileName, createBackRef);
}

/**
* Create a new HFileLink
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param dstTableName - Destination table name
* @param dstRegionName - Destination region name
* @param linkedTable - Linked Table Name
* @param linkedRegion - Linked Region Name
* @param hfileName - Linked HFile name
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return the file link name.
* @throws IOException on file or parent directory creation failure
*/
public static String create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final String familyName, final String dstTableName,
final String dstRegionName, final TableName linkedTable, final String linkedRegion,
final String hfileName, final boolean createBackRef) throws IOException {
String name = createHFileLinkName(linkedTable, linkedRegion, hfileName);
String refName = createBackReferenceName(dstTableName, dstRegionName);

// Make sure the destination directory exists
fs.mkdirs(dstFamilyPath);

// Make sure the FileLink reference directory exists
Path archiveStoreDir =
HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion, familyName);
Path backRefPath = null;
if (createBackRef) {
Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
fs.mkdirs(backRefssDir);

// Create the reference for the link
backRefPath = new Path(backRefssDir, refName);
fs.createNewFile(backRefPath);
}
try {
// Create the link
if (fs.createNewFile(new Path(dstFamilyPath, name))) {
return name;
}
} catch (IOException e) {
LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
// Revert the reference if the link creation failed
if (createBackRef) {
fs.delete(backRefPath, false);
}
throw e;
}
throw new IOException(
"File link=" + name + " already exists under " + dstFamilyPath + " folder.");
}

/**
* Create a new HFileLink starting from a hfileLink name
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param hfileLinkName - HFileLink name (it contains hfile-region-table)
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
public static String createFromHFileLink(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
throws IOException {
Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
if (!m.matches()) {
throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
}
return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)), m.group(3),
m.group(4), createBackRef);
}

/**
* Create the back reference name
*/
Expand Down
40 changes: 26 additions & 14 deletions ...er/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -751,7 +751,8 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
for (Map.Entry<String, Collection<StoreFileInfo>> e : files.entrySet()) {
byte[] familyName = Bytes.toBytes(e.getKey());
final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName);
final Collection<StoreFileInfo> storeFiles = e.getValue();
Collection<StoreFileInfo> storeFileInfos = e.getValue();
final Collection<StoreFileInfo> storeFiles = storeFileInfos;
if (storeFiles != null && storeFiles.size() > 0) {
final Configuration storeConfiguration =
StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
Expand All @@ -762,7 +763,7 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
// is running in a regionserver's Store context, or we might not be able
// to read the hfiles.
storeFileInfo.setConf(storeConfiguration);
StoreFileSplitter sfs = new StoreFileSplitter(regionFs, familyName,
StoreFileSplitter sfs = new StoreFileSplitter(regionFs, htd, hcd,
new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
futures.add(threadPool.submit(sfs));
}
Expand Down Expand Up @@ -829,19 +830,27 @@ private void assertSplitResultFilesCount(final FileSystem fs,
}
}

private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
throws IOException {
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, TableDescriptor htd,
ColumnFamilyDescriptor hcd, HStoreFile sf) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath()
+ " for region: " + getParentRegion().getShortNameToLog());
}

final byte[] splitRow = getSplitRow();
final String familyName = Bytes.toString(family);
final Path path_first =
regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, false, splitPolicy);
final Path path_second =
regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, true, splitPolicy);
final String familyName = hcd.getNameAsString();
StoreFileTracker daughterOneSft =
StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), htd, hcd,
HRegionFileSystem.create(regionFs.getFileSystem().getConf(), regionFs.getFileSystem(),
regionFs.getTableDir(), daughterOneRI));
StoreFileTracker daughterTwoSft =
StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), htd, hcd,
HRegionFileSystem.create(regionFs.getFileSystem().getConf(), regionFs.getFileSystem(),
regionFs.getTableDir(), daughterTwoRI));
final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
false, splitPolicy, daughterOneSft);
final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
true, splitPolicy, daughterTwoSft);
if (LOG.isDebugEnabled()) {
LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath()
+ " for region: " + getParentRegion().getShortNameToLog());
Expand All @@ -855,24 +864,27 @@ private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] famil
*/
private class StoreFileSplitter implements Callable<Pair<Path, Path>> {
private final HRegionFileSystem regionFs;
private final byte[] family;
private final ColumnFamilyDescriptor hcd;
private final HStoreFile sf;
private final TableDescriptor htd;

/**
* Constructor that takes what it needs to split
* @param regionFs the file system
* @param family Family that contains the store file
* @param hcd Family that contains the store file
* @param sf which file
*/
public StoreFileSplitter(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) {
public StoreFileSplitter(HRegionFileSystem regionFs, TableDescriptor htd,
ColumnFamilyDescriptor hcd, HStoreFile sf) {
this.regionFs = regionFs;
this.sf = sf;
this.family = family;
this.hcd = hcd;
this.htd = htd;
}

@Override
public Pair<Path, Path> call() throws IOException {
return splitStoreFile(regionFs, family, sf);
return splitStoreFile(regionFs, htd, hcd, sf);
}
}

Expand Down
3 changes: 1 addition & 2 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,8 +742,7 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte
hfileName = m.group(4);
}
// must create back reference here
HFileLink.create(conf, fs, splitDir, familyName, hri.getTable().getNameAsString(),
hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true);
tracker.createHFileLink(linkedTable, linkedRegion, hfileName, true);
Path path =
new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName));
LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName()
Expand Down
54 changes: 54 additions & 0 deletions ...src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
import java.io.IOException;
import java.util.Collection;
import java.util.List;
<<<<<<< HEAD
=======
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
>>>>>>> a680795f71 (HBASE-28969 Move HFileLink file creations to SFT (#6459))
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
Expand Down Expand Up @@ -94,4 +100,52 @@ void replace(Collection<StoreFileInfo> compactedFiles, Collection<StoreFileInfo>
* does not allow broken store files under the actual data directory.
*/
boolean requireWritingToTmpDirFirst();

Reference createReference(Reference reference, Path path) throws IOException;

/**
* Reads the reference file from the given path.
* @param path the {@link Path} to the reference file in the file system.
* @return a {@link Reference} that points at top/bottom half of a an hfile
*/
Reference readReference(Path path) throws IOException;

/**
* Returns true if the specified family has reference files
* @return true if family contains reference files
*/
boolean hasReferences() throws IOException;

StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPath,
final boolean primaryReplica) throws IOException;

StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryReplica)
throws IOException;

/**
* Create a new HFileLink
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param hfileLinkName - HFileLink name (it contains hfile-region-table)
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
String createHFileLink(final TableName linkedTable, final String linkedRegion,
final String hfileName, final boolean createBackRef) throws IOException;

/**
* Create a new HFileLink starting from a hfileLink name
* <p>
* It also adds a back-reference to the hfile back-reference directory to simplify the
* reference-count and the cleaning process.
* @param hfileLinkName - HFileLink name (it contains hfile-region-table)
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return the file link name.
* @throws IOException on file or parent directory creation failure.
*/
String createFromHFileLink(final String hfileName, final boolean createBackRef)
throws IOException;

}
Loading