Skip to content

Commit c39ad64

Browse files
author
Wellington Chevreuil
committed
Addressing latest review suggestions:
1) Changes to load files to split/merge from tracker, instead of listing from FS; 2) UT for above; 3) Changed tracker factory to load traker impl from CF first, then Table, then Configuration; Change-Id: I3a09afd28afe5fe6297b28dafa5dec0eca4c323c
1 parent 888f140 commit c39ad64

File tree

8 files changed

+217
-39
lines changed

8 files changed

+217
-39
lines changed

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,16 @@
2424
import java.util.Collections;
2525
import java.util.List;
2626
import java.util.stream.Stream;
27+
28+
import org.apache.hadoop.conf.Configuration;
2729
import org.apache.hadoop.fs.FileSystem;
2830
import org.apache.hadoop.fs.Path;
2931
import org.apache.hadoop.hbase.MetaMutationAnnotation;
3032
import org.apache.hadoop.hbase.ServerName;
3133
import org.apache.hadoop.hbase.TableName;
3234
import org.apache.hadoop.hbase.UnknownRegionException;
3335
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
36+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
3437
import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
3538
import org.apache.hadoop.hbase.client.MasterSwitchType;
3639
import org.apache.hadoop.hbase.client.Mutation;
@@ -52,7 +55,10 @@
5255
import org.apache.hadoop.hbase.quotas.QuotaExceededException;
5356
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
5457
import org.apache.hadoop.hbase.regionserver.HStoreFile;
58+
import org.apache.hadoop.hbase.regionserver.StoreContext;
5559
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
60+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
61+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
5662
import org.apache.hadoop.hbase.util.Bytes;
5763
import org.apache.hadoop.hbase.util.CommonFSUtils;
5864
import org.apache.hadoop.hbase.wal.WALSplitUtil;
@@ -594,7 +600,7 @@ private void createMergedRegion(final MasterProcedureEnv env) throws IOException
594600
mergedFiles.addAll(mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion));
595601
}
596602
assert mergeRegionFs != null;
597-
mergeRegionFs.commitMergedRegion(mergedFiles);
603+
mergeRegionFs.commitMergedRegion(mergedFiles, env);
598604

599605
// Prepare to create merged regions
600606
env.getAssignmentManager().getRegionStates().
@@ -608,7 +614,11 @@ private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg
608614
List<Path> mergedFiles = new ArrayList<>();
609615
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
610616
String family = hcd.getNameAsString();
611-
final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
617+
Configuration trackerConfig =
618+
StoreFileTrackerFactory.mergeConfigurations(env.getMasterConfiguration(), htd, hcd);
619+
StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig,
620+
mergedRegion.getTable(), true, family, regionFs);
621+
final Collection<StoreFileInfo> storeFiles = tracker.load();
612622
if (storeFiles != null && storeFiles.size() > 0) {
613623
for (StoreFileInfo storeFileInfo : storeFiles) {
614624
// Create reference file(s) to parent region file here in mergedDir.

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@
6464
import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
6565
import org.apache.hadoop.hbase.regionserver.RegionSplitRestriction;
6666
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
67+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
68+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
6769
import org.apache.hadoop.hbase.util.Bytes;
6870
import org.apache.hadoop.hbase.util.CommonFSUtils;
6971
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -624,13 +626,13 @@ public void createDaughterRegions(final MasterProcedureEnv env) throws IOExcepti
624626

625627
assertReferenceFileCount(fs, expectedReferences.getFirst().size(),
626628
regionFs.getSplitsDir(daughterOneRI));
627-
regionFs.commitDaughterRegion(daughterOneRI, expectedReferences.getFirst());
629+
regionFs.commitDaughterRegion(daughterOneRI, expectedReferences.getFirst(), env);
628630
assertReferenceFileCount(fs, expectedReferences.getFirst().size(),
629631
new Path(tabledir, daughterOneRI.getEncodedName()));
630632

631633
assertReferenceFileCount(fs, expectedReferences.getSecond().size(),
632634
regionFs.getSplitsDir(daughterTwoRI));
633-
regionFs.commitDaughterRegion(daughterTwoRI, expectedReferences.getSecond());
635+
regionFs.commitDaughterRegion(daughterTwoRI, expectedReferences.getSecond(), env);
634636
assertReferenceFileCount(fs, expectedReferences.getSecond().size(),
635637
new Path(tabledir, daughterTwoRI.getEncodedName()));
636638
}
@@ -664,7 +666,11 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
664666
new HashMap<String, Collection<StoreFileInfo>>(htd.getColumnFamilyCount());
665667
for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
666668
String family = cfd.getNameAsString();
667-
Collection<StoreFileInfo> sfis = regionFs.getStoreFiles(family);
669+
Configuration trackerConfig = StoreFileTrackerFactory.
670+
mergeConfigurations(env.getMasterConfiguration(), htd, htd.getColumnFamily(cfd.getName()));
671+
StoreFileTracker tracker = StoreFileTrackerFactory.create(trackerConfig, htd.getTableName(),
672+
true, family, regionFs);
673+
Collection<StoreFileInfo> sfis = tracker.load();
668674
if (sfis == null) {
669675
continue;
670676
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,15 @@
4141
import org.apache.hadoop.hbase.Cell;
4242
import org.apache.hadoop.hbase.HConstants;
4343
import org.apache.hadoop.hbase.PrivateCellUtil;
44+
import org.apache.hadoop.hbase.TableName;
4445
import org.apache.hadoop.hbase.backup.HFileArchiver;
4546
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
4647
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
4748
import org.apache.hadoop.hbase.client.RegionInfo;
4849
import org.apache.hadoop.hbase.client.TableDescriptor;
4950
import org.apache.hadoop.hbase.fs.HFileSystem;
5051
import org.apache.hadoop.hbase.io.Reference;
52+
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
5153
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
5254
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
5355
import org.apache.hadoop.hbase.util.Bytes;
@@ -597,31 +599,36 @@ void cleanupDaughterRegion(final RegionInfo regionInfo) throws IOException {
597599
* @param regionInfo daughter {@link org.apache.hadoop.hbase.client.RegionInfo}
598600
* @throws IOException
599601
*/
600-
public Path commitDaughterRegion(final RegionInfo regionInfo, List<Path> allRegionFiles)
601-
throws IOException {
602+
public Path commitDaughterRegion(final RegionInfo regionInfo, List<Path> allRegionFiles,
603+
MasterProcedureEnv env) throws IOException {
602604
Path regionDir = this.getSplitsDir(regionInfo);
603605
if (fs.exists(regionDir)) {
604606
// Write HRI to a file in case we need to recover hbase:meta
605607
Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE);
606608
byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
607609
writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
608-
loadRegionFilesIntoStoreTracker(allRegionFiles);
610+
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
611+
env.getMasterConfiguration(), fs, getTableDir(), regionInfo, false);
612+
loadRegionFilesIntoStoreTracker(allRegionFiles, env, regionFs);
609613
}
610614
return regionDir;
611615
}
612616

613-
private void loadRegionFilesIntoStoreTracker(List<Path> allFiles) throws IOException {
617+
private void loadRegionFilesIntoStoreTracker(List<Path> allFiles, MasterProcedureEnv env,
618+
HRegionFileSystem regionFs) throws IOException {
619+
TableDescriptor tblDesc = env.getMasterServices().getTableDescriptors().
620+
get(regionInfo.getTable());
614621
//we need to map trackers per store
615622
Map<String, StoreFileTracker> trackerMap = new HashMap<>();
616623
//we need to map store files per store
617624
Map<String, List<StoreFileInfo>> fileInfoMap = new HashMap<>();
618625
for(Path file : allFiles) {
619626
String familyName = file.getParent().getName();
620627
trackerMap.computeIfAbsent(familyName, t -> {
621-
ColumnFamilyDescriptorBuilder fDescBuilder =
622-
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(familyName));
623-
return StoreFileTrackerFactory.create(conf, regionInfo.getTable(), true,
624-
StoreContext.getBuilder().withColumnFamilyDescriptor(fDescBuilder.build()).build());
628+
Configuration config = StoreFileTrackerFactory.mergeConfigurations(conf, tblDesc,
629+
tblDesc.getColumnFamily(Bytes.toBytes(familyName)));
630+
return StoreFileTrackerFactory.
631+
create(config, regionInfo.getTable(), true, familyName, regionFs);
625632
});
626633
fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>());
627634
List<StoreFileInfo> infos = fileInfoMap.get(familyName);
@@ -782,14 +789,16 @@ public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFi
782789
* Commit a merged region, making it ready for use.
783790
* @throws IOException
784791
*/
785-
public void commitMergedRegion(List<Path> allMergedFiles) throws IOException {
792+
public void commitMergedRegion(List<Path> allMergedFiles, MasterProcedureEnv env)
793+
throws IOException {
786794
Path regionDir = getMergesDir(regionInfoForFs);
795+
TableName tableName = TableName.valueOf(regionDir.getParent().getName());
787796
if (regionDir != null && fs.exists(regionDir)) {
788797
// Write HRI to a file in case we need to recover hbase:meta
789798
Path regionInfoFile = new Path(regionDir, REGION_INFO_FILE);
790799
byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
791800
writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
792-
loadRegionFilesIntoStoreTracker(allMergedFiles);
801+
loadRegionFilesIntoStoreTracker(allMergedFiles, env, this);
793802
}
794803
}
795804

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,18 @@
1717
*/
1818
package org.apache.hadoop.hbase.regionserver.storefiletracker;
1919

20-
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker.STORE_FILE_TRACKER;
20+
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker.
21+
STORE_FILE_TRACKER;
22+
import org.apache.commons.lang3.StringUtils;
2123
import org.apache.hadoop.conf.Configuration;
2224
import org.apache.hadoop.hbase.TableName;
25+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
26+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
27+
import org.apache.hadoop.hbase.client.RegionInfo;
28+
import org.apache.hadoop.hbase.client.TableDescriptor;
29+
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
2330
import org.apache.hadoop.hbase.regionserver.StoreContext;
31+
import org.apache.hadoop.hbase.util.Bytes;
2432
import org.apache.hadoop.hbase.util.ReflectionUtils;
2533
import org.apache.yetus.audience.InterfaceAudience;
2634
import org.slf4j.Logger;
@@ -47,4 +55,25 @@ public static StoreFileTracker create(Configuration conf, TableName tableName,
4755
throw new RuntimeException(e);
4856
}
4957
}
58+
59+
public static StoreFileTracker create(Configuration conf, TableName tableName,
60+
boolean isPrimaryReplica, String family, HRegionFileSystem regionFs) {
61+
ColumnFamilyDescriptorBuilder fDescBuilder =
62+
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family));
63+
StoreContext ctx = StoreContext.getBuilder().
64+
withColumnFamilyDescriptor(fDescBuilder.build()).
65+
withRegionFileSystem(regionFs).
66+
build();
67+
return StoreFileTrackerFactory.create(conf, tableName, isPrimaryReplica, ctx);
68+
}
69+
70+
public static Configuration mergeConfigurations(Configuration global,
71+
TableDescriptor table, ColumnFamilyDescriptor family) {
72+
if(!StringUtils.isEmpty(family.getConfigurationValue(STORE_FILE_TRACKER))){
73+
global.set(STORE_FILE_TRACKER, family.getConfigurationValue(STORE_FILE_TRACKER));
74+
} else if(!StringUtils.isEmpty(table.getValue(STORE_FILE_TRACKER))) {
75+
global.set(STORE_FILE_TRACKER, table.getValue(STORE_FILE_TRACKER));
76+
}
77+
return global;
78+
}
5079
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
3434
import org.apache.hadoop.hbase.client.Table;
3535
import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure;
36+
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
3637
import org.apache.hadoop.hbase.procedure2.Procedure;
3738
import org.apache.hadoop.hbase.testclassification.LargeTests;
3839
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
@@ -140,7 +141,9 @@ public void testCommitDaughterRegionNoFiles() throws Exception {
140141
setRegionId(region.getRegionInfo().getRegionId() +
141142
EnvironmentEdgeManager.currentTime()).build();
142143
Path splitDir = regionFS.getSplitsDir(daughterA);
143-
Path result = regionFS.commitDaughterRegion(daughterA, new ArrayList<>());
144+
MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().
145+
getMasterProcedureExecutor().getEnvironment();
146+
Path result = regionFS.commitDaughterRegion(daughterA, new ArrayList<>(), env);
144147
assertEquals(splitDir, result);
145148
}
146149

@@ -171,8 +174,10 @@ public void testCommitDaughterRegionWithFiles() throws Exception {
171174
filesB.add(regionFS
172175
.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file,
173176
Bytes.toBytes("002"), true, region.getSplitPolicy()));
174-
Path resultA = regionFS.commitDaughterRegion(daughterA, filesA);
175-
Path resultB = regionFS.commitDaughterRegion(daughterB, filesB);
177+
MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().
178+
getMasterProcedureExecutor().getEnvironment();
179+
Path resultA = regionFS.commitDaughterRegion(daughterA, filesA, env);
180+
Path resultB = regionFS.commitDaughterRegion(daughterB, filesB, env);
176181
assertEquals(splitDirA, resultA);
177182
assertEquals(splitDirB, resultB);
178183
}
@@ -208,7 +213,9 @@ public void testCommitMergedRegion() throws Exception {
208213
file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
209214
List<Path> mergedFiles = new ArrayList<>();
210215
mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file));
211-
mergeRegionFs.commitMergedRegion(mergedFiles);
216+
MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().
217+
getMasterProcedureExecutor().getEnvironment();
218+
mergeRegionFs.commitMergedRegion(mergedFiles, env);
212219
}
213220

214221
private void waitForSplitProcComplete(int attempts, int waitTime) throws Exception {

‎hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import static org.junit.Assert.assertNull;
2525
import static org.junit.Assert.assertTrue;
2626
import static org.junit.Assert.fail;
27+
import static org.mockito.ArgumentMatchers.any;
2728
import static org.mockito.Mockito.mock;
2829
import static org.mockito.Mockito.when;
2930

@@ -49,12 +50,14 @@
4950
import org.apache.hadoop.hbase.KeyValue;
5051
import org.apache.hadoop.hbase.KeyValueUtil;
5152
import org.apache.hadoop.hbase.PrivateCellUtil;
53+
import org.apache.hadoop.hbase.TableDescriptors;
5254
import org.apache.hadoop.hbase.TableName;
5355
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
5456
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
5557
import org.apache.hadoop.hbase.client.RegionInfo;
5658
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
5759
import org.apache.hadoop.hbase.client.Scan;
60+
import org.apache.hadoop.hbase.client.TableDescriptor;
5861
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
5962
import org.apache.hadoop.hbase.io.HFileLink;
6063
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
@@ -69,6 +72,8 @@
6972
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
7073
import org.apache.hadoop.hbase.io.hfile.ReaderContext;
7174
import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder;
75+
import org.apache.hadoop.hbase.master.MasterServices;
76+
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
7277
import org.apache.hadoop.hbase.testclassification.MediumTests;
7378
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
7479
import org.apache.hadoop.hbase.util.BloomFilterFactory;
@@ -1057,7 +1062,17 @@ private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo h
10571062
}
10581063
List<Path> splitFiles = new ArrayList<>();
10591064
splitFiles.add(path);
1060-
Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles);
1065+
MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class);
1066+
MasterServices mockServices = mock(MasterServices.class);
1067+
when(mockEnv.getMasterServices()).thenReturn(mockServices);
1068+
when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration());
1069+
TableDescriptors mockTblDescs = mock(TableDescriptors.class);
1070+
when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs);
1071+
TableDescriptor mockTblDesc = mock(TableDescriptor.class);
1072+
when(mockTblDescs.get(any())).thenReturn(mockTblDesc);
1073+
ColumnFamilyDescriptor mockCfDesc = mock(ColumnFamilyDescriptor.class);
1074+
when(mockTblDesc.getColumnFamily(any())).thenReturn(mockCfDesc);
1075+
Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv);
10611076
return new Path(new Path(regionDir, family), path.getName());
10621077
}
10631078

0 commit comments

Comments
 (0)