Skip to content

HBASE-21070 Fix SnapshotFileCache for HBase backed by S3 #209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 6, 2019
+41 −23
Merged

HBASE-21070 Fix SnapshotFileCache for HBase backed by S3 #209

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 41 additions & 23 deletions ...e-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,18 @@ public class TestSnapshotFileCache {

private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
// don't refresh the cache unless we tell it to
private static final long PERIOD = Long.MAX_VALUE;
private static FileSystem fs;
private static Path rootDir;
private static Path snapshotDir;

@BeforeClass
public static void startCluster() throws Exception {
UTIL.startMiniDFSCluster(1);
fs = UTIL.getDFSCluster().getFileSystem();
rootDir = UTIL.getDefaultRootDirPath();
snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
}

@AfterClass
Expand All @@ -76,48 +80,57 @@ public static void stopCluster() throws Exception {
@After
public void cleanupFiles() throws Exception {
// cleanup the snapshot directory
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
fs.delete(snapshotDir, true);
}

@Test
public void testLoadAndDelete() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());

createAndTestSnapshotV1(cache, "snapshot1a", false, true);
createAndTestSnapshotV1(cache, "snapshot1a", false, true, false);

createAndTestSnapshotV2(cache, "snapshot2a", false, true);
createAndTestSnapshotV2(cache, "snapshot2a", false, true, false);
}

@Test
public void testReloadModifiedDirectory() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());

createAndTestSnapshotV1(cache, "snapshot1", false, true);
createAndTestSnapshotV1(cache, "snapshot1", false, true, false);
// now delete the snapshot and add a file with a different name
createAndTestSnapshotV1(cache, "snapshot1", false, false);
createAndTestSnapshotV1(cache, "snapshot1", false, false, false);

createAndTestSnapshotV2(cache, "snapshot2", false, true);
createAndTestSnapshotV2(cache, "snapshot2", false, true, false);
// now delete the snapshot and add a file with a different name
createAndTestSnapshotV2(cache, "snapshot2", false, false);
createAndTestSnapshotV2(cache, "snapshot2", false, false, false);
}

@Test
public void testSnapshotTempDirReload() throws IOException {
long period = Long.MAX_VALUE;
// This doesn't refresh cache until we invoke it explicitly
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache =
new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles());

// Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false, false);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false, false);
}

@Test
public void testCacheUpdatedWhenLastModifiedOfSnapDirNotUpdated() throws IOException {
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());

// Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
createAndTestSnapshotV1(cache, "snapshot1v1", false, false, true);
createAndTestSnapshotV1(cache, "snapshot1v2", false, false, true);

// Add a new tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v1", true, false, true);

// Add another tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v2", true, false, true);
}

class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
Expand All @@ -130,23 +143,24 @@ public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOEx
}

private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache,
final String name, final boolean tmp, final boolean removeOnExit) throws IOException {
final String name, final boolean tmp, final boolean removeOnExit, boolean setFolderTime)
throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit);
createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
return builder;
}

private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
final boolean tmp, final boolean removeOnExit) throws IOException {
final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit);
createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
}

private void createAndTestSnapshot(final SnapshotFileCache cache,
final SnapshotMock.SnapshotBuilder builder,
final boolean tmp, final boolean removeOnExit) throws IOException {
final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
List<Path> files = new ArrayList<>();
for (int i = 0; i < 3; ++i) {
for (Path filePath: builder.addRegion()) {
Expand All @@ -157,6 +171,10 @@ private void createAndTestSnapshot(final SnapshotFileCache cache,
// Finalize the snapshot
builder.commit();

if (setFolderTime) {
fs.setTimes(snapshotDir, 0, -1);
}

// Make sure that all files are still present
for (Path path: files) {
assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path));
Expand Down