From 70bb8e11d0375e70cec034ac31cb7ee66ca57587 Mon Sep 17 00:00:00 2001 From: Zach York Date: Tue, 30 Apr 2019 12:57:38 -0700 Subject: [PATCH] HBASE-21070 Add Test for SnapshotFileCache for HBase backed by S3 SnapshotFileCache depends on getting the last modified time of the snapshot directory, however, S3 FileSystem's do not update the last modified time of the top 'folder' when objects are added/removed. This commit adds a test for the previously fixed SnapshotFileCache. --- .../snapshot/TestSnapshotFileCache.java | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java index 7ef547782eb5..6f403ab3e11b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java @@ -58,14 +58,18 @@ public class TestSnapshotFileCache { private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class); private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + // don't refresh the cache unless we tell it to + private static final long PERIOD = Long.MAX_VALUE; private static FileSystem fs; private static Path rootDir; + private static Path snapshotDir; @BeforeClass public static void startCluster() throws Exception { UTIL.startMiniDFSCluster(1); fs = UTIL.getDFSCluster().getFileSystem(); rootDir = UTIL.getDefaultRootDirPath(); + snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); } @AfterClass @@ -76,48 +80,57 @@ public static void stopCluster() throws Exception { @After public void cleanupFiles() throws Exception { // cleanup the snapshot directory - Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); fs.delete(snapshotDir, true); } @Test public void testLoadAndDelete() throws IOException { - // don't refresh the cache unless we tell it to - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); - createAndTestSnapshotV1(cache, "snapshot1a", false, true); + createAndTestSnapshotV1(cache, "snapshot1a", false, true, false); - createAndTestSnapshotV2(cache, "snapshot2a", false, true); + createAndTestSnapshotV2(cache, "snapshot2a", false, true, false); } @Test public void testReloadModifiedDirectory() throws IOException { - // don't refresh the cache unless we tell it to - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); - createAndTestSnapshotV1(cache, "snapshot1", false, true); + createAndTestSnapshotV1(cache, "snapshot1", false, true, false); // now delete the snapshot and add a file with a different name - createAndTestSnapshotV1(cache, "snapshot1", false, false); + createAndTestSnapshotV1(cache, "snapshot1", false, false, false); - createAndTestSnapshotV2(cache, "snapshot2", false, true); + createAndTestSnapshotV2(cache, "snapshot2", false, true, false); // now delete the snapshot and add a file with a different name - createAndTestSnapshotV2(cache, "snapshot2", false, false); + createAndTestSnapshotV2(cache, "snapshot2", false, false, false); } @Test public void testSnapshotTempDirReload() throws IOException { - long period = Long.MAX_VALUE; - // This doesn't refresh cache until we invoke it explicitly - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + SnapshotFileCache cache = + new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); + + // Add a new non-tmp snapshot + createAndTestSnapshotV1(cache, "snapshot0v1", false, false, false); + createAndTestSnapshotV1(cache, "snapshot0v2", false, false, false); + } + + @Test + public void testCacheUpdatedWhenLastModifiedOfSnapDirNotUpdated() throws IOException { + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); // Add a new non-tmp snapshot - createAndTestSnapshotV1(cache, "snapshot0v1", false, false); - createAndTestSnapshotV1(cache, "snapshot0v2", false, false); + createAndTestSnapshotV1(cache, "snapshot1v1", false, false, true); + createAndTestSnapshotV1(cache, "snapshot1v2", false, false, true); + + // Add a new tmp snapshot + createAndTestSnapshotV2(cache, "snapshot2v1", true, false, true); + + // Add another tmp snapshot + createAndTestSnapshotV2(cache, "snapshot2v2", true, false, true); } class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { @@ -130,23 +143,24 @@ public Collection filesUnderSnapshot(final Path snapshotDir) throws IOEx } private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, - final String name, final boolean tmp, final boolean removeOnExit) throws IOException { + final String name, final boolean tmp, final boolean removeOnExit, boolean setFolderTime) + throws IOException { SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); - createAndTestSnapshot(cache, builder, tmp, removeOnExit); + createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime); return builder; } private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, - final boolean tmp, final boolean removeOnExit) throws IOException { + final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException { SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); - createAndTestSnapshot(cache, builder, tmp, removeOnExit); + createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime); } private void createAndTestSnapshot(final SnapshotFileCache cache, final SnapshotMock.SnapshotBuilder builder, - final boolean tmp, final boolean removeOnExit) throws IOException { + final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException { List files = new ArrayList<>(); for (int i = 0; i < 3; ++i) { for (Path filePath: builder.addRegion()) { @@ -157,6 +171,10 @@ private void createAndTestSnapshot(final SnapshotFileCache cache, // Finalize the snapshot builder.commit(); + if (setFolderTime) { + fs.setTimes(snapshotDir, 0, -1); + } + // Make sure that all files are still present for (Path path: files) { assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path));