From cf76691f4b7de01ef58f19e0de88a6f072690502 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sat, 22 Jun 2019 02:25:25 +0530 Subject: [PATCH 1/5] [HBASE-22606] : BucketCache additional tests --- .../io/hfile/bucket/TestBucketCache.java | 125 ++++++++++++++---- 1 file changed, 100 insertions(+), 25 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java index 19c1d66b0936..de6636cc0db5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java @@ -108,8 +108,7 @@ public static Iterable data() { final long capacitySize = 32 * 1024 * 1024; final int writeThreads = BucketCache.DEFAULT_WRITER_THREADS; final int writerQLen = BucketCache.DEFAULT_WRITER_QUEUE_ITEMS; - String ioEngineName = "offheap"; - String persistencePath = null; + private String ioEngineName = "offheap"; private static class MockedBucketCache extends BucketCache { @@ -133,10 +132,9 @@ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { } @Before - public void setup() throws FileNotFoundException, IOException { - cache = - new MockedBucketCache(ioEngineName, capacitySize, constructedBlockSize, - constructedBlockSizes, writeThreads, writerQLen, persistencePath); + public void setup() throws IOException { + cache = new MockedBucketCache(ioEngineName, capacitySize, constructedBlockSize, + constructedBlockSizes, writeThreads, writerQLen, null); } @After @@ -251,17 +249,52 @@ public void testRetrieveFromFile() throws Exception { HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); Path testDir = TEST_UTIL.getDataTestDir(); TEST_UTIL.getTestFileSystem().mkdirs(testDir); - String ioEngineName = "file:" + testDir + "/bucket.cache"; + testRetrievalUtils(testDir, ioEngineName); + int[] smallBucketSizes = new int[]{3 * 1024, 5 * 1024}; String persistencePath = testDir + "/bucket.persistence"; + BucketCache bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, + smallBucketSizes, writeThreads, writerQLen, persistencePath); + assertFalse(new File(persistencePath).exists()); + assertEquals(0, bucketCache.getAllocator().getUsedSize()); + assertEquals(0, bucketCache.backingMap.size()); + TEST_UTIL.cleanupTestDir(); + } + + @Test + public void testRetrieveFromMMap() throws Exception { + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + Path testDir = TEST_UTIL.getDataTestDir(); + TEST_UTIL.getTestFileSystem().mkdirs(testDir); + final String ioEngineName = "mmap:" + testDir + "/bucket.cache"; + testRetrievalUtils(testDir, ioEngineName); + } + @Test + public void testRetrieveFromPMem() throws Exception { + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + Path testDir = TEST_UTIL.getDataTestDir(); + TEST_UTIL.getTestFileSystem().mkdirs(testDir); + final String ioEngineName = "pmem:" + testDir + "/bucket.cache"; + testRetrievalUtils(testDir, ioEngineName); + int[] smallBucketSizes = new int[]{3 * 1024, 5 * 1024}; + String persistencePath = testDir + "/bucket.persistence"; BucketCache bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, - constructedBlockSizes, writeThreads, writerQLen, persistencePath); + smallBucketSizes, writeThreads, writerQLen, persistencePath); + assertFalse(new File(persistencePath).exists()); + assertEquals(0, bucketCache.getAllocator().getUsedSize()); + assertEquals(0, bucketCache.backingMap.size()); + TEST_UTIL.cleanupTestDir(); + } + + private void testRetrievalUtils(Path testDir, String ioEngineName) + throws IOException, InterruptedException { + final String persistencePath = testDir + "/bucket.persistence"; + BucketCache bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, + constructedBlockSizes, writeThreads, writerQLen, persistencePath); long usedSize = bucketCache.getAllocator().getUsedSize(); assertEquals(0, usedSize); - HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1); - // Add blocks for (HFileBlockPair block : blocks) { bucketCache.cacheBlock(block.getBlockName(), block.getBlock()); } @@ -270,28 +303,49 @@ public void testRetrieveFromFile() throws Exception { } usedSize = bucketCache.getAllocator().getUsedSize(); assertNotEquals(0, usedSize); - // persist cache to file bucketCache.shutdown(); assertTrue(new File(persistencePath).exists()); - - // restore cache from file bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, - constructedBlockSizes, writeThreads, writerQLen, persistencePath); + constructedBlockSizes, writeThreads, writerQLen, persistencePath); assertFalse(new File(persistencePath).exists()); assertEquals(usedSize, bucketCache.getAllocator().getUsedSize()); - // persist cache to file bucketCache.shutdown(); assertTrue(new File(persistencePath).exists()); + } - // reconfig buckets sizes, the biggest bucket is small than constructedBlockSize (8k or 16k) - // so it can't restore cache from file - int[] smallBucketSizes = new int[] { 2 * 1024 + 1024, 4 * 1024 + 1024 }; + @Test(expected = IllegalArgumentException.class) + public void testRetrieveFailure() throws Exception { + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + Path testDir = TEST_UTIL.getDataTestDir(); + TEST_UTIL.getTestFileSystem().mkdirs(testDir); + final String ioEngineName = testDir + "/bucket.cache"; + testRetrievalUtils(testDir, ioEngineName); + } + + @Test + public void testRetrieveFromFileWithoutPersistence() throws Exception { + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + Path testDir = TEST_UTIL.getDataTestDir(); + TEST_UTIL.getTestFileSystem().mkdirs(testDir); + String ioEngineName = "file:" + testDir + "/bucket.cache"; + BucketCache bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, + constructedBlockSizes, writeThreads, writerQLen, null); + long usedSize = bucketCache.getAllocator().getUsedSize(); + assertEquals(0, usedSize); + HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1); + for (HFileBlockPair block : blocks) { + bucketCache.cacheBlock(block.getBlockName(), block.getBlock()); + } + for (HFileBlockPair block : blocks) { + cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock()); + } + usedSize = bucketCache.getAllocator().getUsedSize(); + assertNotEquals(0, usedSize); + bucketCache.shutdown(); bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, - smallBucketSizes, writeThreads, writerQLen, persistencePath); - assertFalse(new File(persistencePath).exists()); + constructedBlockSizes, writeThreads, writerQLen, null); assertEquals(0, bucketCache.getAllocator().getUsedSize()); - assertEquals(0, bucketCache.backingMap.size()); - + bucketCache.shutdown(); TEST_UTIL.cleanupTestDir(); } @@ -315,13 +369,34 @@ public void testGetPartitionSize() throws IOException { conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, - constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); + constructedBlockSizes, writeThreads, writerQLen, null, 100, conf); validateGetPartitionSize(cache, 0.1f, 0.5f); validateGetPartitionSize(cache, 0.7f, 0.5f); validateGetPartitionSize(cache, 0.2f, 0.5f); } + @Test + public void testCacheSizeCapacity() throws IOException { + validateGetPartitionSize(cache, BucketCache.DEFAULT_SINGLE_FACTOR, + BucketCache.DEFAULT_MIN_FACTOR); + Configuration conf = HBaseConfiguration.create(); + conf.setFloat(BucketCache.MIN_FACTOR_CONFIG_NAME, 0.5f); + conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f); + conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f); + conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); + boolean isTestCompleted = false; + try { + new BucketCache(ioEngineName, Long.MAX_VALUE, 1, constructedBlockSizes, writeThreads, + writerQLen, null, 100, conf); + } catch (IllegalArgumentException e) { + if ("Cache capacity is too large, only support 32TB now".equals(e.getMessage())) { + isTestCompleted = true; + } + } + Assert.assertTrue(isTestCompleted); + } + @Test public void testValidBucketCacheConfigs() throws IOException { Configuration conf = HBaseConfiguration.create(); @@ -333,7 +408,7 @@ public void testValidBucketCacheConfigs() throws IOException { conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, - constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); + constructedBlockSizes, writeThreads, writerQLen, null, 100, conf); assertEquals(BucketCache.ACCEPT_FACTOR_CONFIG_NAME + " failed to propagate.", 0.9f, cache.getAcceptableFactor(), 0); @@ -401,7 +476,7 @@ private void checkConfigValues(Configuration conf, Map configMa conf.setFloat(configName, configMap.get(configName)[i]); } BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, - constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); + constructedBlockSizes, writeThreads, writerQLen, null, 100, conf); assertTrue("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]); } catch (IllegalArgumentException e) { assertFalse("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]); From 351215d247359f2700ef595d1386fb98c7cd5317 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sun, 23 Jun 2019 19:55:37 +0530 Subject: [PATCH 2/5] minor changes --- .../apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java index de6636cc0db5..85ec7e1e2384 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java @@ -390,9 +390,8 @@ public void testCacheSizeCapacity() throws IOException { new BucketCache(ioEngineName, Long.MAX_VALUE, 1, constructedBlockSizes, writeThreads, writerQLen, null, 100, conf); } catch (IllegalArgumentException e) { - if ("Cache capacity is too large, only support 32TB now".equals(e.getMessage())) { - isTestCompleted = true; - } + Assert.assertEquals("Cache capacity is too large, only support 32TB now", e.getMessage()); + isTestCompleted = true; } Assert.assertTrue(isTestCompleted); } From 43d3d350825d48fed7551d41cdb78a5c6348c94b Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 24 Jun 2019 08:24:43 +0530 Subject: [PATCH 3/5] trigger build --- .../org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java index 85ec7e1e2384..b1b0ef514754 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java @@ -396,6 +396,7 @@ public void testCacheSizeCapacity() throws IOException { Assert.assertTrue(isTestCompleted); } + @Test public void testValidBucketCacheConfigs() throws IOException { Configuration conf = HBaseConfiguration.create(); From 03fdaa528c50ab2b01c5586a3f2a2a302d91a191 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 24 Jun 2019 09:04:12 +0530 Subject: [PATCH 4/5] fail if IAE is not thrown --- .../apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java index 54df336c95ad..060f4c7db34a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java @@ -393,15 +393,13 @@ public void testCacheSizeCapacity() throws IOException { conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f); conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f); conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); - boolean isTestCompleted = false; try { new BucketCache(ioEngineName, Long.MAX_VALUE, 1, constructedBlockSizes, writeThreads, writerQLen, null, 100, conf); + Assert.fail("Should have thrown IllegalArgumentException because of large cache capacity"); } catch (IllegalArgumentException e) { Assert.assertEquals("Cache capacity is too large, only support 32TB now", e.getMessage()); - isTestCompleted = true; } - Assert.assertTrue(isTestCompleted); } From b5a73382aa83341ba10236405769751f1d0a4968 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 24 Jun 2019 09:17:02 +0530 Subject: [PATCH 5/5] trigger the build --- .../apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java index 060f4c7db34a..ae0c84701058 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java @@ -396,7 +396,7 @@ public void testCacheSizeCapacity() throws IOException { try { new BucketCache(ioEngineName, Long.MAX_VALUE, 1, constructedBlockSizes, writeThreads, writerQLen, null, 100, conf); - Assert.fail("Should have thrown IllegalArgumentException because of large cache capacity"); + Assert.fail("Should have thrown IllegalArgumentException because of large cache capacity!"); } catch (IllegalArgumentException e) { Assert.assertEquals("Cache capacity is too large, only support 32TB now", e.getMessage()); }