From 6e245b445856c77316addf585589a409ee52e3f9 Mon Sep 17 00:00:00 2001 From: Zheng Wang <18031031@qq.com> Date: Wed, 10 Jun 2020 19:41:11 +0800 Subject: [PATCH 1/3] HBASE-24530 Introduce a split policy similar with SteppingSplitPolicy but count all store size --- ...creasingToUpperBoundRegionSplitPolicy.java | 26 +++++----- .../SteppingAllStoresSizeSplitPolicy.java | 49 +++++++++++++++++++ .../regionserver/TestRegionSplitPolicy.java | 35 +++++++++++++ 3 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingAllStoresSizeSplitPolicy.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java index c40d6aafd622..8b54fd6b7183 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java @@ -70,31 +70,35 @@ protected void configureForRegion(HRegion region) { @Override protected boolean shouldSplit() { - boolean foundABigStore = false; + // If any of the stores is unable to split (eg they contain reference files) + // then don't split + for (HStore store : region.getStores()) { + if (!store.canSplit()) { + return false; + } + } // Get count of regions that have the same common table as this.region int tableRegionsCount = getCountOfCommonTableRegions(); // Get size to check long sizeToCheck = getSizeToCheck(tableRegionsCount); + return isExceedSize(tableRegionsCount, sizeToCheck); + } + /** + * @return true if any store's size exceed the sizeToCheck + */ + protected boolean isExceedSize(int tableRegionsCount, long sizeToCheck) { for (HStore store : region.getStores()) { - // If any of the stores is unable to split (eg they contain reference files) - // then don't split - if (!store.canSplit()) { - return false; - } - - // Mark if any store is big enough long size = store.getSize(); if (size > sizeToCheck) { LOG.debug("ShouldSplit because " + store.getColumnFamilyName() + " size=" + StringUtils.humanSize(size) + ", sizeToCheck=" + StringUtils.humanSize(sizeToCheck) + ", regionsWithCommonTable=" + tableRegionsCount); - foundABigStore = true; + return true; } } - - return foundABigStore; + return false; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingAllStoresSizeSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingAllStoresSizeSplitPolicy.java new file mode 100644 index 000000000000..c7ab5a8d2911 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingAllStoresSizeSplitPolicy.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.procedure2.util.StringUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.Private +public class SteppingAllStoresSizeSplitPolicy extends SteppingSplitPolicy { + private static final Logger LOG = + LoggerFactory.getLogger(SteppingAllStoresSizeSplitPolicy.class); + + /** + * @return true if sum of store's size exceed the sizeToCheck + */ + @Override + protected boolean isExceedSize(int tableRegionsCount, long sizeToCheck) { + long sumSize = 0; + for (HStore store : region.getStores()) { + sumSize += store.getSize(); + } + if (sumSize > sizeToCheck) { + LOG.debug("ShouldSplit because region size is big enough " + + "size={}, sizeToCheck={}, regionsWithCommonTable={}", + StringUtils.humanSize(sumSize), StringUtils.humanSize(sizeToCheck), + tableRegionsCount); + return true; + } + return false; + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java index 5ca693c5a939..a5c5813e8a97 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java @@ -153,6 +153,41 @@ public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException { assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0)); } + @Test + public void testIsExceedSize() throws IOException { + // Configure SteppingAllStoresSizeSplitPolicy as our split policy + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + SteppingAllStoresSizeSplitPolicy.class.getName()); + // Now make it so the mock region has a RegionServerService that will + // return 'online regions'. + RegionServerServices rss = Mockito.mock(RegionServerServices.class); + final List regions = new ArrayList<>(); + Mockito.doReturn(regions).when(rss).getRegions(TABLENAME); + Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss); + + SteppingAllStoresSizeSplitPolicy policy = + (SteppingAllStoresSizeSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); + regions.add(mockRegion); + + HStore mockStore1 = Mockito.mock(HStore.class); + Mockito.doReturn(100L).when(mockStore1).getSize(); + HStore mockStore2 = Mockito.mock(HStore.class); + Mockito.doReturn(924L).when(mockStore2).getSize(); + HStore mockStore3 = Mockito.mock(HStore.class); + Mockito.doReturn(925L).when(mockStore3).getSize(); + + // test sum of store's size not greater than sizeToCheck + stores.add(mockStore1); + stores.add(mockStore2); + assertFalse(policy.isExceedSize(1, 1024)); + stores.clear(); + + // test sum of store's size greater than sizeToCheck + stores.add(mockStore1); + stores.add(mockStore3); + assertTrue(policy.isExceedSize(1, 1024)); + } + @Test public void testBusyRegionSplitPolicy() throws Exception { doReturn(TableDescriptorBuilder.newBuilder(TABLENAME).build()).when(mockRegion) From 6aa9cda64eb6a04f14f81f9e7b538ef39ed8aceb Mon Sep 17 00:00:00 2001 From: Zheng Wang <18031031@qq.com> Date: Tue, 30 Jun 2020 11:20:58 +0800 Subject: [PATCH 2/3] fix compile issue --- .../regionserver/TestRegionSplitPolicy.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java index a5c5813e8a97..549adf167f34 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java @@ -160,21 +160,21 @@ public void testIsExceedSize() throws IOException { SteppingAllStoresSizeSplitPolicy.class.getName()); // Now make it so the mock region has a RegionServerService that will // return 'online regions'. - RegionServerServices rss = Mockito.mock(RegionServerServices.class); + RegionServerServices rss = mock(RegionServerServices.class); final List regions = new ArrayList<>(); - Mockito.doReturn(regions).when(rss).getRegions(TABLENAME); - Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss); + doReturn(regions).when(rss).getRegions(TABLENAME); + when(mockRegion.getRegionServerServices()).thenReturn(rss); SteppingAllStoresSizeSplitPolicy policy = (SteppingAllStoresSizeSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); regions.add(mockRegion); - HStore mockStore1 = Mockito.mock(HStore.class); - Mockito.doReturn(100L).when(mockStore1).getSize(); - HStore mockStore2 = Mockito.mock(HStore.class); - Mockito.doReturn(924L).when(mockStore2).getSize(); - HStore mockStore3 = Mockito.mock(HStore.class); - Mockito.doReturn(925L).when(mockStore3).getSize(); + HStore mockStore1 = mock(HStore.class); + doReturn(100L).when(mockStore1).getSize(); + HStore mockStore2 = mock(HStore.class); + doReturn(924L).when(mockStore2).getSize(); + HStore mockStore3 = mock(HStore.class); + doReturn(925L).when(mockStore3).getSize(); // test sum of store's size not greater than sizeToCheck stores.add(mockStore1); From 9026dd35a03da7ee026efcb2a223dd5ed7484b7d Mon Sep 17 00:00:00 2001 From: Zheng Wang <18031031@qq.com> Date: Tue, 30 Jun 2020 14:20:09 +0800 Subject: [PATCH 3/3] fix ut issue --- .../apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java index 549adf167f34..48f830c88b13 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionSplitPolicy.java @@ -165,6 +165,8 @@ public void testIsExceedSize() throws IOException { doReturn(regions).when(rss).getRegions(TABLENAME); when(mockRegion.getRegionServerServices()).thenReturn(rss); + TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build(); + doReturn(td).when(mockRegion).getTableDescriptor(); SteppingAllStoresSizeSplitPolicy policy = (SteppingAllStoresSizeSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); regions.add(mockRegion);