diff --git a/.asf.yaml b/.asf.yaml index a991e8528d78..d1abb7131437 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -38,4 +38,4 @@ notifications: commits: commits@hbase.apache.org issues: issues@hbase.apache.org pullrequests: issues@hbase.apache.org - jira_options: link + jira_options: link label diff --git a/bin/draining_servers.rb b/bin/draining_servers.rb index 9e34b2bcfb07..95707b064b8b 100644 --- a/bin/draining_servers.rb +++ b/bin/draining_servers.rb @@ -25,7 +25,6 @@ java_import org.apache.hadoop.hbase.HBaseConfiguration java_import org.apache.hadoop.hbase.client.ConnectionFactory -java_import org.apache.hadoop.hbase.client.HBaseAdmin java_import org.apache.hadoop.hbase.zookeeper.ZKUtil java_import org.apache.hadoop.hbase.zookeeper.ZNodePaths java_import org.slf4j.LoggerFactory @@ -136,8 +135,12 @@ def listServers(_options) hostOrServers = ARGV[1..ARGV.size] -# Create a logger and save it to ruby global -$LOG = LoggerFactory.getLogger(NAME) +# disable debug/info logging on this script for clarity +log_level = 'ERROR' +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level) + case ARGV[0] when 'add' if ARGV.length < 2 diff --git a/bin/get-active-master.rb b/bin/get-active-master.rb index d8c96fe3d4ad..65e0c3cd0fe0 100644 --- a/bin/get-active-master.rb +++ b/bin/get-active-master.rb @@ -24,9 +24,10 @@ java_import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker # disable debug/info logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) -org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) +log_level = 'ERROR' +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level) config = HBaseConfiguration.create diff --git a/bin/hbase b/bin/hbase index 60cfb8afef9a..81379eaa587d 100755 --- a/bin/hbase +++ b/bin/hbase @@ -496,7 +496,8 @@ add_jdk11_deps_to_classpath() { } add_jdk11_jvm_flags() { - HBASE_OPTS="$HBASE_OPTS -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true --add-modules jdk.unsupported --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED" + # Keep in sync with hbase-surefire.jdk11.flags in the root pom.xml + HBASE_OPTS="$HBASE_OPTS -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true --add-modules jdk.unsupported --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-exports java.base/sun.net.dns=ALL-UNNAMED --add-exports java.base/sun.net.util=ALL-UNNAMED" } add_opentelemetry_agent() { @@ -786,7 +787,7 @@ fi # Add lib/jdk11 jars to the classpath if [ "${DEBUG}" = "true" ]; then - echo "Deciding on addition of lib/jdk11 jars to the classpath" + echo "Deciding on addition of lib/jdk11 jars to the classpath and setting JVM module flags" fi addJDK11Jars=false @@ -879,6 +880,7 @@ export CLASSPATH if [ "${DEBUG}" = "true" ]; then echo "classpath=${CLASSPATH}" >&2 HBASE_OPTS="${HBASE_OPTS} -Xdiag" + echo "HBASE_OPTS=${HBASE_OPTS}" fi # resolve the command arguments diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh index 104e9a0b67c3..0e8b3feed213 100644 --- a/bin/hbase-config.sh +++ b/bin/hbase-config.sh @@ -178,8 +178,12 @@ EOF fi function read_java_version() { - properties="$("${JAVA_HOME}/bin/java" -XshowSettings:properties -version 2>&1)" - echo "${properties}" | "${GREP}" java.runtime.version | head -1 | "${SED}" -e 's/.* = \([^ ]*\)/\1/' + # Avoid calling java repeatedly + if [ -z "$read_java_version_cached" ]; then + properties="$("${JAVA_HOME}/bin/java" -XshowSettings:properties -version 2>&1)" + read_java_version_cached="$(echo "${properties}" | "${GREP}" java.runtime.version | head -1 | "${SED}" -e 's/.* = \([^ ]*\)/\1/')" + fi + echo "$read_java_version_cached" } # Inspect the system properties exposed by this JVM to identify the major diff --git a/bin/replication/copy_tables_desc.rb b/bin/replication/copy_tables_desc.rb index 44a24f9eea09..39bed5423c71 100644 --- a/bin/replication/copy_tables_desc.rb +++ b/bin/replication/copy_tables_desc.rb @@ -27,11 +27,8 @@ java_import org.apache.hadoop.conf.Configuration java_import org.apache.hadoop.hbase.HBaseConfiguration java_import org.apache.hadoop.hbase.HConstants -java_import org.apache.hadoop.hbase.HTableDescriptor java_import org.apache.hadoop.hbase.TableName java_import org.apache.hadoop.hbase.client.ConnectionFactory -java_import org.apache.hadoop.hbase.client.HBaseAdmin -java_import org.slf4j.LoggerFactory # Name of this script NAME = 'copy_tables_desc'.freeze @@ -45,7 +42,7 @@ def usage def copy(src, dst, table) # verify if table exists in source cluster begin - t = src.getTableDescriptor(TableName.valueOf(table)) + t = src.getDescriptor(TableName.valueOf(table)) rescue org.apache.hadoop.hbase.TableNotFoundException puts format("Source table \"%s\" doesn't exist, skipping.", table) return @@ -62,9 +59,13 @@ def copy(src, dst, table) puts format('Schema for table "%s" was succesfully copied to remote cluster.', table) end -usage if ARGV.size < 2 || ARGV.size > 3 +# disable debug/info logging on this script for clarity +log_level = 'ERROR' +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level) -LOG = LoggerFactory.getLogger(NAME) +usage if ARGV.size < 2 || ARGV.size > 3 parts1 = ARGV[0].split(':') diff --git a/bin/shutdown_regionserver.rb b/bin/shutdown_regionserver.rb index a131776e32ae..d8ddb2a8625d 100644 --- a/bin/shutdown_regionserver.rb +++ b/bin/shutdown_regionserver.rb @@ -24,7 +24,6 @@ include Java java_import org.apache.hadoop.hbase.HBaseConfiguration -java_import org.apache.hadoop.hbase.client.HBaseAdmin java_import org.apache.hadoop.hbase.client.ConnectionFactory def usage(msg = nil) @@ -35,6 +34,12 @@ def usage(msg = nil) abort end +# disable debug/info logging on this script for clarity +log_level = 'ERROR' +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level) + usage if ARGV.empty? ARGV.each do |x| diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index bb79ea8928ab..a767775b36b4 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -91,7 +91,7 @@ pipeline { rm -rf "${YETUS_DIR}" "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ --working-dir "${WORKSPACE}/downloads-yetus" \ - --keys 'https://www.apache.org/dist/yetus/KEYS' \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ --verify-tar-gz \ "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \ "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz" @@ -138,7 +138,7 @@ pipeline { echo "Ensure we have a copy of Hadoop ${HADOOP2_VERSION}" "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ --working-dir "${WORKSPACE}/downloads-hadoop-2" \ - --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \ + --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ --verify-tar-gz \ "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \ "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz" @@ -152,7 +152,7 @@ pipeline { } stage ('hadoop 3 cache') { environment { - HADOOP3_VERSION="3.1.1" + HADOOP3_VERSION="3.3.5" } steps { // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( @@ -166,7 +166,7 @@ pipeline { echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}" "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ --working-dir "${WORKSPACE}/downloads-hadoop-3" \ - --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \ + --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ --verify-tar-gz \ "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \ "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz" @@ -661,6 +661,9 @@ pipeline { rm -rf "hadoop-3" && mkdir "hadoop-3" rm -rf ".m2-for-repo" && mkdir ".m2-for-repo" rm -rf ".m2-for-src" && mkdir ".m2-for-src" + # remove old hadoop tarballs in workspace + rm -rf hadoop-2*.tar.gz + rm -rf hadoop-3*.tar.gz echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile ''' @@ -726,6 +729,8 @@ pipeline { echo "Attempting to use run an instance on top of Hadoop 3." artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1) tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" + # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471 + ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3 if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ --single-process \ --working-dir output-integration/hadoop-3 \ diff --git a/dev-support/design-docs/HBASE-26220 Use P2P communicate between region servers to sync the list for bootstrap node.pdf b/dev-support/design-docs/HBASE-26220 Use P2P communicate between region servers to sync the list for bootstrap node.pdf new file mode 100644 index 000000000000..7fcd1f6b9cd9 Binary files /dev/null and b/dev-support/design-docs/HBASE-26220 Use P2P communicate between region servers to sync the list for bootstrap node.pdf differ diff --git a/dev-support/design-docs/HBASE-28196 Yield SCP and TRSP when they are blocked.pdf b/dev-support/design-docs/HBASE-28196 Yield SCP and TRSP when they are blocked.pdf new file mode 100644 index 000000000000..1ed97e793e77 Binary files /dev/null and b/dev-support/design-docs/HBASE-28196 Yield SCP and TRSP when they are blocked.pdf differ diff --git a/dev-support/flaky-tests/python-requirements.txt b/dev-support/flaky-tests/python-requirements.txt index 9ca8c66afb1a..8ef087b0036d 100644 --- a/dev-support/flaky-tests/python-requirements.txt +++ b/dev-support/flaky-tests/python-requirements.txt @@ -17,6 +17,6 @@ # requests==2.31.0 future==0.18.3 -gitpython==3.1.37 +gitpython==3.1.41 rbtools==4.0 -jinja2==3.1.2 +jinja2==3.1.3 diff --git a/dev-support/git-jira-release-audit/requirements.txt b/dev-support/git-jira-release-audit/requirements.txt index 23a4b916fd4b..5e402a1875f4 100644 --- a/dev-support/git-jira-release-audit/requirements.txt +++ b/dev-support/git-jira-release-audit/requirements.txt @@ -19,11 +19,11 @@ blessed==1.17.0 certifi==2023.7.22 cffi==1.13.2 chardet==3.0.4 -cryptography==41.0.6 +cryptography==42.0.2 defusedxml==0.6.0 enlighten==1.4.0 gitdb2==2.0.6 -GitPython==3.1.37 +GitPython==3.1.41 idna==2.8 jira==2.0.0 oauthlib==3.1.0 diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 67aa2d1d168f..c7131c45e2f1 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -602,13 +602,20 @@ function hadoopcheck_rebuild else hbase_hadoop3_versions="3.1.1 3.1.2 3.1.3 3.1.4 3.2.0 3.2.1 3.2.2 3.2.3 3.2.4 3.3.0 3.3.1 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6" fi - else - yetus_info "Setting Hadoop 3 versions to test based on branch-2.5+/master/feature branch rules" + elif [[ "${PATCH_BRANCH}" = branch-2.5 ]]; then + yetus_info "Setting Hadoop 3 versions to test based on branch-2.5 rules" if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then hbase_hadoop3_versions="3.2.4 3.3.6" else hbase_hadoop3_versions="3.2.3 3.2.4 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6" fi + else + yetus_info "Setting Hadoop 3 versions to test based on branch-2.6+/master/feature branch rules" + if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then + hbase_hadoop3_versions="3.3.6" + else + hbase_hadoop3_versions="3.3.5 3.3.6" + fi fi export MAVEN_OPTS="${MAVEN_OPTS}" diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh index ffe630865925..8290f06110f3 100755 --- a/dev-support/hbase_nightly_pseudo-distributed-test.sh +++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh @@ -324,7 +324,7 @@ redirect_and_run "${working_dir}/hadoop_cluster_smoke" \ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R / echo "Starting up HBase" -HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/start-hbase.sh" +HBASE_CONF_DIR="${working_dir}/hbase-conf/" HBASE_LOG_DIR="${working_dir}" "${component_install}/bin/start-hbase.sh" sleep_time=2 until "${component_install}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/waiting_hbase_startup.log" 2>&1 < compile + + org.apache.hbase + hbase-openssl + org.apache.hbase hbase-server diff --git a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java index 3c3852831033..68b8bfa3d9f3 100644 --- a/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java +++ b/hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java @@ -52,10 +52,12 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.FixMethodOrder; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.junit.runners.MethodSorters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +67,7 @@ import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; +@FixMethodOrder(MethodSorters.NAME_ASCENDING) @Category({ MiscTests.class, MediumTests.class }) public class TestFanOutOneBlockAsyncDFSOutput extends AsyncFSTestBase { @@ -142,8 +145,12 @@ public void test() throws IOException, InterruptedException, ExecutionException writeAndVerify(FS, f, out); } + /** + * Test method has been renamed to be the first in NAME_ASCENDING. It's an ugly hack to avoid + * flakiness. + */ @Test - public void testRecover() throws IOException, InterruptedException, ExecutionException { + public void test0Recover() throws IOException, InterruptedException, ExecutionException { Path f = new Path("/" + name.getMethodName()); EventLoop eventLoop = EVENT_LOOP_GROUP.next(); FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml index 478c8d8245ea..77d033d67fe6 100644 --- a/hbase-backup/pom.xml +++ b/hbase-backup/pom.xml @@ -105,6 +105,12 @@ + + + org.bouncycastle + bcpkix-jdk18on + test + org.apache.commons diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/Abortable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/Abortable.java index b9736d573454..b0a5a86d50bb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/Abortable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/Abortable.java @@ -37,7 +37,7 @@ public interface Abortable { void abort(String why, Throwable e); /** - * It just call another abort method and the Throwable parameter is null. + * It just calls another abort method and the Throwable parameter is null. * @param why Why we're aborting. * @see Abortable#abort(String, Throwable) */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java index 4e97dcab24dd..62c6951b4535 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java @@ -33,22 +33,17 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil; import org.apache.hadoop.hbase.exceptions.MasterRegistryFetchException; import org.apache.hadoop.hbase.ipc.HBaseRpcController; -import org.apache.hadoop.hbase.ipc.RpcClient; -import org.apache.hadoop.hbase.ipc.RpcClientFactory; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; @@ -79,30 +74,21 @@ abstract class AbstractRpcBasedConnectionRegistry implements ConnectionRegistry private final int hedgedReadFanOut; - // Configured list of end points to probe the meta information from. - private volatile ImmutableMap addr2Stub; - // RPC client used to talk to the masters. - private final RpcClient rpcClient; + private final ConnectionRegistryRpcStubHolder rpcStubHolder; private final RpcControllerFactory rpcControllerFactory; - private final int rpcTimeoutMs; private final RegistryEndpointsRefresher registryEndpointRefresher; - protected AbstractRpcBasedConnectionRegistry(Configuration conf, + protected AbstractRpcBasedConnectionRegistry(Configuration conf, User user, String hedgedReqsFanoutConfigName, String initialRefreshDelaySecsConfigName, String refreshIntervalSecsConfigName, String minRefreshIntervalSecsConfigName) throws IOException { this.hedgedReadFanOut = Math.max(1, conf.getInt(hedgedReqsFanoutConfigName, HEDGED_REQS_FANOUT_DEFAULT)); - rpcTimeoutMs = (int) Math.min(Integer.MAX_VALUE, - conf.getLong(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT)); - // XXX: we pass cluster id as null here since we do not have a cluster id yet, we have to fetch - // this through the master registry... - // This is a problem as we will use the cluster id to determine the authentication method - rpcClient = RpcClientFactory.createClient(conf, null); rpcControllerFactory = RpcControllerFactory.instantiate(conf); - populateStubs(getBootstrapNodes(conf)); + rpcStubHolder = new ConnectionRegistryRpcStubHolder(conf, user, rpcControllerFactory, + getBootstrapNodes(conf)); // could return null here is refresh interval is less than zero registryEndpointRefresher = RegistryEndpointsRefresher.create(conf, initialRefreshDelaySecsConfigName, @@ -114,19 +100,7 @@ protected AbstractRpcBasedConnectionRegistry(Configuration conf, protected abstract CompletableFuture> fetchEndpoints(); private void refreshStubs() throws IOException { - populateStubs(FutureUtils.get(fetchEndpoints())); - } - - private void populateStubs(Set addrs) throws IOException { - Preconditions.checkNotNull(addrs); - ImmutableMap.Builder builder = - ImmutableMap.builderWithExpectedSize(addrs.size()); - User user = User.getCurrent(); - for (ServerName masterAddr : addrs) { - builder.put(masterAddr, - ClientMetaService.newStub(rpcClient.createRpcChannel(masterAddr, user, rpcTimeoutMs))); - } - addr2Stub = builder.build(); + rpcStubHolder.refreshStubs(() -> FutureUtils.get(fetchEndpoints())); } /** @@ -211,20 +185,25 @@ private void groupCall(CompletableFuture future, Set CompletableFuture call(Callable callable, Predicate isValidResp, String debug) { - ImmutableMap addr2StubRef = addr2Stub; - Set servers = addr2StubRef.keySet(); - List stubs = new ArrayList<>(addr2StubRef.values()); - Collections.shuffle(stubs, ThreadLocalRandom.current()); CompletableFuture future = new CompletableFuture<>(); - groupCall(future, servers, stubs, 0, callable, isValidResp, debug, - new ConcurrentLinkedQueue<>()); + FutureUtils.addListener(rpcStubHolder.getStubs(), (addr2Stub, error) -> { + if (error != null) { + future.completeExceptionally(error); + return; + } + Set servers = addr2Stub.keySet(); + List stubs = new ArrayList<>(addr2Stub.values()); + Collections.shuffle(stubs, ThreadLocalRandom.current()); + groupCall(future, servers, stubs, 0, callable, isValidResp, debug, + new ConcurrentLinkedQueue<>()); + }); return future; } @RestrictedApi(explanation = "Should only be called in tests", link = "", allowedOnPath = ".*/src/test/.*") - Set getParsedServers() { - return addr2Stub.keySet(); + Set getParsedServers() throws IOException { + return FutureUtils.get(rpcStubHolder.getStubs()).keySet(); } /** @@ -277,8 +256,8 @@ public void close() { if (registryEndpointRefresher != null) { registryEndpointRefresher.stop(); } - if (rpcClient != null) { - rpcClient.close(); + if (rpcStubHolder != null) { + rpcStubHolder.close(); } }, getClass().getSimpleName() + ".close"); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterIdFetcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterIdFetcher.java new file mode 100644 index 000000000000..277629681ec6 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterIdFetcher.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ipc.HBaseRpcController; +import org.apache.hadoop.hbase.ipc.RpcClient; +import org.apache.hadoop.hbase.ipc.RpcClientFactory; +import org.apache.hadoop.hbase.ipc.RpcControllerFactory; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcChannel; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.ConnectionRegistryService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetConnectionRegistryRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetConnectionRegistryResponse; + +/** + * Fetch cluster id through special preamble header. + *

+ * An instance of this class should only be used once, like: + * + * 

+ * new ClusterIdFetcher().fetchClusterId()
+ *
+ * + * Calling the fetchClusterId multiple times will lead unexpected behavior. + *

+ * See HBASE-25051 for more details. + */ +@InterfaceAudience.Private +class ClusterIdFetcher { + + private static final Logger LOG = LoggerFactory.getLogger(ClusterIdFetcher.class); + + private final List bootstrapServers; + + private final User user; + + private final RpcClient rpcClient; + + private final RpcControllerFactory rpcControllerFactory; + + private final CompletableFuture future; + + ClusterIdFetcher(Configuration conf, User user, RpcControllerFactory rpcControllerFactory, + Set bootstrapServers) { + this.user = user; + // use null cluster id here as we do not know the cluster id yet, we will fetch it through this + // rpc client + this.rpcClient = RpcClientFactory.createClient(conf, null); + this.rpcControllerFactory = rpcControllerFactory; + this.bootstrapServers = new ArrayList(bootstrapServers); + // shuffle the bootstrap servers so we will not always fetch from the same one + Collections.shuffle(this.bootstrapServers); + future = new CompletableFuture(); + } + + /** + * Try get cluster id from the server with the given {@code index} in {@link #bootstrapServers}. + */ + private void getClusterId(int index) { + ServerName server = bootstrapServers.get(index); + LOG.debug("Going to request {} for getting cluster id", server); + // user and rpcTimeout are both not important here, as we will not actually send any rpc calls + // out, only a preamble connection header, but if we pass null as user, there will be NPE in + // some code paths... + RpcChannel channel = rpcClient.createRpcChannel(server, user, 0); + ConnectionRegistryService.Interface stub = ConnectionRegistryService.newStub(channel); + HBaseRpcController controller = rpcControllerFactory.newController(); + stub.getConnectionRegistry(controller, GetConnectionRegistryRequest.getDefaultInstance(), + new RpcCallback() { + + @Override + public void run(GetConnectionRegistryResponse resp) { + if (!controller.failed()) { + LOG.debug("Got connection registry info: {}", resp); + future.complete(resp.getClusterId()); + return; + } + if (ConnectionUtils.isUnexpectedPreambleHeaderException(controller.getFailed())) { + // this means we have connected to an old server where it does not support passing + // cluster id through preamble connnection header, so we fallback to use null + // cluster id, which is the old behavior + LOG.debug("Failed to get connection registry info, should be an old server," + + " fallback to use null cluster id", controller.getFailed()); + future.complete(null); + } else { + LOG.debug("Failed to get connection registry info", controller.getFailed()); + if (index == bootstrapServers.size() - 1) { + future.completeExceptionally(controller.getFailed()); + } else { + // try next bootstrap server + getClusterId(index + 1); + } + } + } + }); + + } + + CompletableFuture fetchClusterId() { + getClusterId(0); + // close the rpc client after we finish the request + FutureUtils.addListener(future, (r, e) -> rpcClient.close()); + return future; + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java index ac70091dcf65..716fb4863fe8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java @@ -338,7 +338,7 @@ public static CompletableFuture createAsyncConnection(Configura final User user, Map connectionAttributes) { return TraceUtil.tracedFuture(() -> { CompletableFuture future = new CompletableFuture<>(); - ConnectionRegistry registry = ConnectionRegistryFactory.getRegistry(conf); + ConnectionRegistry registry = ConnectionRegistryFactory.getRegistry(conf, user); addListener(registry.getClusterId(), (clusterId, error) -> { if (error != null) { registry.close(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionOverAsyncConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionOverAsyncConnection.java index 51368fc23c15..30c348e6d1f1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionOverAsyncConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionOverAsyncConnection.java @@ -29,8 +29,8 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.log.HBaseMarkers; -import org.apache.hadoop.hbase.util.ConcurrentMapUtils.IOExceptionSupplier; import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.hadoop.hbase.util.IOExceptionSupplier; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionRegistryFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionRegistryFactory.java index f198c3c22002..415d46397b8f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionRegistryFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionRegistryFactory.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hbase.HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -33,10 +34,10 @@ private ConnectionRegistryFactory() { } /** Returns The connection registry implementation to use. */ - static ConnectionRegistry getRegistry(Configuration conf) { + static ConnectionRegistry getRegistry(Configuration conf, User user) { Class clazz = conf.getClass(CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, RpcConnectionRegistry.class, ConnectionRegistry.class); - return ReflectionUtils.newInstance(clazz, conf); + return ReflectionUtils.newInstance(clazz, conf, user); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionRegistryRpcStubHolder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionRegistryRpcStubHolder.java new file mode 100644 index 000000000000..3dbcfbe8e6bf --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionRegistryRpcStubHolder.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ipc.RpcClient; +import org.apache.hadoop.hbase.ipc.RpcClientFactory; +import org.apache.hadoop.hbase.ipc.RpcControllerFactory; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.hadoop.hbase.util.IOExceptionSupplier; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.ClientMetaService; + +/** + * A class for creating {@link RpcClient} and related stubs used by + * {@link AbstractRpcBasedConnectionRegistry}. We need to connect to bootstrap nodes to get the + * cluster id first, before creating the final {@link RpcClient} and related stubs. + *

+ * See HBASE-25051 for more details. + */ +@InterfaceAudience.Private +class ConnectionRegistryRpcStubHolder implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(ConnectionRegistryRpcStubHolder.class); + + private final Configuration conf; + + // used for getting cluster id + private final Configuration noAuthConf; + + private final User user; + + private final RpcControllerFactory rpcControllerFactory; + + private final Set bootstrapNodes; + + private final int rpcTimeoutMs; + + private volatile ImmutableMap addr2Stub; + + private volatile RpcClient rpcClient; + + private CompletableFuture> addr2StubFuture; + + ConnectionRegistryRpcStubHolder(Configuration conf, User user, + RpcControllerFactory rpcControllerFactory, Set bootstrapNodes) { + this.conf = conf; + if (User.isHBaseSecurityEnabled(conf)) { + this.noAuthConf = new Configuration(conf); + this.noAuthConf.set(User.HBASE_SECURITY_CONF_KEY, "simple"); + } else { + this.noAuthConf = conf; + } + this.user = user; + this.rpcControllerFactory = rpcControllerFactory; + this.bootstrapNodes = Collections.unmodifiableSet(bootstrapNodes); + this.rpcTimeoutMs = (int) Math.min(Integer.MAX_VALUE, + conf.getLong(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT)); + } + + private ImmutableMap createStubs(RpcClient rpcClient, + Collection addrs) { + LOG.debug("Going to use new servers to create stubs: {}", addrs); + Preconditions.checkNotNull(addrs); + ImmutableMap.Builder builder = + ImmutableMap.builderWithExpectedSize(addrs.size()); + for (ServerName masterAddr : addrs) { + builder.put(masterAddr, + ClientMetaService.newStub(rpcClient.createRpcChannel(masterAddr, user, rpcTimeoutMs))); + } + return builder.build(); + } + + private CompletableFuture> + fetchClusterIdAndCreateStubs() { + CompletableFuture> future = + new CompletableFuture<>(); + addr2StubFuture = future; + FutureUtils.addListener( + new ClusterIdFetcher(noAuthConf, user, rpcControllerFactory, bootstrapNodes).fetchClusterId(), + (clusterId, error) -> { + synchronized (ConnectionRegistryRpcStubHolder.this) { + if (error != null) { + addr2StubFuture.completeExceptionally(error); + } else { + RpcClient c = RpcClientFactory.createClient(conf, clusterId); + ImmutableMap m = + createStubs(c, bootstrapNodes); + rpcClient = c; + addr2Stub = m; + addr2StubFuture.complete(m); + } + addr2StubFuture = null; + } + }); + // here we must use the local variable future instead of addr2StubFuture, as the above listener + // could be executed directly in the same thread(if the future completes quick enough), since + // the synchronized lock is reentrant, it could set addr2StubFuture to null in the end, so when + // arriving here the addr2StubFuture could be null. + return future; + } + + CompletableFuture> getStubs() { + ImmutableMap s = this.addr2Stub; + if (s != null) { + return CompletableFuture.completedFuture(s); + } + synchronized (this) { + s = this.addr2Stub; + if (s != null) { + return CompletableFuture.completedFuture(s); + } + if (addr2StubFuture != null) { + return addr2StubFuture; + } + return fetchClusterIdAndCreateStubs(); + } + } + + void refreshStubs(IOExceptionSupplier> fetchEndpoints) throws IOException { + // There is no actual call yet so we have not initialize the rpc client and related stubs yet, + // give up refreshing + if (addr2Stub == null) { + LOG.debug("Skip refreshing stubs as we have not initialized rpc client yet"); + return; + } + LOG.debug("Going to refresh stubs"); + assert rpcClient != null; + addr2Stub = createStubs(rpcClient, fetchEndpoints.get()); + } + + @Override + public void close() { + if (rpcClient != null) { + rpcClient.close(); + } + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java index 4827708a02e3..d073fef929fd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.ipc.FatalConnectionException; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.util.Bytes; @@ -663,4 +664,13 @@ static void setCoprocessorError(RpcController controller, Throwable error) { controller.setFailed(error.toString()); } } + + static boolean isUnexpectedPreambleHeaderException(IOException e) { + if (!(e instanceof RemoteException)) { + return false; + } + RemoteException re = (RemoteException) e; + return FatalConnectionException.class.getName().equals(re.getClassName()) + && re.getMessage().startsWith("Expected HEADER="); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java index 8df0504b2a9a..83b53ccba3c3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java @@ -135,10 +135,11 @@ public Map setRegionStateInMeta( } @Override - public List assigns(List encodedRegionNames, boolean override) throws IOException { + public List assigns(List encodedRegionNames, boolean override, boolean force) + throws IOException { try { AssignsResponse response = this.hbck.assigns(rpcControllerFactory.newController(), - RequestConverter.toAssignRegionsRequest(encodedRegionNames, override)); + RequestConverter.toAssignRegionsRequest(encodedRegionNames, override, force)); return response.getPidList(); } catch (ServiceException se) { LOG.debug(toCommaDelimitedString(encodedRegionNames), se); @@ -147,11 +148,11 @@ public List assigns(List encodedRegionNames, boolean override) thr } @Override - public List unassigns(List encodedRegionNames, boolean override) + public List unassigns(List encodedRegionNames, boolean override, boolean force) throws IOException { try { UnassignsResponse response = this.hbck.unassigns(rpcControllerFactory.newController(), - RequestConverter.toUnassignRegionsRequest(encodedRegionNames, override)); + RequestConverter.toUnassignRegionsRequest(encodedRegionNames, override, force)); return response.getPidList(); } catch (ServiceException se) { LOG.debug(toCommaDelimitedString(encodedRegionNames), se); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java index b5ba25058838..6baa876f9387 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java @@ -62,19 +62,26 @@ public interface Hbck extends Abortable, Closeable { * good if many Regions to online -- and it will schedule the assigns even in the case where * Master is initializing (as long as the ProcedureExecutor is up). Does NOT call Coprocessor * hooks. - * @param override You need to add the override for case where a region has previously - * been bypassed. When a Procedure has been bypassed, a Procedure will - * have completed but no other Procedure will be able to make progress - * on the target entity (intentionally). This override flag will - * override this fencing mechanism. + * @param override You need to add override for unset of the procedure from + * RegionStateNode without byPassing preTransitCheck + * @param force You need to add force for case where a region has previously been + * bypassed. When a Procedure has been bypassed, a Procedure will have + * completed but no other Procedure will be able to make progress on the + * target entity (intentionally). Skips preTransitCheck only when + * selected along with override option * @param encodedRegionNames Region encoded names; e.g. 1588230740 is the hard-coded encoding for * hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an example * of what a random user-space encoded Region name looks like. */ - List assigns(List encodedRegionNames, boolean override) throws IOException; + List assigns(List encodedRegionNames, boolean override, boolean force) + throws IOException; + + default List assigns(List encodedRegionNames, boolean override) throws IOException { + return assigns(encodedRegionNames, override, true); + } default List assigns(List encodedRegionNames) throws IOException { - return assigns(encodedRegionNames, false); + return assigns(encodedRegionNames, false, false); } /** @@ -82,19 +89,27 @@ default List assigns(List encodedRegionNames) throws IOException { * at a time -- good if many Regions to offline -- and it will schedule the assigns even in the * case where Master is initializing (as long as the ProcedureExecutor is up). Does NOT call * Coprocessor hooks. - * @param override You need to add the override for case where a region has previously - * been bypassed. When a Procedure has been bypassed, a Procedure will - * have completed but no other Procedure will be able to make progress - * on the target entity (intentionally). This override flag will - * override this fencing mechanism. + * @param override You need to add override for unset of the procedure from + * RegionStateNode without byPassing preTransitCheck + * @param force You need to add force for case where a region has previously been + * bypassed. When a Procedure has been bypassed, a Procedure will have + * completed but no other Procedure will be able to make progress on the + * target entity (intentionally). Skips preTransitCheck only when + * selected along with override option * @param encodedRegionNames Region encoded names; e.g. 1588230740 is the hard-coded encoding for * hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an example * of what a random user-space encoded Region name looks like. */ - List unassigns(List encodedRegionNames, boolean override) throws IOException; + List unassigns(List encodedRegionNames, boolean override, boolean force) + throws IOException; + + default List unassigns(List encodedRegionNames, boolean override) + throws IOException { + return unassigns(encodedRegionNames, override, true); + } default List unassigns(List encodedRegionNames) throws IOException { - return unassigns(encodedRegionNames, false); + return unassigns(encodedRegionNames, false, true); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterRegistry.java index b6f81c30f0bd..364180fe1414 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterRegistry.java @@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.DNS.ServerType; import org.apache.yetus.audience.InterfaceAudience; @@ -105,9 +106,10 @@ public static Set parseMasterAddrs(Configuration conf) throws Unknow private final String connectionString; - MasterRegistry(Configuration conf) throws IOException { - super(conf, MASTER_REGISTRY_HEDGED_REQS_FANOUT_KEY, MASTER_REGISTRY_INITIAL_REFRESH_DELAY_SECS, - MASTER_REGISTRY_PERIODIC_REFRESH_INTERVAL_SECS, MASTER_REGISTRY_MIN_SECS_BETWEEN_REFRESHES); + MasterRegistry(Configuration conf, User user) throws IOException { + super(conf, user, MASTER_REGISTRY_HEDGED_REQS_FANOUT_KEY, + MASTER_REGISTRY_INITIAL_REFRESH_DELAY_SECS, MASTER_REGISTRY_PERIODIC_REFRESH_INTERVAL_SECS, + MASTER_REGISTRY_MIN_SECS_BETWEEN_REFRESHES); connectionString = getConnectionString(conf); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OnlineLogRecord.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OnlineLogRecord.java index d9fd51e80a95..26979129cf18 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OnlineLogRecord.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OnlineLogRecord.java @@ -81,6 +81,7 @@ final public class OnlineLogRecord extends LogEntry { private final int queueTime; private final long responseSize; private final long blockBytesScanned; + private final long fsReadTime; private final String clientAddress; private final String serverClass; private final String methodName; @@ -120,6 +121,10 @@ public long getBlockBytesScanned() { return blockBytesScanned; } + public long getFsReadTime() { + return fsReadTime; + } + public String getClientAddress() { return clientAddress; } @@ -178,16 +183,18 @@ public Map getConnectionAttributes() { } OnlineLogRecord(final long startTime, final int processingTime, final int queueTime, - final long responseSize, final long blockBytesScanned, final String clientAddress, - final String serverClass, final String methodName, final String callDetails, final String param, - final String regionName, final String userName, final int multiGetsCount, - final int multiMutationsCount, final int multiServiceCalls, final Scan scan, - final Map requestAttributes, final Map connectionAttributes) { + final long responseSize, final long blockBytesScanned, final long fsReadTime, + final String clientAddress, final String serverClass, final String methodName, + final String callDetails, final String param, final String regionName, final String userName, + final int multiGetsCount, final int multiMutationsCount, final int multiServiceCalls, + final Scan scan, final Map requestAttributes, + final Map connectionAttributes) { this.startTime = startTime; this.processingTime = processingTime; this.queueTime = queueTime; this.responseSize = responseSize; this.blockBytesScanned = blockBytesScanned; + this.fsReadTime = fsReadTime; this.clientAddress = clientAddress; this.serverClass = serverClass; this.methodName = methodName; @@ -209,6 +216,7 @@ public static class OnlineLogRecordBuilder { private int queueTime; private long responseSize; private long blockBytesScanned; + private long fsReadTime; private String clientAddress; private String serverClass; private String methodName; @@ -251,6 +259,11 @@ public OnlineLogRecordBuilder setBlockBytesScanned(long blockBytesScanned) { return this; } + public OnlineLogRecordBuilder setFsReadTime(long fsReadTime) { + this.fsReadTime = fsReadTime; + return this; + } + public OnlineLogRecordBuilder setClientAddress(String clientAddress) { this.clientAddress = clientAddress; return this; @@ -319,9 +332,9 @@ public OnlineLogRecordBuilder setRequestAttributes(Map requestAt public OnlineLogRecord build() { return new OnlineLogRecord(startTime, processingTime, queueTime, responseSize, - blockBytesScanned, clientAddress, serverClass, methodName, callDetails, param, regionName, - userName, multiGetsCount, multiMutationsCount, multiServiceCalls, scan, requestAttributes, - connectionAttributes); + blockBytesScanned, fsReadTime, clientAddress, serverClass, methodName, callDetails, param, + regionName, userName, multiGetsCount, multiMutationsCount, multiServiceCalls, scan, + requestAttributes, connectionAttributes); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 103a64e520a1..9943aa936ea4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -81,6 +81,7 @@ import org.apache.hadoop.hbase.quotas.QuotaSettings; import org.apache.hadoop.hbase.quotas.QuotaTableUtil; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; @@ -966,6 +967,8 @@ public CompletableFuture flush(TableName tableName) { @Override public CompletableFuture flush(TableName tableName, byte[] columnFamily) { + Preconditions.checkNotNull(columnFamily, + "columnFamily is null, If you don't specify a columnFamily, use flush(TableName) instead."); return flush(tableName, Collections.singletonList(columnFamily)); } @@ -975,29 +978,50 @@ public CompletableFuture flush(TableName tableName, List columnFam // If the server version is lower than the client version, it's possible that the // flushTable method is not present in the server side, if so, we need to fall back // to the old implementation. + Preconditions.checkNotNull(columnFamilyList, + "columnFamily is null, If you don't specify a columnFamily, use flush(TableName) instead."); List columnFamilies = columnFamilyList.stream() .filter(cf -> cf != null && cf.length > 0).distinct().collect(Collectors.toList()); - FlushTableRequest request = RequestConverter.buildFlushTableRequest(tableName, columnFamilies, - ng.getNonceGroup(), ng.newNonce()); - CompletableFuture procFuture = this. procedureCall( - tableName, request, (s, c, req, done) -> s.flushTable(c, req, done), - (resp) -> resp.getProcId(), new FlushTableProcedureBiConsumer(tableName)); CompletableFuture future = new CompletableFuture<>(); - addListener(procFuture, (ret, error) -> { + addListener(getDescriptor(tableName), (tDesc, error) -> { if (error != null) { - if (error instanceof TableNotFoundException || error instanceof TableNotEnabledException) { - future.completeExceptionally(error); - } else if (error instanceof DoNotRetryIOException) { - // usually this is caused by the method is not present on the server or - // the hbase hadoop version does not match the running hadoop version. - // if that happens, we need fall back to the old flush implementation. - LOG.info("Unrecoverable error in master side. Fallback to FlushTableProcedure V1", error); - legacyFlush(future, tableName, columnFamilies); + future.completeExceptionally(error); + } else { + List nonFaimilies = columnFamilies.stream().filter(cf -> !tDesc.hasColumnFamily(cf)) + .map(cf -> Bytes.toString(cf)).collect(Collectors.toList()); + if (nonFaimilies.size() > 0) { + String noSuchFamiliesMsg = + String.format("There are non-existing families %s, we cannot flush the table %s", + nonFaimilies, tableName.getNameAsString()); + future.completeExceptionally(new NoSuchColumnFamilyException(noSuchFamiliesMsg)); } else { - future.completeExceptionally(error); + FlushTableRequest request = RequestConverter.buildFlushTableRequest(tableName, columnFamilies, + ng.getNonceGroup(), ng.newNonce()); + CompletableFuture procFuture = this. procedureCall( + tableName, request, (s, c, req, done) -> s.flushTable(c, req, done), + (resp) -> resp.getProcId(), new FlushTableProcedureBiConsumer(tableName)); + addListener(procFuture, (ret, error2) -> { + if (error2 != null) { + if ( + error2 instanceof TableNotFoundException + || error2 instanceof TableNotEnabledException + ) { + future.completeExceptionally(error2); + } else if (error2 instanceof DoNotRetryIOException) { + // usually this is caused by the method is not present on the server or + // the hbase hadoop version does not match the running hadoop version. + // if that happens, we need fall back to the old flush implementation. + LOG.info("Unrecoverable error in master side. Fallback to FlushTableProcedure V1", + error2); + legacyFlush(future, tableName, columnFamilies); + } else { + future.completeExceptionally(error2); + } + } else { + future.complete(ret); + } + }); } - } else { - future.complete(ret); } }); return future; @@ -1071,14 +1095,29 @@ CompletableFuture flushRegionInternal(byte[] regionName, by .completeExceptionally(new NoServerForRegionException(Bytes.toStringBinary(regionName))); return; } - addListener(flush(serverName, location.getRegion(), columnFamily, writeFlushWALMarker), - (ret, err2) -> { - if (err2 != null) { - future.completeExceptionally(err2); - } else { - future.complete(ret); - } - }); + TableName tableName = location.getRegion().getTable(); + addListener(getDescriptor(tableName), (tDesc, error2) -> { + if (error2 != null) { + future.completeExceptionally(error2); + return; + } + if (columnFamily != null && !tDesc.hasColumnFamily(columnFamily)) { + String noSuchFamiliedMsg = String.format( + "There are non-existing family %s, we cannot flush the region %s, in table %s", + Bytes.toString(columnFamily), location.getRegion().getRegionNameAsString(), + tableName.getNameAsString()); + future.completeExceptionally(new NoSuchColumnFamilyException(noSuchFamiliedMsg)); + return; + } + addListener(flush(serverName, location.getRegion(), columnFamily, writeFlushWALMarker), + (ret, error3) -> { + if (error3 != null) { + future.completeExceptionally(error3); + } else { + future.complete(ret); + } + }); + }); }); return future; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegistryEndpointsRefresher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegistryEndpointsRefresher.java index ac7cad275813..cdb2dd92b4fc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegistryEndpointsRefresher.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegistryEndpointsRefresher.java @@ -154,7 +154,7 @@ static RegistryEndpointsRefresher create(Configuration conf, String initialDelay TimeUnit.SECONDS.toMillis(conf.getLong(initialDelaySecsConfigName, periodicRefreshMs / 10))); long minTimeBetweenRefreshesMs = TimeUnit.SECONDS .toMillis(conf.getLong(minIntervalSecsConfigName, MIN_SECS_BETWEEN_REFRESHES_DEFAULT)); - Preconditions.checkArgument(minTimeBetweenRefreshesMs < periodicRefreshMs); + Preconditions.checkArgument(minTimeBetweenRefreshesMs <= periodicRefreshMs); return new RegistryEndpointsRefresher(initialDelayMs, periodicRefreshMs, minTimeBetweenRefreshesMs, refresher); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java index 86cdaaeef3cc..6d1050196d83 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java @@ -791,9 +791,7 @@ public static Result createCompleteResult(Iterable partialResults) throw } prevRow = currentRow; stale = stale || r.isStale(); - for (Cell c : r.rawCells()) { - cells.add(c); - } + Collections.addAll(cells, r.rawCells()); } return Result.create(cells, null, stale); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcConnectionRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcConnectionRegistry.java index 2c320d3a9d1d..c3ed560923ff 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcConnectionRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcConnectionRegistry.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.security.User; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.base.Splitter; @@ -75,9 +76,9 @@ public class RpcConnectionRegistry extends AbstractRpcBasedConnectionRegistry { private final String connectionString; - RpcConnectionRegistry(Configuration conf) throws IOException { - super(conf, HEDGED_REQS_FANOUT_KEY, INITIAL_REFRESH_DELAY_SECS, PERIODIC_REFRESH_INTERVAL_SECS, - MIN_SECS_BETWEEN_REFRESHES); + RpcConnectionRegistry(Configuration conf, User user) throws IOException { + super(conf, user, HEDGED_REQS_FANOUT_KEY, INITIAL_REFRESH_DELAY_SECS, + PERIODIC_REFRESH_INTERVAL_SECS, MIN_SECS_BETWEEN_REFRESHES); connectionString = buildConnectionString(conf); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java index 1c91819ac4b9..817f9e2d4b1b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java @@ -266,6 +266,15 @@ public interface TableDescriptor { */ boolean isReadOnly(); + /** + * The HDFS erasure coding policy for a table. This will be set on the data dir of the table, and + * is an alternative to normal replication which takes less space at the cost of locality. + * @return the current policy, or null if undefined + */ + default String getErasureCodingPolicy() { + return null; + } + /** * Returns Name of this table and then a map of all of the column family descriptors (with only * the non-default column family attributes) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java index 43ca935ffa17..fcdbe4e4ae64 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java @@ -143,6 +143,14 @@ public class TableDescriptorBuilder { private static final Bytes REGION_MEMSTORE_REPLICATION_KEY = new Bytes(Bytes.toBytes(REGION_MEMSTORE_REPLICATION)); + /** + * If non-null, the HDFS erasure coding policy to set on the data dir of the table + */ + public static final String ERASURE_CODING_POLICY = "ERASURE_CODING_POLICY"; + private static final Bytes ERASURE_CODING_POLICY_KEY = + new Bytes(Bytes.toBytes(ERASURE_CODING_POLICY)); + + private static final String DEFAULT_ERASURE_CODING_POLICY = null; /** * Used by shell/rest interface to access this metadata attribute which denotes if the table * should be treated by region normalizer. @@ -226,6 +234,8 @@ public class TableDescriptorBuilder { DEFAULT_VALUES.put(DURABILITY, DEFAULT_DURABLITY.name()); // use the enum name DEFAULT_VALUES.put(REGION_REPLICATION, String.valueOf(DEFAULT_REGION_REPLICATION)); DEFAULT_VALUES.put(PRIORITY, String.valueOf(DEFAULT_PRIORITY)); + // Setting ERASURE_CODING_POLICY to NULL so that it is not considered as metadata + DEFAULT_VALUES.put(ERASURE_CODING_POLICY, String.valueOf(DEFAULT_ERASURE_CODING_POLICY)); DEFAULT_VALUES.keySet().stream().map(s -> new Bytes(Bytes.toBytes(s))) .forEach(RESERVED_KEYWORDS::add); RESERVED_KEYWORDS.add(IS_META_KEY); @@ -490,6 +500,11 @@ public TableDescriptorBuilder setReadOnly(final boolean readOnly) { return this; } + public TableDescriptorBuilder setErasureCodingPolicy(String policy) { + desc.setErasureCodingPolicy(policy); + return this; + } + public TableDescriptorBuilder setRegionMemStoreReplication(boolean memstoreReplication) { desc.setRegionMemStoreReplication(memstoreReplication); return this; @@ -748,6 +763,28 @@ public ModifyableTableDescriptor setReadOnly(final boolean readOnly) { return setValue(READONLY_KEY, Boolean.toString(readOnly)); } + /** + * The HDFS erasure coding policy for a table. This will be set on the data dir of the table, + * and is an alternative to normal replication which takes less space at the cost of locality. + * @return the current policy, or null if undefined + */ + @Override + public String getErasureCodingPolicy() { + return getValue(ERASURE_CODING_POLICY); + } + + /** + * Sets the HDFS erasure coding policy for the table. This will be propagated to HDFS for the + * data dir of the table. Erasure coding is an alternative to normal replication which takes + * less space at the cost of locality. The policy must be available and enabled on the hdfs + * cluster before being set. + * @param policy the policy to set, or null to disable erasure coding + * @return the modifyable TD + */ + public ModifyableTableDescriptor setErasureCodingPolicy(String policy) { + return setValue(ERASURE_CODING_POLICY_KEY, policy); + } + /** * Check if the compaction enable flag of the table is true. If flag is false then no * minor/major compactions will be done in real. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableOverAsyncTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableOverAsyncTable.java index 0a7dabd476ce..8c61e8b584f9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableOverAsyncTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableOverAsyncTable.java @@ -55,9 +55,9 @@ import org.apache.hadoop.hbase.trace.HBaseSemanticAttributes; import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.ConcurrentMapUtils.IOExceptionSupplier; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.hadoop.hbase.util.IOExceptionSupplier; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKConnectionRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKConnectionRegistry.java index 1634b13ec7e8..0e13f0b83c91 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKConnectionRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKConnectionRegistry.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient; import org.apache.hadoop.hbase.zookeeper.ZNodePaths; @@ -60,7 +61,8 @@ class ZKConnectionRegistry implements ConnectionRegistry { private final ZNodePaths znodePaths; - ZKConnectionRegistry(Configuration conf) { + // User not used, but for rpc based registry we need it + ZKConnectionRegistry(Configuration conf, User user) { this.znodePaths = new ZNodePaths(conf); this.zk = new ReadOnlyZKClient(conf); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java index cf730501be0a..ff83584ccb44 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java @@ -50,6 +50,8 @@ protected AtomicLong createCounter(String counterName) { public static final String BLOCK_BYTES_SCANNED_KEY_METRIC_NAME = "BLOCK_BYTES_SCANNED"; + public static final String FS_READ_TIME_METRIC_NAME = "FS_READ_TIME"; + /** * number of rows filtered during scan RPC */ @@ -65,6 +67,8 @@ protected AtomicLong createCounter(String counterName) { public final AtomicLong countOfBlockBytesScanned = createCounter(BLOCK_BYTES_SCANNED_KEY_METRIC_NAME); + public final AtomicLong fsReadTime = createCounter(FS_READ_TIME_METRIC_NAME); + public void setCounter(String counterName, long value) { AtomicLong c = this.counters.get(counterName); if (c != null) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java index fcded9f5b69d..7972cc08acd2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java @@ -209,7 +209,7 @@ private void cleanupIdleConnections() { for (T conn : connections.values()) { // Remove connection if it has not been chosen by anyone for more than maxIdleTime, and the // connection itself has already shutdown. The latter check is because we may still - // have some pending calls on connection so we should not shutdown the connection outside. + // have some pending calls on connection, so we should not shut down the connection outside. // The connection itself will disconnect if there is no pending call for maxIdleTime. if (conn.getLastTouched() < closeBeforeTime && !conn.isActive()) { if (LOG.isTraceEnabled()) { @@ -454,7 +454,7 @@ public void run(Call call) { } } - private static Address createAddr(ServerName sn) { + static Address createAddr(ServerName sn) { return Address.fromParts(sn.getHostname(), sn.getPort()); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java index 81ad4d2f056d..0478000a2375 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hbase.ipc; import static org.apache.hadoop.hbase.ipc.IPCUtil.buildRequestHeader; -import static org.apache.hadoop.hbase.ipc.IPCUtil.createRemoteException; -import static org.apache.hadoop.hbase.ipc.IPCUtil.getTotalSizeWhenWrittenDelimited; -import static org.apache.hadoop.hbase.ipc.IPCUtil.isFatalConnectionException; import static org.apache.hadoop.hbase.ipc.IPCUtil.setCancelled; import static org.apache.hadoop.hbase.ipc.IPCUtil.write; @@ -45,7 +42,6 @@ import java.util.concurrent.ThreadLocalRandom; import javax.security.sasl.SaslException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.exceptions.ConnectionClosingException; import org.apache.hadoop.hbase.io.ByteArrayOutputStream; @@ -66,18 +62,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.protobuf.Message; import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; import org.apache.hbase.thirdparty.io.netty.buffer.ByteBuf; import org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.CellBlockMeta; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ConnectionHeader; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ExceptionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ResponseHeader; /** * Thread that reads responses and notifies callers. Each connection owns a socket connected to a @@ -219,7 +211,7 @@ public void cleanup(IOException e) { BlockingRpcConnection(BlockingRpcClient rpcClient, ConnectionId remoteId) throws IOException { super(rpcClient.conf, AbstractRpcClient.WHEEL_TIMER, remoteId, rpcClient.clusterId, rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.codec, rpcClient.compressor, - rpcClient.metrics, rpcClient.connectionAttributes); + rpcClient.cellBlockBuilder, rpcClient.metrics, rpcClient.connectionAttributes); this.rpcClient = rpcClient; this.connectionHeaderPreamble = getConnectionHeaderPreamble(); ConnectionHeader header = getConnectionHeader(); @@ -436,6 +428,15 @@ public Object run() throws IOException, InterruptedException { }); } + private void getConnectionRegistry(OutputStream outStream) throws IOException { + outStream.write(RpcClient.REGISTRY_PREAMBLE_HEADER); + } + + private void createStreams(InputStream inStream, OutputStream outStream) { + this.in = new DataInputStream(new BufferedInputStream(inStream)); + this.out = new DataOutputStream(new BufferedOutputStream(outStream)); + } + private void setupIOstreams() throws IOException { if (socket != null) { // The connection is already available. Perfect. @@ -463,6 +464,11 @@ private void setupIOstreams() throws IOException { InputStream inStream = NetUtils.getInputStream(socket); // This creates a socket with a write timeout. This timeout cannot be changed. OutputStream outStream = NetUtils.getOutputStream(socket, this.rpcClient.writeTO); + if (connectionRegistryCall != null) { + getConnectionRegistry(outStream); + createStreams(inStream, outStream); + break; + } // Write out the preamble -- MAGIC, version, and auth to use. writeConnectionHeaderPreamble(outStream); if (useSasl) { @@ -495,13 +501,11 @@ public Boolean run() throws IOException { // reconnecting because regionserver may change its sasl config after restart. } } - this.in = new DataInputStream(new BufferedInputStream(inStream)); - this.out = new DataOutputStream(new BufferedOutputStream(outStream)); + createStreams(inStream, outStream); // Now write out the connection header writeConnectionHeader(); // process the response from server for connection header if necessary processResponseForConnectionHeader(); - break; } } catch (Throwable t) { @@ -612,7 +616,9 @@ private void writeRequest(Call call) throws IOException { cellBlockMeta = null; } RequestHeader requestHeader = buildRequestHeader(call, cellBlockMeta); - + if (call.isConnectionRegistryCall()) { + connectionRegistryCall = call; + } setupIOstreams(); // Now we're going to write the call. We take the lock, then check that the connection @@ -647,70 +653,13 @@ private void writeRequest(Call call) throws IOException { * Receive a response. Because only one receiver, so no synchronization on in. */ private void readResponse() { - Call call = null; - boolean expectedCall = false; try { - // See HBaseServer.Call.setResponse for where we write out the response. - // Total size of the response. Unused. But have to read it in anyways. - int totalSize = in.readInt(); - - // Read the header - ResponseHeader responseHeader = ResponseHeader.parseDelimitedFrom(in); - int id = responseHeader.getCallId(); - call = calls.remove(id); // call.done have to be set before leaving this method - expectedCall = (call != null && !call.isDone()); - if (!expectedCall) { - // So we got a response for which we have no corresponding 'call' here on the client-side. - // We probably timed out waiting, cleaned up all references, and now the server decides - // to return a response. There is nothing we can do w/ the response at this stage. Clean - // out the wire of the response so its out of the way and we can get other responses on - // this connection. - int readSoFar = getTotalSizeWhenWrittenDelimited(responseHeader); - int whatIsLeftToRead = totalSize - readSoFar; - IOUtils.skipFully(in, whatIsLeftToRead); - if (call != null) { - call.callStats.setResponseSizeBytes(totalSize); - call.callStats - .setCallTimeMs(EnvironmentEdgeManager.currentTime() - call.callStats.getStartTime()); - } - return; - } - if (responseHeader.hasException()) { - ExceptionResponse exceptionResponse = responseHeader.getException(); - RemoteException re = createRemoteException(exceptionResponse); - call.setException(re); - call.callStats.setResponseSizeBytes(totalSize); - call.callStats - .setCallTimeMs(EnvironmentEdgeManager.currentTime() - call.callStats.getStartTime()); - if (isFatalConnectionException(exceptionResponse)) { - synchronized (this) { - closeConn(re); - } - } - } else { - Message value = null; - if (call.responseDefaultType != null) { - Message.Builder builder = call.responseDefaultType.newBuilderForType(); - ProtobufUtil.mergeDelimitedFrom(builder, in); - value = builder.build(); - } - CellScanner cellBlockScanner = null; - if (responseHeader.hasCellBlockMeta()) { - int size = responseHeader.getCellBlockMeta().getLength(); - byte[] cellBlock = new byte[size]; - IOUtils.readFully(this.in, cellBlock, 0, cellBlock.length); - cellBlockScanner = this.rpcClient.cellBlockBuilder.createCellScanner(this.codec, - this.compressor, cellBlock); + readResponse(in, calls, remoteExc -> { + synchronized (this) { + closeConn(remoteExc); } - call.setResponse(value, cellBlockScanner); - call.callStats.setResponseSizeBytes(totalSize); - call.callStats - .setCallTimeMs(EnvironmentEdgeManager.currentTime() - call.callStats.getStartTime()); - } + }); } catch (IOException e) { - if (expectedCall) { - call.setException(e); - } if (e instanceof SocketTimeoutException) { // Clean up open calls but don't treat this as a fatal condition, // since we expect certain responses to not make it by the specified diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java index 669fc73a3bfa..d175ea0b6e90 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java @@ -34,6 +34,7 @@ import org.apache.hbase.thirdparty.io.netty.util.Timeout; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.ConnectionRegistryService; /** A call waiting for a value. */ @InterfaceAudience.Private @@ -156,4 +157,8 @@ public synchronized boolean isDone() { public long getStartTime() { return this.callStats.getStartTime(); } + + public boolean isConnectionRegistryCall() { + return md.getService().equals(ConnectionRegistryService.getDescriptor()); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java index d6df6c974ccf..bf4b833e856c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java @@ -41,6 +41,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.ipc.RemoteException; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream; @@ -62,6 +64,8 @@ @InterfaceAudience.Private class IPCUtil { + private static final Logger LOG = LoggerFactory.getLogger(IPCUtil.class); + /** * Write out header, param, and cell block if there is one. * @param dos Stream to write into @@ -159,8 +163,19 @@ static RemoteException createRemoteException(final ExceptionResponse e) { } /** Returns True if the exception is a fatal connection exception. */ - static boolean isFatalConnectionException(final ExceptionResponse e) { - return e.getExceptionClassName().equals(FatalConnectionException.class.getName()); + static boolean isFatalConnectionException(ExceptionResponse e) { + if (e.getExceptionClassName().equals(FatalConnectionException.class.getName())) { + return true; + } + // try our best to check for sub classes of FatalConnectionException + try { + return e.getExceptionClassName() != null && FatalConnectionException.class.isAssignableFrom( + Class.forName(e.getExceptionClassName(), false, IPCUtil.class.getClassLoader())); + // Class.forName may throw ExceptionInInitializerError so we have to catch Throwable here + } catch (Throwable t) { + LOG.debug("Can not get class object for {}", e.getExceptionClassName(), t); + return false; + } } static IOException toIOE(Throwable t) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java index 408ea347e7a3..a0f8f10d1cf9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java @@ -104,7 +104,7 @@ class NettyRpcConnection extends RpcConnection { NettyRpcConnection(NettyRpcClient rpcClient, ConnectionId remoteId) throws IOException { super(rpcClient.conf, AbstractRpcClient.WHEEL_TIMER, remoteId, rpcClient.clusterId, rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.codec, rpcClient.compressor, - rpcClient.metrics, rpcClient.connectionAttributes); + rpcClient.cellBlockBuilder, rpcClient.metrics, rpcClient.connectionAttributes); this.rpcClient = rpcClient; this.eventLoop = rpcClient.group.next(); byte[] connectionHeaderPreamble = getConnectionHeaderPreamble(); @@ -274,6 +274,12 @@ public void operationComplete(Future future) throws Exception { }); } + private void getConnectionRegistry(Channel ch) throws IOException { + established(ch); + NettyFutureUtils.safeWriteAndFlush(ch, + Unpooled.directBuffer(6).writeBytes(RpcClient.REGISTRY_PREAMBLE_HEADER)); + } + private void connect() throws UnknownHostException { assert eventLoop.inEventLoop(); LOG.trace("Connecting to {}", remoteId.getAddress()); @@ -303,12 +309,16 @@ protected void initChannel(Channel ch) throws Exception { .addListener(new ChannelFutureListener() { private void succeed(Channel ch) throws IOException { - ch.writeAndFlush(connectionHeaderPreamble.retainedDuplicate()); + if (connectionRegistryCall != null) { + getConnectionRegistry(ch); + return; + } + NettyFutureUtils.safeWriteAndFlush(ch, connectionHeaderPreamble.retainedDuplicate()); if (useSasl) { saslNegotiate(ch); } else { // send the connection header to server - ch.write(connectionHeaderWithLength.retainedDuplicate()); + NettyFutureUtils.safeWrite(ch, connectionHeaderWithLength.retainedDuplicate()); established(ch); } } @@ -317,6 +327,9 @@ private void fail(Channel ch, Throwable error) { IOException ex = toIOE(error); LOG.warn("Exception encountered while connecting to the server " + remoteId.getAddress(), ex); + if (connectionRegistryCall != null) { + connectionRegistryCall.setException(ex); + } failInit(ch, ex); rpcClient.failedServers.addToFailedServers(remoteId.getAddress(), error); } @@ -346,6 +359,13 @@ public void operationComplete(ChannelFuture future) throws Exception { private void sendRequest0(Call call, HBaseRpcController hrc) throws IOException { assert eventLoop.inEventLoop(); + if (call.isConnectionRegistryCall()) { + connectionRegistryCall = call; + // For get connection registry call, we will send a special preamble header to get the + // response, instead of sending a real rpc call. See HBASE-25051 + connect(); + return; + } if (reloginInProgress) { throw new IOException(RpcConnectionConstants.RELOGIN_IS_IN_PROGRESS); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java index ad8c51568a32..44772ae2dbf9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcDuplexHandler.java @@ -18,21 +18,16 @@ package org.apache.hadoop.hbase.ipc; import io.opentelemetry.context.Scope; -import java.io.EOFException; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.exceptions.ConnectionClosedException; import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.ipc.RemoteException; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.protobuf.Message; -import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat; import org.apache.hbase.thirdparty.io.netty.buffer.ByteBuf; import org.apache.hbase.thirdparty.io.netty.buffer.ByteBufInputStream; import org.apache.hbase.thirdparty.io.netty.buffer.ByteBufOutputStream; @@ -44,9 +39,7 @@ import org.apache.hbase.thirdparty.io.netty.util.concurrent.PromiseCombiner; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.CellBlockMeta; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ExceptionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ResponseHeader; /** * The netty rpc handler. @@ -127,88 +120,15 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) } } - private void finishCall(ResponseHeader responseHeader, ByteBufInputStream in, Call call) - throws IOException { - Message value; - if (call.responseDefaultType != null) { - Message.Builder builder = call.responseDefaultType.newBuilderForType(); - if (!builder.mergeDelimitedFrom(in)) { - // The javadoc of mergeDelimitedFrom says returning false means the stream reaches EOF - // before reading any bytes out, so here we need to manually finish create the EOFException - // and finish the call - call.setException(new EOFException("EOF while reading response with type: " - + call.responseDefaultType.getClass().getName())); - return; - } - value = builder.build(); - } else { - value = null; - } - CellScanner cellBlockScanner; - if (responseHeader.hasCellBlockMeta()) { - int size = responseHeader.getCellBlockMeta().getLength(); - // Maybe we could read directly from the ByteBuf. - // The problem here is that we do not know when to release it. - byte[] cellBlock = new byte[size]; - in.readFully(cellBlock); - cellBlockScanner = cellBlockBuilder.createCellScanner(this.codec, this.compressor, cellBlock); - } else { - cellBlockScanner = null; - } - call.setResponse(value, cellBlockScanner); - } - private void readResponse(ChannelHandlerContext ctx, ByteBuf buf) throws IOException { - int totalSize = buf.readInt(); - ByteBufInputStream in = new ByteBufInputStream(buf); - ResponseHeader responseHeader = ResponseHeader.parseDelimitedFrom(in); - int id = responseHeader.getCallId(); - if (LOG.isTraceEnabled()) { - LOG.trace("got response header " + TextFormat.shortDebugString(responseHeader) - + ", totalSize: " + totalSize + " bytes"); - } - RemoteException remoteExc; - if (responseHeader.hasException()) { - ExceptionResponse exceptionResponse = responseHeader.getException(); - remoteExc = IPCUtil.createRemoteException(exceptionResponse); - if (IPCUtil.isFatalConnectionException(exceptionResponse)) { - // Here we will cleanup all calls so do not need to fall back, just return. - exceptionCaught(ctx, remoteExc); - return; - } - } else { - remoteExc = null; - } - Call call = id2Call.remove(id); - if (call == null) { - // So we got a response for which we have no corresponding 'call' here on the client-side. - // We probably timed out waiting, cleaned up all references, and now the server decides - // to return a response. There is nothing we can do w/ the response at this stage. Clean - // out the wire of the response so its out of the way and we can get other responses on - // this connection. - if (LOG.isDebugEnabled()) { - int readSoFar = IPCUtil.getTotalSizeWhenWrittenDelimited(responseHeader); - int whatIsLeftToRead = totalSize - readSoFar; - LOG.debug("Unknown callId: " + id + ", skipping over this response of " + whatIsLeftToRead - + " bytes"); - } - return; - } - call.callStats.setResponseSizeBytes(totalSize); - if (remoteExc != null) { - call.setException(remoteExc); - return; - } try { - finishCall(responseHeader, in, call); + conn.readResponse(new ByteBufInputStream(buf), id2Call, + remoteExc -> exceptionCaught(ctx, remoteExc)); } catch (IOException e) { - // As the call has been removed from id2Call map, if we hit an exception here, the - // exceptionCaught method can not help us finish the call, so here we need to catch the - // exception and finish it - // And in netty, the decoding the frame based, when reaching here we have already read a full + // In netty, the decoding the frame based, when reaching here we have already read a full // frame, so hitting exception here does not mean the stream decoding is broken, thus we do // not need to throw the exception out and close the connection. - call.setException(e); + LOG.warn("failed to process response", e); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java index 045216e88811..369430e337ae 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java @@ -54,6 +54,8 @@ public interface RpcClient extends Closeable { // The client in 0.99+ does not ping the server. int PING_CALL_ID = -1; + byte[] REGISTRY_PREAMBLE_HEADER = new byte[] { 'R', 'e', 'g', 'i', 's', 't' }; + /** * Creates a "channel" that can be used by a blocking protobuf service. Useful setting up protobuf * blocking stubs. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java index 31698a1a1e8e..65f936d6fc38 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcConnection.java @@ -17,12 +17,17 @@ */ package org.apache.hadoop.hbase.ipc; +import java.io.DataInput; +import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.MetricsConnection; import org.apache.hadoop.hbase.codec.Codec; @@ -34,12 +39,15 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; +import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer; import org.apache.hbase.thirdparty.io.netty.util.Timeout; @@ -48,6 +56,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ConnectionHeader; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ExceptionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ResponseHeader; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.UserInformation; /** @@ -72,6 +82,8 @@ abstract class RpcConnection { protected final CompressionCodec compressor; + protected final CellBlockBuilder cellBlockBuilder; + protected final MetricsConnection metrics; private final Map connectionAttributes; @@ -90,10 +102,12 @@ abstract class RpcConnection { protected RpcConnection(Configuration conf, HashedWheelTimer timeoutTimer, ConnectionId remoteId, String clusterId, boolean isSecurityEnabled, Codec codec, CompressionCodec compressor, - MetricsConnection metrics, Map connectionAttributes) throws IOException { + CellBlockBuilder cellBlockBuilder, MetricsConnection metrics, + Map connectionAttributes) throws IOException { this.timeoutTimer = timeoutTimer; this.codec = codec; this.compressor = compressor; + this.cellBlockBuilder = cellBlockBuilder; this.conf = conf; this.metrics = metrics; this.connectionAttributes = connectionAttributes; @@ -145,18 +159,18 @@ public void run(Timeout timeout) throws Exception { } } - protected final byte[] getConnectionHeaderPreamble() { + // will be overridden in tests + protected byte[] getConnectionHeaderPreamble() { // Assemble the preamble up in a buffer first and then send it. Writing individual elements, // they are getting sent across piecemeal according to wireshark and then server is messing // up the reading on occasion (the passed in stream is not buffered yet). - - // Preamble is six bytes -- 'HBas' + VERSION + AUTH_CODE int rpcHeaderLen = HConstants.RPC_HEADER.length; + // Preamble is six bytes -- 'HBas' + VERSION + AUTH_CODE byte[] preamble = new byte[rpcHeaderLen + 2]; System.arraycopy(HConstants.RPC_HEADER, 0, preamble, 0, rpcHeaderLen); preamble[rpcHeaderLen] = HConstants.RPC_CURRENT_VERSION; synchronized (this) { - preamble[rpcHeaderLen + 1] = provider.getSaslAuthMethod().getCode(); + preamble[preamble.length - 1] = provider.getSaslAuthMethod().getCode(); } return preamble; } @@ -237,4 +251,103 @@ public void setLastTouched(long lastTouched) { * Does the clean up work after the connection is removed from the connection pool */ public abstract void cleanupConnection(); + + protected Call connectionRegistryCall; + + private void finishCall(ResponseHeader responseHeader, T in, + Call call) throws IOException { + Message value; + if (call.responseDefaultType != null) { + Message.Builder builder = call.responseDefaultType.newBuilderForType(); + if (!builder.mergeDelimitedFrom(in)) { + // The javadoc of mergeDelimitedFrom says returning false means the stream reaches EOF + // before reading any bytes out, so here we need to manually finish create the EOFException + // and finish the call + call.setException(new EOFException("EOF while reading response with type: " + + call.responseDefaultType.getClass().getName())); + return; + } + value = builder.build(); + } else { + value = null; + } + CellScanner cellBlockScanner; + if (responseHeader.hasCellBlockMeta()) { + int size = responseHeader.getCellBlockMeta().getLength(); + // Maybe we could read directly from the ByteBuf. + // The problem here is that we do not know when to release it. + byte[] cellBlock = new byte[size]; + in.readFully(cellBlock); + cellBlockScanner = cellBlockBuilder.createCellScanner(this.codec, this.compressor, cellBlock); + } else { + cellBlockScanner = null; + } + call.setResponse(value, cellBlockScanner); + } + + void readResponse(T in, Map id2Call, + Consumer fatalConnectionErrorConsumer) throws IOException { + int totalSize = in.readInt(); + ResponseHeader responseHeader = ResponseHeader.parseDelimitedFrom(in); + int id = responseHeader.getCallId(); + if (LOG.isTraceEnabled()) { + LOG.trace("got response header " + TextFormat.shortDebugString(responseHeader) + + ", totalSize: " + totalSize + " bytes"); + } + RemoteException remoteExc; + if (responseHeader.hasException()) { + ExceptionResponse exceptionResponse = responseHeader.getException(); + remoteExc = IPCUtil.createRemoteException(exceptionResponse); + if (IPCUtil.isFatalConnectionException(exceptionResponse)) { + // Here we will cleanup all calls so do not need to fall back, just return. + fatalConnectionErrorConsumer.accept(remoteExc); + if (connectionRegistryCall != null) { + connectionRegistryCall.setException(remoteExc); + connectionRegistryCall = null; + } + return; + } + } else { + remoteExc = null; + } + if (id < 0) { + if (connectionRegistryCall != null) { + LOG.debug("process connection registry call"); + finishCall(responseHeader, in, connectionRegistryCall); + connectionRegistryCall = null; + return; + } + } + Call call = id2Call.remove(id); + if (call == null) { + // So we got a response for which we have no corresponding 'call' here on the client-side. + // We probably timed out waiting, cleaned up all references, and now the server decides + // to return a response. There is nothing we can do w/ the response at this stage. Clean + // out the wire of the response so its out of the way and we can get other responses on + // this connection. + if (LOG.isDebugEnabled()) { + int readSoFar = IPCUtil.getTotalSizeWhenWrittenDelimited(responseHeader); + int whatIsLeftToRead = totalSize - readSoFar; + LOG.debug("Unknown callId: " + id + ", skipping over this response of " + whatIsLeftToRead + + " bytes"); + } + return; + } + call.callStats.setResponseSizeBytes(totalSize); + if (remoteExc != null) { + call.setException(remoteExc); + return; + } + try { + finishCall(responseHeader, in, call); + } catch (IOException e) { + // As the call has been removed from id2Call map, if we hit an exception here, the + // exceptionCaught method can not help us finish the call, so here we need to catch the + // exception and finish it + call.setException(e); + // throw the exception out, the upper layer should determine whether this is a critical + // problem + throw e; + } + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/NettyHBaseSaslRpcClientHandler.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/NettyHBaseSaslRpcClientHandler.java index 48e631c76299..cc71355d4297 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/NettyHBaseSaslRpcClientHandler.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/NettyHBaseSaslRpcClientHandler.java @@ -145,6 +145,12 @@ public byte[] run() throws Exception { // Mechanisms which have multiple steps will not return true on `SaslClient#isComplete()` // until the handshake has fully completed. Mechanisms which only send a single buffer may // return true on `isComplete()` after that initial response is calculated. + + // HBASE-28337 We still want to check if the SaslClient completed the handshake, because + // there are certain mechs like PLAIN which doesn't have a server response after the + // initial authentication request. We cannot remove this tryComplete(), otherwise mechs + // like PLAIN will fail with call timeout. + tryComplete(ctx); } catch (Exception e) { // the exception thrown by handlerAdded will not be passed to the exceptionCaught below // because netty will remove a handler if handlerAdded throws an exception. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecurityInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecurityInfo.java index dbb4c83844a4..2e16d5646953 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecurityInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecurityInfo.java @@ -23,7 +23,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.TokenIdentifier.Kind; +import org.apache.hadoop.hbase.shaded.protobuf.generated.BootstrapNodeProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos; @@ -50,6 +52,10 @@ public class SecurityInfo { new SecurityInfo(SecurityConstants.MASTER_KRB_PRINCIPAL, Kind.HBASE_AUTH_TOKEN)); infos.put(RegistryProtos.ClientMetaService.getDescriptor().getName(), new SecurityInfo(SecurityConstants.MASTER_KRB_PRINCIPAL, Kind.HBASE_AUTH_TOKEN)); + infos.put(BootstrapNodeProtos.BootstrapNodeService.getDescriptor().getName(), + new SecurityInfo(SecurityConstants.REGIONSERVER_KRB_PRINCIPAL, Kind.HBASE_AUTH_TOKEN)); + infos.put(LockServiceProtos.LockService.getDescriptor().getName(), + new SecurityInfo(SecurityConstants.MASTER_KRB_PRINCIPAL, Kind.HBASE_AUTH_TOKEN)); // NOTE: IF ADDING A NEW SERVICE, BE SURE TO UPDATE HBasePolicyProvider ALSO ELSE // new Service will not be found when all is Kerberized!!!! } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/BuiltInSaslAuthenticationProvider.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/BuiltInSaslAuthenticationProvider.java index 712d4035448b..b573a5ee771e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/BuiltInSaslAuthenticationProvider.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/BuiltInSaslAuthenticationProvider.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.security.provider; +import org.apache.hadoop.hbase.security.AuthMethod; import org.apache.yetus.audience.InterfaceAudience; /** @@ -35,4 +36,9 @@ public abstract class BuiltInSaslAuthenticationProvider implements SaslAuthentic public String getTokenKind() { return AUTH_TOKEN_TYPE; } + + protected static SaslAuthMethod createSaslAuthMethod(AuthMethod authMethod) { + return new SaslAuthMethod(authMethod.name(), authMethod.code, authMethod.mechanismName, + authMethod.authenticationMethod); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/DigestSaslAuthenticationProvider.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/DigestSaslAuthenticationProvider.java index d71c07d1575a..f22a06474aef 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/DigestSaslAuthenticationProvider.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/DigestSaslAuthenticationProvider.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.security.provider; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.hbase.security.AuthMethod; import org.apache.yetus.audience.InterfaceAudience; /** @@ -26,8 +26,7 @@ @InterfaceAudience.Private public class DigestSaslAuthenticationProvider extends BuiltInSaslAuthenticationProvider { - public static final SaslAuthMethod SASL_AUTH_METHOD = - new SaslAuthMethod("DIGEST", (byte) 82, "DIGEST-MD5", AuthenticationMethod.TOKEN); + public static final SaslAuthMethod SASL_AUTH_METHOD = createSaslAuthMethod(AuthMethod.DIGEST); @Override public SaslAuthMethod getSaslAuthMethod() { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/GssSaslAuthenticationProvider.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/GssSaslAuthenticationProvider.java index 7dea40f2657a..df6fce859b7f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/GssSaslAuthenticationProvider.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/GssSaslAuthenticationProvider.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.security.provider; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.hbase.security.AuthMethod; import org.apache.yetus.audience.InterfaceAudience; /** @@ -26,8 +26,7 @@ @InterfaceAudience.Private public class GssSaslAuthenticationProvider extends BuiltInSaslAuthenticationProvider { - public static final SaslAuthMethod SASL_AUTH_METHOD = - new SaslAuthMethod("KERBEROS", (byte) 81, "GSSAPI", AuthenticationMethod.KERBEROS); + public static final SaslAuthMethod SASL_AUTH_METHOD = createSaslAuthMethod(AuthMethod.KERBEROS); @Override public SaslAuthMethod getSaslAuthMethod() { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SimpleSaslAuthenticationProvider.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SimpleSaslAuthenticationProvider.java index 01b1f452685a..9d79b648c6e4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SimpleSaslAuthenticationProvider.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/provider/SimpleSaslAuthenticationProvider.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.security.provider; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.hbase.security.AuthMethod; import org.apache.yetus.audience.InterfaceAudience; /** @@ -25,8 +25,8 @@ */ @InterfaceAudience.Private public class SimpleSaslAuthenticationProvider extends BuiltInSaslAuthenticationProvider { - public static final SaslAuthMethod SASL_AUTH_METHOD = - new SaslAuthMethod("SIMPLE", (byte) 80, "", AuthenticationMethod.SIMPLE); + + public static final SaslAuthMethod SASL_AUTH_METHOD = createSaslAuthMethod(AuthMethod.SIMPLE); @Override public SaslAuthMethod getSaslAuthMethod() { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 598ad932e679..e50b54e8eb02 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -3462,6 +3462,7 @@ private static LogEntry getSlowLogRecord(final TooSlowLog.SlowLogPayload slowLog .setQueueTime(slowLogPayload.getQueueTime()).setRegionName(slowLogPayload.getRegionName()) .setResponseSize(slowLogPayload.getResponseSize()) .setBlockBytesScanned(slowLogPayload.getBlockBytesScanned()) + .setFsReadTime(slowLogPayload.getFsReadTime()) .setServerClass(slowLogPayload.getServerClass()).setStartTime(slowLogPayload.getStartTime()) .setUserName(slowLogPayload.getUserName()) .setRequestAttributes(convertNameBytesPairsToMap(slowLogPayload.getRequestAttributeList())) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 377b46494633..ce12aaea0d24 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -1590,17 +1590,17 @@ private static List toProtoServerNames(List // HBCK2 public static MasterProtos.AssignsRequest toAssignRegionsRequest(List encodedRegionNames, - boolean override) { + boolean override, boolean force) { MasterProtos.AssignsRequest.Builder b = MasterProtos.AssignsRequest.newBuilder(); return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)) - .setOverride(override).build(); + .setOverride(override).setForce(force).build(); } public static MasterProtos.UnassignsRequest - toUnassignRegionsRequest(List encodedRegionNames, boolean override) { + toUnassignRegionsRequest(List encodedRegionNames, boolean override, boolean force) { MasterProtos.UnassignsRequest.Builder b = MasterProtos.UnassignsRequest.newBuilder(); return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)) - .setOverride(override).build(); + .setOverride(override).setForce(force).build(); } public static MasterProtos.ScheduleServerCrashProcedureRequest diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/DoNothingConnectionRegistry.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/DoNothingConnectionRegistry.java index 3b792a5bd15f..30d69d4b3f9e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/DoNothingConnectionRegistry.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/DoNothingConnectionRegistry.java @@ -21,15 +21,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.security.User; import org.apache.yetus.audience.InterfaceAudience; /** * Registry that does nothing. Otherwise, default Registry wants zookeeper up and running. */ @InterfaceAudience.Private -class DoNothingConnectionRegistry implements ConnectionRegistry { +public class DoNothingConnectionRegistry implements ConnectionRegistry { - public DoNothingConnectionRegistry(Configuration conf) { + public DoNothingConnectionRegistry(Configuration conf, User user) { } @Override diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java index 4bf425ed562e..f65c7ccb6e75 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminRpcPriority.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.HBaseRpcController; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -141,9 +142,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable { } }).when(adminStub).stopServer(any(HBaseRpcController.class), any(StopServerRequest.class), any()); - - conn = new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF), "test", null, - UserProvider.instantiate(CONF).getCurrent()) { + User user = UserProvider.instantiate(CONF).getCurrent(); + conn = new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF, user), "test", null, + user) { @Override CompletableFuture getMasterStub() { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java index ff4a92ae394d..e56fffbb2642 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncConnectionTracing.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -63,15 +64,15 @@ public class TestAsyncConnectionTracing { @Before public void setUp() throws IOException { - ConnectionRegistry registry = new DoNothingConnectionRegistry(CONF) { + User user = UserProvider.instantiate(CONF).getCurrent(); + ConnectionRegistry registry = new DoNothingConnectionRegistry(CONF, user) { @Override public CompletableFuture getActiveMaster() { return CompletableFuture.completedFuture(masterServer); } }; - conn = new AsyncConnectionImpl(CONF, registry, "test", null, - UserProvider.instantiate(CONF).getCurrent()); + conn = new AsyncConnectionImpl(CONF, registry, "test", null, user); } @After diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java index b306500c8b13..6380f1f6fb0f 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocatorFailFast.java @@ -24,6 +24,8 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FutureUtils; @@ -45,8 +47,8 @@ public class TestAsyncMetaRegionLocatorFailFast { private static final class FaultyConnectionRegistry extends DoNothingConnectionRegistry { - public FaultyConnectionRegistry(Configuration conf) { - super(conf); + public FaultyConnectionRegistry(Configuration conf, User user) { + super(conf, user); } @Override @@ -56,8 +58,9 @@ public CompletableFuture getMetaRegionLocations() { } @BeforeClass - public static void setUp() { - LOCATOR = new AsyncMetaRegionLocator(new FaultyConnectionRegistry(CONF)); + public static void setUp() throws IOException { + LOCATOR = new AsyncMetaRegionLocator( + new FaultyConnectionRegistry(CONF, UserProvider.instantiate(CONF).getCurrent())); } @Test(expected = DoNotRetryIOException.class) diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java index 335894303c08..a7df92999d08 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -92,13 +93,14 @@ public void setUp() throws IOException { ServerName.valueOf("127.0.0.2", 12345, EnvironmentEdgeManager.currentTime())), new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(metaRegionInfo, 2), ServerName.valueOf("127.0.0.3", 12345, EnvironmentEdgeManager.currentTime()))); - conn = new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF) { + User user = UserProvider.instantiate(CONF).getCurrent(); + conn = new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF, user) { @Override public CompletableFuture getMetaRegionLocations() { return CompletableFuture.completedFuture(locs); } - }, "test", null, UserProvider.instantiate(CONF).getCurrent()); + }, "test", null, user); } @After diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java index e57967ae7211..cb5431c35d3e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRpcPriority.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.HBaseRpcController; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -160,8 +161,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable { return null; } }).when(stub).get(any(HBaseRpcController.class), any(GetRequest.class), any()); - conn = new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF), "test", null, - UserProvider.instantiate(CONF).getCurrent()) { + User user = UserProvider.instantiate(CONF).getCurrent(); + conn = new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF, user), "test", null, + user) { @Override AsyncRegionLocator getLocator() { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java index f9b86221af1e..2cecc974b6ef 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableTracing.java @@ -209,37 +209,37 @@ public Void answer(InvocationOnMock invocation) throws Throwable { } }).when(stub).get(any(HBaseRpcController.class), any(GetRequest.class), any()); final User user = UserProvider.instantiate(CONF).getCurrent(); - conn = - new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF), "test", null, user) { - - @Override - AsyncRegionLocator getLocator() { - AsyncRegionLocator locator = mock(AsyncRegionLocator.class); - Answer> answer = - new Answer>() { - - @Override - public CompletableFuture answer(InvocationOnMock invocation) - throws Throwable { - TableName tableName = invocation.getArgument(0); - RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); - ServerName serverName = ServerName.valueOf("rs", 16010, 12345); - HRegionLocation loc = new HRegionLocation(info, serverName); - return CompletableFuture.completedFuture(loc); - } - }; - doAnswer(answer).when(locator).getRegionLocation(any(TableName.class), any(byte[].class), - any(RegionLocateType.class), anyLong()); - doAnswer(answer).when(locator).getRegionLocation(any(TableName.class), any(byte[].class), - anyInt(), any(RegionLocateType.class), anyLong()); - return locator; - } + conn = new AsyncConnectionImpl(CONF, new DoNothingConnectionRegistry(CONF, user), "test", null, + user) { - @Override - ClientService.Interface getRegionServerStub(ServerName serverName) throws IOException { - return stub; - } - }; + @Override + AsyncRegionLocator getLocator() { + AsyncRegionLocator locator = mock(AsyncRegionLocator.class); + Answer> answer = + new Answer>() { + + @Override + public CompletableFuture answer(InvocationOnMock invocation) + throws Throwable { + TableName tableName = invocation.getArgument(0); + RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); + ServerName serverName = ServerName.valueOf("rs", 16010, 12345); + HRegionLocation loc = new HRegionLocation(info, serverName); + return CompletableFuture.completedFuture(loc); + } + }; + doAnswer(answer).when(locator).getRegionLocation(any(TableName.class), any(byte[].class), + any(RegionLocateType.class), anyLong()); + doAnswer(answer).when(locator).getRegionLocation(any(TableName.class), any(byte[].class), + anyInt(), any(RegionLocateType.class), anyLong()); + return locator; + } + + @Override + ClientService.Interface getRegionServerStub(ServerName serverName) throws IOException { + return stub; + } + }; table = conn.getTable(TableName.valueOf("table"), ForkJoinPool.commonPool()); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java index a2df7e932395..bd2ca9867f34 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestConnectionRegistryLeak.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FutureUtils; @@ -49,8 +50,8 @@ public static final class ConnectionRegistryForTest extends DoNothingConnectionR private boolean closed = false; - public ConnectionRegistryForTest(Configuration conf) { - super(conf); + public ConnectionRegistryForTest(Configuration conf, User user) { + super(conf, user); CREATED.add(this); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java index fe753973ae20..a16993d56591 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOnlineLogRecord.java @@ -46,15 +46,15 @@ public void itSerializesScan() { scan.withStopRow(Bytes.toBytes(456)); String expectedOutput = "{\n" + " \"startTime\": 1,\n" + " \"processingTime\": 2,\n" + " \"queueTime\": 3,\n" + " \"responseSize\": 4,\n" + " \"blockBytesScanned\": 5,\n" - + " \"multiGetsCount\": 6,\n" + " \"multiMutationsCount\": 7,\n" + " \"scan\": {\n" - + " \"startRow\": \"\\\\x00\\\\x00\\\\x00{\",\n" + + " \"fsReadTime\": 6,\n" + " \"multiGetsCount\": 6,\n" + " \"multiMutationsCount\": 7,\n" + + " \"scan\": {\n" + " \"startRow\": \"\\\\x00\\\\x00\\\\x00{\",\n" + " \"stopRow\": \"\\\\x00\\\\x00\\\\x01\\\\xC8\",\n" + " \"batch\": -1,\n" + " \"cacheBlocks\": true,\n" + " \"totalColumns\": 0,\n" + " \"maxResultSize\": -1,\n" + " \"families\": {},\n" + " \"caching\": -1,\n" + " \"maxVersions\": 1,\n" + " \"timeRange\": [\n" + " 0,\n" + " 9223372036854775807\n" + " ]\n" + " }\n" + "}"; - OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, null, null, null, null, null, null, null, - 6, 7, 0, scan, Collections.emptyMap(), Collections.emptyMap()); + OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, 6, null, null, null, null, null, null, + null, 6, 7, 0, scan, Collections.emptyMap(), Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); Assert.assertEquals(actualOutput, expectedOutput); @@ -67,8 +67,8 @@ public void itSerializesRequestAttributes() { Set expectedOutputs = ImmutableSet. builder().add("requestAttributes").add("\"r\": \"1\"") .add("\"2\": \"\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\"").build(); - OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, null, null, null, null, null, null, null, - 6, 7, 0, null, requestAttributes, Collections.emptyMap()); + OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, 6, null, null, null, null, null, null, + null, 6, 7, 0, null, requestAttributes, Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); expectedOutputs.forEach(expected -> Assert.assertTrue(actualOutput.contains(expected))); @@ -76,8 +76,8 @@ ImmutableSet. builder().add("requestAttributes").add("\"r\": \"1\"") @Test public void itOmitsEmptyRequestAttributes() { - OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, null, null, null, null, null, null, null, - 6, 7, 0, null, Collections.emptyMap(), Collections.emptyMap()); + OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, 6, null, null, null, null, null, null, + null, 6, 7, 0, null, Collections.emptyMap(), Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); Assert.assertFalse(actualOutput.contains("requestAttributes")); @@ -90,8 +90,8 @@ public void itSerializesConnectionAttributes() { Set expectedOutputs = ImmutableSet. builder().add("connectionAttributes").add("\"c\": \"1\"") .add("\"2\": \"\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\"").build(); - OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, null, null, null, null, null, null, null, - 6, 7, 0, null, Collections.emptyMap(), connectionAttributes); + OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, 6, null, null, null, null, null, null, + null, 6, 7, 0, null, Collections.emptyMap(), connectionAttributes); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); expectedOutputs.forEach(expected -> Assert.assertTrue(actualOutput.contains(expected))); @@ -99,8 +99,8 @@ ImmutableSet. builder().add("connectionAttributes").add("\"c\": \"1\"") @Test public void itOmitsEmptyConnectionAttributes() { - OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, null, null, null, null, null, null, null, - 6, 7, 0, null, Collections.emptyMap(), Collections.emptyMap()); + OnlineLogRecord o = new OnlineLogRecord(1, 2, 3, 4, 5, 6, null, null, null, null, null, null, + null, 6, 7, 0, null, Collections.emptyMap(), Collections.emptyMap()); String actualOutput = o.toJsonPrettyPrint(); System.out.println(actualOutput); Assert.assertFalse(actualOutput.contains("connectionAttributes")); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java index 54b351f00a3b..08c56fe95868 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java @@ -58,7 +58,9 @@ import org.apache.hbase.thirdparty.com.google.protobuf.RpcChannel; import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.ConnectionRegistryService; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetClusterIdResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetConnectionRegistryResponse; @Category({ ClientTests.class, SmallTests.class }) public class TestRpcBasedRegistryHedgedReads { @@ -132,6 +134,12 @@ public static final class RpcChannelImpl implements RpcChannel { @Override public void callMethod(MethodDescriptor method, RpcController controller, Message request, Message responsePrototype, RpcCallback done) { + if (method.getService().equals(ConnectionRegistryService.getDescriptor())) { + // this is for setting up the rpc client + done.run( + GetConnectionRegistryResponse.newBuilder().setClusterId(RESP.getClusterId()).build()); + return; + } if (!method.getName().equals("GetClusterId")) { // On RPC failures, MasterRegistry internally runs getMasters() RPC to keep the master list // fresh. We do not want to intercept those RPCs here and double count. @@ -155,9 +163,9 @@ public void callMethod(MethodDescriptor method, RpcController controller, Messag private AbstractRpcBasedConnectionRegistry createRegistry(int hedged) throws IOException { Configuration conf = UTIL.getConfiguration(); conf.setInt(HEDGED_REQS_FANOUT_CONFIG_NAME, hedged); - return new AbstractRpcBasedConnectionRegistry(conf, HEDGED_REQS_FANOUT_CONFIG_NAME, - INITIAL_DELAY_SECS_CONFIG_NAME, REFRESH_INTERVAL_SECS_CONFIG_NAME, - MIN_REFRESH_INTERVAL_SECS_CONFIG_NAME) { + return new AbstractRpcBasedConnectionRegistry(conf, User.getCurrent(), + HEDGED_REQS_FANOUT_CONFIG_NAME, INITIAL_DELAY_SECS_CONFIG_NAME, + REFRESH_INTERVAL_SECS_CONFIG_NAME, MIN_REFRESH_INTERVAL_SECS_CONFIG_NAME) { @Override protected Set getBootstrapNodes(Configuration conf) throws IOException { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java index 81db193a3042..53f33845ef7d 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTableDescriptorBuilder.java @@ -362,10 +362,10 @@ public void testStringCustomizedValues() throws HBaseException { htd.toStringCustomizedValues()); htd = TableDescriptorBuilder.newBuilder(htd).setMaxFileSize("10737942528") - .setMemStoreFlushSize("256MB").build(); + .setMemStoreFlushSize("256MB").setErasureCodingPolicy("RS-6-3-1024k").build(); assertEquals( "'testStringCustomizedValues', " + "{TABLE_ATTRIBUTES => {DURABILITY => 'ASYNC_WAL', " - + "MAX_FILESIZE => '10737942528 B (10GB 512KB)', " + + "ERASURE_CODING_POLICY => 'RS-6-3-1024k', MAX_FILESIZE => '10737942528 B (10GB 512KB)', " + "MEMSTORE_FLUSHSIZE => '268435456 B (256MB)'}}, " + "{NAME => 'cf', BLOCKSIZE => '131072 B (128KB)'}", htd.toStringCustomizedValues()); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/DummyException.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/DummyException.java new file mode 100644 index 000000000000..407c1248a98d --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/DummyException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.ipc; + +/** + * Just a dummy exception for testing IPCUtil.isFatalConnectionException. + */ +public class DummyException extends Exception { + + private static final long serialVersionUID = 215191975455115118L; + +} diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/DummyFatalConnectionException.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/DummyFatalConnectionException.java new file mode 100644 index 000000000000..437b60b031b6 --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/DummyFatalConnectionException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.ipc; + +/** + * Just a dummy exception for testing IPCUtil.isFatalConnectionException. + */ +public class DummyFatalConnectionException extends FatalConnectionException { + + private static final long serialVersionUID = -1966815615846798490L; + +} diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java index 67a8d15c1d02..d0e4044b0456 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java @@ -19,6 +19,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -44,6 +45,8 @@ import org.apache.hbase.thirdparty.io.netty.channel.DefaultEventLoop; import org.apache.hbase.thirdparty.io.netty.channel.EventLoop; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ExceptionResponse; + @Category({ ClientTests.class, SmallTests.class }) public class TestIPCUtil { @@ -159,4 +162,23 @@ public void run() { eventLoop.shutdownGracefully().get(); } } + + @Test + public void testIsFatalConnectionException() { + // intentionally not reference the class object directly, so here we will not load the class, to + // make sure that in isFatalConnectionException, we can use initialized = false when calling + // Class.forName + ExceptionResponse resp = ExceptionResponse.newBuilder() + .setExceptionClassName("org.apache.hadoop.hbase.ipc.DummyFatalConnectionException").build(); + assertTrue(IPCUtil.isFatalConnectionException(resp)); + + resp = ExceptionResponse.newBuilder() + .setExceptionClassName("org.apache.hadoop.hbase.ipc.DummyException").build(); + assertFalse(IPCUtil.isFatalConnectionException(resp)); + + // class not found + resp = ExceptionResponse.newBuilder() + .setExceptionClassName("org.apache.hadoop.hbase.ipc.WhatEver").build(); + assertFalse(IPCUtil.isFatalConnectionException(resp)); + } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java index 10948358ff92..8aadce85651d 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestTLSHandshadeFailure.java @@ -56,6 +56,8 @@ import org.apache.hbase.thirdparty.com.google.common.io.Closeables; import org.apache.hbase.thirdparty.io.netty.handler.ssl.NotSslRecordException; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.ClientMetaService; + /** * A simple UT to make sure that we do not leak the SslExceptions to netty's TailContext, where it * will generate a confusing WARN message. @@ -149,11 +151,12 @@ public Void answer(InvocationOnMock invocation) throws Throwable { Address.fromParts("127.0.0.1", server.getLocalPort())); NettyRpcConnection conn = client.createConnection(id); BlockingRpcCallback done = new BlockingRpcCallback<>(); - Call call = - new Call(1, null, null, null, null, 0, 0, Collections.emptyMap(), done, new CallStats()); + Call call = new Call(1, ClientMetaService.getDescriptor().getMethods().get(0), null, null, null, + 0, 0, Collections.emptyMap(), done, new CallStats()); HBaseRpcController hrc = new HBaseRpcControllerImpl(); conn.sendRequest(call, hrc); done.get(); + call.error.printStackTrace(); assertThat(call.error, instanceOf(NotSslRecordException.class)); Waiter.waitFor(conf, 5000, () -> msg.get() != null); verify(mockAppender).append(any()); diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index fc3136e05558..dd30a7a6f581 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -31,6 +31,15 @@ Apache HBase - Common Common functionality for HBase + + + ${project.version} + + org.apache.hbase @@ -211,7 +220,7 @@ process-resources - + @@ -227,7 +236,7 @@ - + diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 2aa9ecf69ec4..5b53d2b2c0d3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -153,6 +153,13 @@ public enum OperationStatusCode { /** Default value for the balancer period */ public static final int DEFAULT_HBASE_BALANCER_PERIOD = 300000; + /** Config for the oldWALs directory size updater period */ + public static final String HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD = + "hbase.master.oldwals.dir.updater.period"; + + /** Default value for the oldWALs directory size updater period */ + public static final int DEFAULT_HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD = 300000; + /** * Config key for enable/disable automatically separate child regions to different region servers * in the procedure of split regions. One child will be kept to the server where parent region is diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java index 41acfbbf48f4..46809050b5a3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java @@ -105,6 +105,15 @@ public final class X509Util { public static final String HBASE_SERVER_NETTY_TLS_SUPPORTPLAINTEXT = "hbase.server.netty.tls.supportplaintext"; + /** + * Set the SSL wrapSize for netty. This is only a maximum wrap size. Buffers smaller than this + * will not be consolidated, but buffers larger than this will be split into multiple wrap + * buffers. The netty default of 16k is not great for hbase which tends to return larger payloads + * than that, meaning most responses end up getting chunked up. This leads to more memory + * contention in netty's PoolArena. See https://github.com/netty/netty/pull/13551 + */ + public static final String HBASE_SERVER_NETTY_TLS_WRAP_SIZE = "hbase.server.netty.tls.wrapSize"; + public static final int DEFAULT_HBASE_SERVER_NETTY_TLS_WRAP_SIZE = 1024 * 1024; // // Client-side specific configs // @@ -290,7 +299,8 @@ public static SslContext createSslContextForClient(Configuration config) * Adds SslProvider.OPENSSL if OpenSsl is available and enabled. In order to make it available, * one must ensure that a properly shaded netty-tcnative is on the classpath. Properly shaded * means relocated to be prefixed with "org.apache.hbase.thirdparty" like the rest of the netty - * classes. + * classes. We make available org.apache.hbase:hbase-openssl as a convenience module which one can + * use to pull in a shaded netty-tcnative statically linked against boringssl. */ private static boolean configureOpenSslIfAvailable(SslContextBuilder sslContextBuilder, Configuration conf) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java index 422a607dc850..c19bdec560bb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -119,9 +120,7 @@ public int run(String[] args) throws IOException { CommandLine cmd; List argsList = new ArrayList<>(args.length); - for (String arg : args) { - argsList.add(arg); - } + Collections.addAll(argsList, args); // For backward compatibility of args which can't be parsed as Option. See javadoc for // processOldArgs(..) processOldArgs(argsList); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java index 054de74d7d1e..d6b936323404 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java @@ -468,38 +468,75 @@ public static void writeVLong(ByteBuffer out, long i) { } } - private interface ByteVisitor { - byte get(); - } - - private static long readVLong(ByteVisitor visitor) { - byte firstByte = visitor.get(); + /** + * Similar to {@link WritableUtils#readVLong(java.io.DataInput)} but reads from a + * {@link ByteBuff}. + */ + public static long readVLong(ByteBuff buf) { + byte firstByte = buf.get(); int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; + } else { + int remaining = len - 1; + long i = 0; + int offsetFromPos = 0; + if (remaining >= Bytes.SIZEOF_INT) { + // The int read has to be converted to unsigned long so the & op + i = (buf.getIntAfterPosition(offsetFromPos) & 0x00000000ffffffffL); + remaining -= Bytes.SIZEOF_INT; + offsetFromPos += Bytes.SIZEOF_INT; + } + if (remaining >= Bytes.SIZEOF_SHORT) { + short s = buf.getShortAfterPosition(offsetFromPos); + i = i << 16; + i = i | (s & 0xFFFF); + remaining -= Bytes.SIZEOF_SHORT; + offsetFromPos += Bytes.SIZEOF_SHORT; + } + for (int idx = 0; idx < remaining; idx++) { + byte b = buf.getByteAfterPosition(offsetFromPos + idx); + i = i << 8; + i = i | (b & 0xFF); + } + buf.skip(len - 1); + return WritableUtils.isNegativeVInt(firstByte) ? ~i : i; } - long i = 0; - for (int idx = 0; idx < len - 1; idx++) { - byte b = visitor.get(); - i = i << 8; - i = i | (b & 0xFF); - } - return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); } /** * Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuffer}. */ - public static long readVLong(ByteBuffer in) { - return readVLong(in::get); - } - - /** - * Similar to {@link WritableUtils#readVLong(java.io.DataInput)} but reads from a - * {@link ByteBuff}. - */ - public static long readVLong(ByteBuff in) { - return readVLong(in::get); + public static long readVLong(ByteBuffer buf) { + byte firstByte = buf.get(); + int len = WritableUtils.decodeVIntSize(firstByte); + if (len == 1) { + return firstByte; + } else { + int remaining = len - 1; + long i = 0; + int offsetFromPos = 0; + if (remaining >= Bytes.SIZEOF_INT) { + // The int read has to be converted to unsigned long so the & op + i = (buf.getInt(buf.position() + offsetFromPos) & 0x00000000ffffffffL); + remaining -= Bytes.SIZEOF_INT; + offsetFromPos += Bytes.SIZEOF_INT; + } + if (remaining >= Bytes.SIZEOF_SHORT) { + short s = buf.getShort(buf.position() + offsetFromPos); + i = i << 16; + i = i | (s & 0xFFFF); + remaining -= Bytes.SIZEOF_SHORT; + offsetFromPos += Bytes.SIZEOF_SHORT; + } + for (int idx = 0; idx < remaining; idx++) { + byte b = buf.get(buf.position() + offsetFromPos + idx); + i = i << 8; + i = i | (b & 0xFF); + } + buf.position(buf.position() + len - 1); + return WritableUtils.isNegativeVInt(firstByte) ? ~i : i; + } } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentMapUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentMapUtils.java index cf8130e624e2..9c5ebe8519f4 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentMapUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentMapUtils.java @@ -38,14 +38,6 @@ public static V computeIfAbsent(ConcurrentMap map, K key, Supplier< }); } - /** - * A supplier that throws IOException when get. - */ - @FunctionalInterface - public interface IOExceptionSupplier { - V get() throws IOException; - } - /** * In HBASE-16648 we found that ConcurrentHashMap.get is much faster than computeIfAbsent if the * value already exists. So here we copy the implementation of diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/IOExceptionSupplier.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/IOExceptionSupplier.java new file mode 100644 index 000000000000..11771a47c083 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/IOExceptionSupplier.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * A supplier that throws IOException when get. + */ +@InterfaceAudience.Private +@FunctionalInterface +public interface IOExceptionSupplier { + V get() throws IOException; +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java index 12d81fee6586..87885e2b9fd5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map.Entry; import java.util.Properties; -import java.util.Set; import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; @@ -29,7 +28,6 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.base.Splitter; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; /** * Utility methods for reading, and building the ZooKeeper configuration. The order and priority for @@ -42,12 +40,6 @@ public final class ZKConfig { private static final String VARIABLE_START = "${"; private static final String ZOOKEEPER_JAVA_PROPERTY_PREFIX = "zookeeper."; - /** Supported ZooKeeper client TLS properties */ - static final Set ZOOKEEPER_CLIENT_TLS_PROPERTIES = - ImmutableSet.of("client.secure", "clientCnxnSocket", "ssl.keyStore.location", - "ssl.keyStore.password", "ssl.keyStore.passwordPath", "ssl.trustStore.location", - "ssl.trustStore.password", "ssl.trustStore.passwordPath"); - private ZKConfig() { } @@ -62,16 +54,12 @@ public static Properties makeZKProps(Configuration conf) { } /** - * Make a Properties object holding ZooKeeper config. Parses the corresponding config options from - * the HBase XML configs and generates the appropriate ZooKeeper properties. - * @param conf Configuration to read from. - * @return Properties holding mappings representing ZooKeeper config file. + * Directly map all the hbase.zookeeper.property.KEY properties. Synchronize on conf so no loading + * of configs while we iterate */ - private static Properties makeZKPropsFromHbaseConfig(Configuration conf) { + private static Properties extractZKPropsFromHBaseConfig(final Configuration conf) { Properties zkProperties = new Properties(); - // Directly map all of the hbase.zookeeper.property.KEY properties. - // Synchronize on conf so no loading of configs while we iterate synchronized (conf) { for (Entry entry : conf) { String key = entry.getKey(); @@ -87,6 +75,18 @@ private static Properties makeZKPropsFromHbaseConfig(Configuration conf) { } } + return zkProperties; + } + + /** + * Make a Properties object holding ZooKeeper config. Parses the corresponding config options from + * the HBase XML configs and generates the appropriate ZooKeeper properties. + * @param conf Configuration to read from. + * @return Properties holding mappings representing ZooKeeper config file. + */ + private static Properties makeZKPropsFromHbaseConfig(Configuration conf) { + Properties zkProperties = extractZKPropsFromHBaseConfig(conf); + // If clientPort is not set, assign the default. if (zkProperties.getProperty(HConstants.CLIENT_PORT_STR) == null) { zkProperties.put(HConstants.CLIENT_PORT_STR, HConstants.DEFAULT_ZOOKEEPER_CLIENT_PORT); @@ -343,24 +343,12 @@ public static String getClientZKQuorumServersString(Configuration conf) { } private static void setZooKeeperClientSystemProperties(String prefix, Configuration conf) { - synchronized (conf) { - for (Entry entry : conf) { - String key = entry.getKey(); - if (!key.startsWith(prefix)) { - continue; - } - String zkKey = key.substring(prefix.length()); - if (!ZOOKEEPER_CLIENT_TLS_PROPERTIES.contains(zkKey)) { - continue; - } - String value = entry.getValue(); - // If the value has variables substitutions, need to do a get. - if (value.contains(VARIABLE_START)) { - value = conf.get(key); - } - if (System.getProperty(ZOOKEEPER_JAVA_PROPERTY_PREFIX + zkKey) == null) { - System.setProperty(ZOOKEEPER_JAVA_PROPERTY_PREFIX + zkKey, value); - } + Properties zkProperties = extractZKPropsFromHBaseConfig(conf); + for (Entry entry : zkProperties.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + if (System.getProperty(ZOOKEEPER_JAVA_PROPERTY_PREFIX + key) == null) { + System.setProperty(ZOOKEEPER_JAVA_PROPERTY_PREFIX + key, value); } } } diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 17a9853d2ad3..1bf63b136e04 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -606,6 +606,12 @@ possible configurations would overwhelm and obscure the important. Period at which the region balancer runs in the Master, in milliseconds. + + hbase.master.oldwals.dir.updater.period + 300000 + Period at which the oldWALs directory size calculator/updater will run in the + Master, in milliseconds. + hbase.regions.slop 0.2 diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java index eabfed2042ca..e07e75bffdb2 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtil; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; @@ -179,6 +180,23 @@ public void testReadWriteVLong() { ByteBufferUtils.writeVLong(b, l); b.flip(); assertEquals(l, ByteBufferUtils.readVLong(b)); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + } + } + + @Test + public void testReadWriteConsecutiveVLong() { + for (long l : testNumbers) { + ByteBuffer b = ByteBuffer.allocate(2 * MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + ByteBufferUtils.writeVLong(b, l - 4); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(b)); + assertEquals(l - 4, ByteBufferUtils.readVLong(b)); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + assertEquals(l - 4, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKConfig.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKConfig.java index 7418afe5d222..63df9043bae3 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKConfig.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKConfig.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hbase.zookeeper; -import static org.apache.hadoop.hbase.zookeeper.ZKConfig.ZOOKEEPER_CLIENT_TLS_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Properties; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -33,6 +33,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + @Category({ MiscTests.class, SmallTests.class }) public class TestZKConfig { @@ -40,6 +42,12 @@ public class TestZKConfig { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestZKConfig.class); + /** Supported ZooKeeper client TLS properties */ + private static final Set ZOOKEEPER_CLIENT_TLS_PROPERTIES = ImmutableSet.of( + "client.secure", "clientCnxnSocket", "ssl.keyStore.location", "ssl.keyStore.password", + "ssl.keyStore.passwordPath", "ssl.keyStore.type", "ssl.trustStore.location", + "ssl.trustStore.password", "ssl.trustStore.passwordPath", "ssl.trustStore.type"); + @Test public void testZKConfigLoading() throws Exception { Configuration conf = HBaseConfiguration.create(); @@ -133,6 +141,21 @@ public void testZooKeeperTlsPropertiesServer() { } } + @Test + public void testZooKeeperPropertiesDoesntOverwriteSystem() { + // Arrange + System.setProperty("zookeeper.a.b.c", "foo"); + Configuration conf = HBaseConfiguration.create(); + conf.set(HConstants.ZK_CFG_PROPERTY_PREFIX + "a.b.c", "bar"); + + // Act + ZKConfig.getZKQuorumServersString(conf); + + // Assert + assertEquals("foo", System.getProperty("zookeeper.a.b.c")); + System.clearProperty("zookeeper.a.b.c"); + } + private void testKey(String ensemble, int port, String znode) throws IOException { testKey(ensemble, port, znode, false); // not support multiple client ports } diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index 1a5ca5bd09aa..5a7d8f957da0 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -33,7 +33,7 @@ - 3.21.12 + 3.24.3 diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java index 53288be872ea..fa7ccf737365 100644 --- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java @@ -41,7 +41,7 @@ import org.apache.yetus.audience.InterfaceAudience; /** - * This is an example showing how a RegionObserver could configured via ZooKeeper in order to + * This is an example showing how a RegionObserver could be configured via ZooKeeper in order to * control a Region compaction, flush, and scan policy. This also demonstrated the use of shared * {@link org.apache.hadoop.hbase.coprocessor.RegionObserver} state. See * {@link RegionCoprocessorEnvironment#getSharedData()}. diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java index a479310691b1..26a8943096a9 100644 --- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java +++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/security/provider/example/TestShadeSaslAuthenticationProvider.java @@ -27,8 +27,6 @@ import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -69,10 +67,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -84,8 +80,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.base.Throwables; - @Category({ MediumTests.class, SecurityTests.class }) public class TestShadeSaslAuthenticationProvider { private static final Logger LOG = @@ -212,21 +206,23 @@ public String run() throws Exception { @Test public void testPositiveAuthentication() throws Exception { final Configuration clientConf = new Configuration(CONF); - try (Connection conn = ConnectionFactory.createConnection(clientConf)) { + try (Connection conn1 = ConnectionFactory.createConnection(clientConf)) { UserGroupInformation user1 = UserGroupInformation.createUserForTesting("user1", new String[0]); - user1.addToken(ShadeClientTokenUtil.obtainToken(conn, "user1", USER1_PASSWORD)); + user1.addToken(ShadeClientTokenUtil.obtainToken(conn1, "user1", USER1_PASSWORD)); user1.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { - try (Table t = conn.getTable(tableName)) { - Result r = t.get(new Get(Bytes.toBytes("r1"))); - assertNotNull(r); - assertFalse("Should have read a non-empty Result", r.isEmpty()); - final Cell cell = r.getColumnLatestCell(Bytes.toBytes("f1"), Bytes.toBytes("q1")); - assertTrue("Unexpected value", CellUtil.matchingValue(cell, Bytes.toBytes("1"))); + try (Connection conn = ConnectionFactory.createConnection(clientConf)) { + try (Table t = conn.getTable(tableName)) { + Result r = t.get(new Get(Bytes.toBytes("r1"))); + assertNotNull(r); + assertFalse("Should have read a non-empty Result", r.isEmpty()); + final Cell cell = r.getColumnLatestCell(Bytes.toBytes("f1"), Bytes.toBytes("q1")); + assertTrue("Unexpected value", CellUtil.matchingValue(cell, Bytes.toBytes("1"))); - return null; + return null; + } } } }); @@ -268,7 +264,6 @@ public Void run() throws Exception { } catch (Exception e) { LOG.info("Caught exception in negative Master connectivity test", e); assertEquals("Found unexpected exception", pair.getSecond(), e.getClass()); - validateRootCause(Throwables.getRootCause(e)); } return null; } @@ -287,7 +282,6 @@ public Void run() throws Exception { } catch (Exception e) { LOG.info("Caught exception in negative RegionServer connectivity test", e); assertEquals("Found unexpected exception", pair.getSecond(), e.getClass()); - validateRootCause(Throwables.getRootCause(e)); } return null; } @@ -301,19 +295,4 @@ public Void run() throws Exception { } }); } - - void validateRootCause(Throwable rootCause) { - LOG.info("Root cause was", rootCause); - if (rootCause instanceof RemoteException) { - RemoteException re = (RemoteException) rootCause; - IOException actualException = re.unwrapRemoteException(); - assertEquals(InvalidToken.class, actualException.getClass()); - } else { - StringWriter writer = new StringWriter(); - rootCause.printStackTrace(new PrintWriter(writer)); - String text = writer.toString(); - assertTrue("Message did not contain expected text", - text.contains(InvalidToken.class.getName())); - } - } } diff --git a/hbase-extensions/hbase-openssl/pom.xml b/hbase-extensions/hbase-openssl/pom.xml new file mode 100644 index 000000000000..7158c2f4197e --- /dev/null +++ b/hbase-extensions/hbase-openssl/pom.xml @@ -0,0 +1,43 @@ + + + + 4.0.0 + + org.apache.hbase + hbase-extensions + ${revision} + ../pom.xml + + + hbase-openssl + jar + Apache HBase - OpenSSL support for TLS RPC + Includes tcnative bindings so that netty TLS can use OpenSSL + + + + org.apache.hbase.thirdparty + hbase-shaded-netty-tcnative + runtime + + + + diff --git a/hbase-extensions/pom.xml b/hbase-extensions/pom.xml new file mode 100644 index 000000000000..8a11e7754ea2 --- /dev/null +++ b/hbase-extensions/pom.xml @@ -0,0 +1,99 @@ + + + + 4.0.0 + + org.apache.hbase + hbase-build-configuration + ${revision} + ../hbase-build-configuration + + + hbase-extensions + pom + Apache HBase - Extensions + Parent for optional extension modules + + + hbase-openssl + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${project.basedir}/../../dev-support/spotbugs-exclude.xml + true + true + Max + + + + + maven-assembly-plugin + + true + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + + spotbugs + + false + + ${project.basedir}/../dev-support/spotbugs-exclude.xml + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + maven-assembly-plugin + + true + + + + + + diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSource.java index af7e87483d1d..5b4fc4b2c69a 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSource.java @@ -47,6 +47,9 @@ public interface MetricsIOSource extends BaseSource { String FS_PREAD_TIME_HISTO_KEY = "fsPReadTime"; String FS_WRITE_HISTO_KEY = "fsWriteTime"; + String SLOW_FS_READS_KEY = "fsSlowReadsCount"; + String SLOW_FS_READS_DESC = "Number of HFile reads which were slower than a configured threshold"; + String CHECKSUM_FAILURES_KEY = "fsChecksumFailureCount"; String FS_READ_TIME_HISTO_DESC = @@ -76,4 +79,6 @@ public interface MetricsIOSource extends BaseSource { * @param t time it took, in milliseconds */ void updateFsWriteTime(long t); + + void incrSlowFsRead(); } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSourceImpl.java index 6ef5d180cd5e..5aca9a3d84b9 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/io/MetricsIOSourceImpl.java @@ -22,6 +22,7 @@ import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private @@ -32,6 +33,7 @@ public class MetricsIOSourceImpl extends BaseSourceImpl implements MetricsIOSour private final MetricHistogram fsReadTimeHisto; private final MetricHistogram fsPReadTimeHisto; private final MetricHistogram fsWriteTimeHisto; + private final MutableFastCounter fsSlowReads; public MetricsIOSourceImpl(MetricsIOWrapper wrapper) { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, wrapper); @@ -49,6 +51,7 @@ public MetricsIOSourceImpl(String metricsName, String metricsDescription, String getMetricsRegistry().newTimeHistogram(FS_PREAD_TIME_HISTO_KEY, FS_PREAD_TIME_HISTO_DESC); fsWriteTimeHisto = getMetricsRegistry().newTimeHistogram(FS_WRITE_HISTO_KEY, FS_WRITE_TIME_HISTO_DESC); + fsSlowReads = getMetricsRegistry().newCounter(SLOW_FS_READS_KEY, SLOW_FS_READS_DESC, 0L); } @Override @@ -66,6 +69,11 @@ public void updateFsWriteTime(long t) { fsWriteTimeHisto.add(t); } + @Override + public void incrSlowFsRead() { + fsSlowReads.incr(); + } + @Override public void getMetrics(MetricsCollector metricsCollector, boolean all) { MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index 4a5b97ae66bf..d606ed630881 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -70,6 +70,7 @@ public interface MetricsMasterSource extends BaseSource { String CLUSTER_REQUESTS_NAME = "clusterRequests"; String CLUSTER_READ_REQUESTS_NAME = "clusterReadRequests"; String CLUSTER_WRITE_REQUESTS_NAME = "clusterWriteRequests"; + String OLD_WAL_DIR_SIZE_NAME = "oldWALsDirSize"; String MASTER_ACTIVE_TIME_DESC = "Master Active Time"; String MASTER_START_TIME_DESC = "Master Start Time"; String MASTER_FINISHED_INITIALIZATION_TIME_DESC = @@ -91,6 +92,7 @@ public interface MetricsMasterSource extends BaseSource { String OFFLINE_REGION_COUNT_DESC = "Number of Offline Regions"; String SERVER_CRASH_METRIC_PREFIX = "serverCrash"; + String OLD_WAL_DIR_SIZE_DESC = "size of old WALs directory in bytes"; /** * Increment the number of requests the cluster has seen. diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index e0abf77bea44..011e66312aa3 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -129,7 +129,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), masterWrapper.getServerName()) .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId()) .tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC), - String.valueOf(masterWrapper.getIsActiveMaster())); + String.valueOf(masterWrapper.getIsActiveMaster())) + .addGauge(Interns.info(OLD_WAL_DIR_SIZE_NAME, OLD_WAL_DIR_SIZE_DESC), + masterWrapper.getOldWALsDirSize()); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index a900edf115e3..83419e2d5501 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -153,4 +153,9 @@ public interface MetricsMasterWrapper { * @return pair of count for online regions and offline regions */ PairOfSameType getRegionCounts(); + + /** + * Get the size of old WALs directory in bytes. + */ + long getOldWALsDirSize(); } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index a53899c476fa..8b64e793bed6 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -639,4 +639,6 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String SCANNER_LEASE_EXPIRED_COUNT = "scannerLeaseExpiredCount"; String SCANNER_LEASE_EXPIRED_COUNT_DESC = "Count of scanners which were expired due to scanner lease timeout"; + String CURRENT_REGION_CACHE_RATIO = "currentRegionCacheRatio"; + String CURRENT_REGION_CACHE_RATIO_DESC = "The percentage of caching completed for this region."; } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 410d775d7d32..2f4fbb431aba 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -233,6 +233,10 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { this.regionWrapper.getNumReferenceFiles()); mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE, MetricsRegionServerSource.STOREFILE_SIZE_DESC), this.regionWrapper.getStoreFileSize()); + mrb.addGauge( + Interns.info(regionNamePrefix + MetricsRegionServerSource.CURRENT_REGION_CACHE_RATIO, + MetricsRegionServerSource.CURRENT_REGION_CACHE_RATIO_DESC), + this.regionWrapper.getCurrentRegionCacheRatio()); mrb.addCounter( Interns.info(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT, MetricsRegionSource.COMPACTIONS_COMPLETED_DESC), diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index 55ddc603979b..3445faf7eaad 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -65,6 +65,11 @@ public interface MetricsRegionWrapper { */ long getStoreFileSize(); + /** + * Gets the current cache % ratio for this region. + */ + float getCurrentRegionCacheRatio(); + /** * Get the total number of read requests that have been issued against this region */ diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 46b6405eb46e..28286cc1c6e2 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -116,6 +116,11 @@ public long getStoreFileSize() { return 0; } + @Override + public float getCurrentRegionCacheRatio() { + return 0; + } + @Override public long getReadRequestCount() { return 0; diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index a2b8a8fb6275..d5af8df1c7fd 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -81,6 +81,7 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ErrorHandler; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.HandlerCollection; @@ -170,7 +171,9 @@ public class HttpServer implements FilterContainer { .put("jmx", new ServletConfig("jmx", "/jmx", "org.apache.hadoop.hbase.http.jmx.JMXJsonServlet")) .put("metrics", - new ServletConfig("metrics", "/metrics", "org.apache.hadoop.metrics.MetricsServlet")) + // MetricsServlet is deprecated in hadoop 2.8 and removed in 3.0. We shouldn't expect it, + // so pass false so that we don't create a noisy warn during instantiation. + new ServletConfig("metrics", "/metrics", "org.apache.hadoop.metrics.MetricsServlet", false)) .put("prometheus", new ServletConfig("prometheus", "/prometheus", "org.apache.hadoop.hbase.http.prometheus.PrometheusHadoopServlet")) .build(); @@ -756,15 +759,10 @@ protected void addDefaultApps(ContextHandlerCollection parent, final String appD ServletContextHandler logContext = new ServletContextHandler(parent, "/logs"); logContext.addServlet(AdminAuthorizedServlet.class, "/*"); logContext.setResourceBase(logDir); - - if ( - conf.getBoolean(ServerConfigurationKeys.HBASE_JETTY_LOGS_SERVE_ALIASES, - ServerConfigurationKeys.DEFAULT_HBASE_JETTY_LOGS_SERVE_ALIASES) - ) { - Map params = logContext.getInitParams(); - params.put("org.mortbay.jetty.servlet.Default.aliases", "true"); - } logContext.setDisplayName("logs"); + configureAliasChecks(logContext, + conf.getBoolean(ServerConfigurationKeys.HBASE_JETTY_LOGS_SERVE_ALIASES, + ServerConfigurationKeys.DEFAULT_HBASE_JETTY_LOGS_SERVE_ALIASES)); setContextAttributes(logContext, conf); addNoCacheFilter(logContext, conf); defaultContexts.put(logContext, true); @@ -778,6 +776,37 @@ protected void addDefaultApps(ContextHandlerCollection parent, final String appD defaultContexts.put(staticContext, true); } + /** + * This method configures the alias checks for the given ServletContextHandler based on the + * provided value of shouldServeAlias.
+ * If shouldServeAlias is set to true, it checks if SymlinkAllowedResourceAliasChecker is already + * a part of the alias check list. If it is already a part of the list, no changes are made, else, + * it adds it to the list.
+ * If shouldServeAlias is set to false, it clears all alias checks from the + * ServletContextHandler.
+ * . + * @param context The ServletContextHandler whose alias checks are to be configured + * @param shouldServeAlias Whether aliases should be allowed or not + */ + private void configureAliasChecks(ServletContextHandler context, boolean shouldServeAlias) { + if (shouldServeAlias) { + Class aliasCheckerClass = SymlinkAllowedResourceAliasChecker.class; + // check if SymlinkAllowedResourceAliasChecker is already part of alias check list + // NOTE: we are doing this because this is already present in the context (by default) + if (context.getAliasChecks().stream().anyMatch(aliasCheckerClass::isInstance)) { + LOG.debug("{} is already part of alias check list", aliasCheckerClass.getName()); + } else { + context.addAliasCheck(new SymlinkAllowedResourceAliasChecker(context)); + LOG.debug("{} added to the alias check list", aliasCheckerClass.getName()); + } + LOG.info("Serving aliases allowed for /logs context"); + } else { + // if aliasing is disabled, then we should clear the alias check list + context.clearAliasChecks(); + LOG.info("Serving aliases disabled for /logs context"); + } + } + private void setContextAttributes(ServletContextHandler context, Configuration conf) { context.getServletContext().setAttribute(CONF_CONTEXT_ATTRIBUTE, conf); context.getServletContext().setAttribute(ADMINS_ACL, adminsAcl); @@ -819,16 +848,19 @@ protected void addDefaultServlets(ContextHandlerCollection contexts, Configurati /* register metrics servlets */ String[] enabledServlets = conf.getStrings(METRIC_SERVLETS_CONF_KEY, METRICS_SERVLETS_DEFAULT); for (String enabledServlet : enabledServlets) { - try { - ServletConfig servletConfig = METRIC_SERVLETS.get(enabledServlet); - if (servletConfig != null) { + ServletConfig servletConfig = METRIC_SERVLETS.get(enabledServlet); + if (servletConfig != null) { + try { Class clz = Class.forName(servletConfig.getClazz()); addPrivilegedServlet(servletConfig.getName(), servletConfig.getPathSpec(), clz.asSubclass(HttpServlet.class)); + } catch (Exception e) { + if (servletConfig.isExpected()) { + // metrics are not critical to read/write, so an exception here shouldn't be fatal + // if the class was expected we should warn though + LOG.warn("Couldn't register the servlet " + enabledServlet, e); + } } - } catch (Exception e) { - /* shouldn't be fatal, so warn the user about it */ - LOG.warn("Couldn't register the servlet " + enabledServlet, e); } } } diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ServletConfig.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ServletConfig.java index befe60957605..366dbfd9f228 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ServletConfig.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ServletConfig.java @@ -23,14 +23,20 @@ @InterfaceAudience.Private class ServletConfig { - private String name; - private String pathSpec; - private String clazz; + private final String name; + private final String pathSpec; + private final String clazz; + private final boolean expected; public ServletConfig(String name, String pathSpec, String clazz) { + this(name, pathSpec, clazz, true); + } + + public ServletConfig(String name, String pathSpec, String clazz, boolean expected) { this.name = name; this.pathSpec = pathSpec; this.clazz = clazz; + this.expected = expected; } public String getName() { @@ -44,4 +50,8 @@ public String getPathSpec() { public String getClazz() { return clazz; } + + public boolean isExpected() { + return expected; + } } diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index c3f884a16b0d..dbf34c404155 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -104,6 +104,12 @@ + + + org.bouncycastle + bcpkix-jdk18on + test + org.apache.hbase hbase-backup diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java index 35db989dd691..e5c1fbed1a56 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java @@ -60,6 +60,7 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.base.Splitter; +import org.apache.hbase.thirdparty.com.google.common.base.Strings; /** * Validate ImportTsv + BulkLoadFiles on a distributed cluster. @@ -85,6 +86,9 @@ public class IntegrationTestImportTsv extends Configured implements Tool { { byte[] family = Bytes.toBytes("d"); for (String line : Splitter.on('\n').split(simple_tsv)) { + if (Strings.isNullOrEmpty(line)) { + continue; + } String[] row = line.split("\t"); byte[] key = Bytes.toBytes(row[0]); long ts = Long.parseLong(row[1]); @@ -141,12 +145,10 @@ protected void doLoadIncrementalHFiles(Path hfiles, TableName tableName) throws ToolRunner.run(new BulkLoadHFilesTool(getConf()), args)); Table table = null; - Scan scan = new Scan() { - { - setCacheBlocks(false); - setCaching(1000); - } - }; + Scan scan = new Scan(); + scan.setCacheBlocks(false); + scan.setCaching(1000); + try { table = util.getConnection().getTable(tableName); Iterator resultsIt = table.getScanner(scan).iterator(); diff --git a/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/InternalLog4jUtils.java b/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/InternalLog4jUtils.java index 18c2c0f6e679..8dff80cf61b9 100644 --- a/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/InternalLog4jUtils.java +++ b/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/InternalLog4jUtils.java @@ -36,13 +36,27 @@ final class InternalLog4jUtils { private InternalLog4jUtils() { } - static void setLogLevel(String loggerName, String levelName) { + private static org.apache.logging.log4j.Level getLevel(String levelName) + throws IllegalArgumentException { org.apache.logging.log4j.Level level = org.apache.logging.log4j.Level.toLevel(levelName.toUpperCase()); if (!level.toString().equalsIgnoreCase(levelName)) { throw new IllegalArgumentException("Unsupported log level " + levelName); } - org.apache.logging.log4j.core.config.Configurator.setLevel(loggerName, level); + return level; + } + + static void setAllLevels(String loggerName, String levelName) { + org.apache.logging.log4j.core.config.Configurator.setAllLevels(loggerName, getLevel(levelName)); + } + + static void setLogLevel(String loggerName, String levelName) { + org.apache.logging.log4j.core.config.Configurator.setLevel(loggerName, getLevel(levelName)); + } + + static void setRootLevel(String levelName) { + String loggerName = org.apache.logging.log4j.LogManager.getRootLogger().getName(); + setLogLevel(loggerName, levelName); } static String getEffectiveLevel(String loggerName) { diff --git a/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/Log4jUtils.java b/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/Log4jUtils.java index ba136663e092..8d9bb4783ee8 100644 --- a/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/Log4jUtils.java +++ b/hbase-logging/src/main/java/org/apache/hadoop/hbase/logging/Log4jUtils.java @@ -58,10 +58,9 @@ private static void throwUnchecked(Throwable throwable) { } } - public static void setLogLevel(String loggerName, String levelName) { - Method method = getMethod("setLogLevel", String.class, String.class); + private static void invoke(Method method, Object... args) throws AssertionError { try { - method.invoke(null, loggerName, levelName); + method.invoke(null, args); } catch (IllegalAccessException e) { throw new AssertionError("should not happen", e); } catch (InvocationTargetException e) { @@ -70,6 +69,21 @@ public static void setLogLevel(String loggerName, String levelName) { } } + public static void setAllLevels(String loggerName, String levelName) { + Method method = getMethod("setAllLevels", String.class, String.class); + invoke(method, loggerName, levelName); + } + + public static void setLogLevel(String loggerName, String levelName) { + Method method = getMethod("setLogLevel", String.class, String.class); + invoke(method, loggerName, levelName); + } + + public static void setRootLevel(String levelName) { + Method method = getMethod("setRootLevel", String.class); + invoke(method, levelName); + } + public static String getEffectiveLevel(String loggerName) { Method method = getMethod("getEffectiveLevel", String.class); try { diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml index 3d9877dbf787..798250600e35 100644 --- a/hbase-mapreduce/pom.xml +++ b/hbase-mapreduce/pom.xml @@ -216,6 +216,12 @@ bcprov-jdk18on test + + + org.bouncycastle + bcpkix-jdk18on + test + diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java index 4ff4a5b95b91..3ccbaab4de12 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -148,9 +149,7 @@ protected List listStatus(JobContext job) throws IOException { for (FileStatus status : super.listStatus(job)) { if (status.isDirectory()) { FileSystem fs = status.getPath().getFileSystem(job.getConfiguration()); - for (FileStatus match : fs.listStatus(status.getPath(), HIDDEN_FILE_FILTER)) { - result.add(match); - } + Collections.addAll(result, fs.listStatus(status.getPath(), HIDDEN_FILE_FILTER)); } else { result.add(status); } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 21070de60e25..37e99c096e5c 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -21,6 +21,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -130,9 +131,7 @@ public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); String[] tables = conf.getStrings(TABLES_KEY); this.multiTableSupport = conf.getBoolean(MULTI_TABLES_SUPPORT, false); - for (String table : tables) { - tableSet.add(table); - } + Collections.addAll(tableSet, tables); } } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index f2a8e00fea5e..c6f655c37306 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; @@ -211,14 +210,12 @@ public void setup(Context context) throws IOException { outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY); try { - srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); inputFs = FileSystem.get(inputRoot.toUri(), srcConf); } catch (IOException e) { throw new IOException("Could not get the input FileSystem with root=" + inputRoot, e); } try { - destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); outputFs = FileSystem.get(outputRoot.toUri(), destConf); } catch (IOException e) { throw new IOException("Could not get the output FileSystem with root=" + outputRoot, e); @@ -241,12 +238,6 @@ public void setup(Context context) throws IOException { } } - @Override - protected void cleanup(Context context) { - IOUtils.closeStream(inputFs); - IOUtils.closeStream(outputFs); - } - @Override public void map(BytesWritable key, NullWritable value, Context context) throws InterruptedException, IOException { @@ -990,10 +981,8 @@ public int doWork() throws IOException { } Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); - srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf); Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); - destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf); boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false) || conf.get(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR) != null; @@ -1149,9 +1138,6 @@ public int doWork() throws IOException { } outputFs.delete(outputSnapshotDir, true); return 1; - } finally { - IOUtils.closeStream(inputFs); - IOUtils.closeStream(outputFs); } } diff --git a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistry.java b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistry.java index b70526e1c5a9..96f4b313d794 100644 --- a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistry.java +++ b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistry.java @@ -96,6 +96,14 @@ public interface MetricRegistry extends MetricSet { */ boolean remove(String name); + /** + * Removes the metric with the given name only if it is registered to the provided metric. + * @param name the name of the metric + * @param metric the metric expected to be registered to the given name + * @return true if the metric is removed. + */ + boolean remove(String name, Metric metric); + /** * Return the MetricRegistryInfo object for this registry. * @return MetricRegistryInfo describing the registry. diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java index 1c8927b15b3a..caea02740c60 100644 --- a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java @@ -114,6 +114,11 @@ public boolean remove(String name) { return metrics.remove(name) != null; } + @Override + public boolean remove(String name, Metric metric) { + return metrics.remove(name, metric); + } + @Override public MetricRegistryInfo getMetricRegistryInfo() { return info; diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java index 56b3f0d6a9ee..8cae06361204 100644 --- a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.metrics.impl; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -141,4 +142,22 @@ public void testGetMetrics() { assertEquals(gauge, metrics.get("mygauge")); assertEquals(timer, metrics.get("mytimer")); } + + @Test + public void testRemove() { + CounterImpl counter1 = new CounterImpl(); + CounterImpl counter2 = new CounterImpl(); + registry.register("mycounter", counter1); + + boolean removed = registry.remove("mycounter", counter2); + Optional metric = registry.get("mycounter"); + assertFalse(removed); + assertTrue(metric.isPresent()); + assertEquals(metric.get(), counter1); + + removed = registry.remove("mycounter"); + metric = registry.get("mycounter"); + assertTrue(removed); + assertFalse(metric.isPresent()); + } } diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java index 4d1d5c1ccd9a..61f73544b1bb 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/AbstractProcedureScheduler.java @@ -236,7 +236,7 @@ public long getNullPollCalls() { // ========================================================================== /** - * Wake up all of the given events. Note that we first take scheduler lock and then wakeInternal() + * Wake up all the given events. Note that we first take scheduler lock and then wakeInternal() * synchronizes on the event. Access should remain package-private. Use ProcedureEvent class to * wake/suspend events. * @param events the list of events to wake diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index 3099c64e00f6..e01a27d74675 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -32,8 +32,11 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -237,6 +240,12 @@ public interface ProcedureExecutorListener { */ private TimeoutExecutorThread workerMonitorExecutor; + private ExecutorService forceUpdateExecutor; + + // A thread pool for executing some asynchronous tasks for procedures, you can find references to + // getAsyncTaskExecutor to see the usage + private ExecutorService asyncTaskExecutor; + private int corePoolSize; private int maxPoolSize; @@ -247,9 +256,6 @@ public interface ProcedureExecutorListener { */ private final ProcedureScheduler scheduler; - private final Executor forceUpdateExecutor = Executors.newSingleThreadExecutor( - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build()); - private final AtomicLong lastProcId = new AtomicLong(-1); private final AtomicLong workerId = new AtomicLong(0); private final AtomicInteger activeExecutorCount = new AtomicInteger(0); @@ -317,19 +323,6 @@ public ProcedureExecutor(final Configuration conf, final TEnvironment environmen this.conf = conf; this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET); refreshConfiguration(conf); - store.registerListener(new ProcedureStoreListener() { - - @Override - public void forceUpdate(long[] procIds) { - Arrays.stream(procIds).forEach(procId -> forceUpdateExecutor.execute(() -> { - try { - forceUpdateProcedure(procId); - } catch (IOException e) { - LOG.warn("Failed to force update procedure with pid={}", procId); - } - })); - } - }); } private void load(final boolean abortOnCorruption) throws IOException { @@ -613,6 +606,36 @@ public void init(int numThreads, boolean abortOnCorruption) throws IOException { this.threadGroup = new ThreadGroup("PEWorkerGroup"); this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, "ProcExecTimeout"); this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, threadGroup, "WorkerMonitor"); + ThreadFactory backingThreadFactory = new ThreadFactory() { + + @Override + public Thread newThread(Runnable r) { + return new Thread(threadGroup, r); + } + }; + int size = Math.max(2, Runtime.getRuntime().availableProcessors()); + ThreadPoolExecutor executor = + new ThreadPoolExecutor(size, size, 1, TimeUnit.MINUTES, new LinkedBlockingQueue(), + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat(getClass().getSimpleName() + "-Async-Task-Executor-%d") + .setThreadFactory(backingThreadFactory).build()); + executor.allowCoreThreadTimeOut(true); + this.asyncTaskExecutor = executor; + forceUpdateExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("Force-Update-PEWorker-%d").setThreadFactory(backingThreadFactory).build()); + store.registerListener(new ProcedureStoreListener() { + + @Override + public void forceUpdate(long[] procIds) { + Arrays.stream(procIds).forEach(procId -> forceUpdateExecutor.execute(() -> { + try { + forceUpdateProcedure(procId); + } catch (IOException e) { + LOG.warn("Failed to force update procedure with pid={}", procId); + } + })); + } + }); // Create the workers workerId.set(0); @@ -670,14 +693,16 @@ public void startWorkers() throws IOException { } public void stop() { - if (!running.getAndSet(false)) { - return; - } - + // it is possible that we fail in init, while loading procedures, so we will not set running to + // true but we should have already started the ProcedureScheduler, and also the two + // ExecutorServices, so here we do not check running state, just stop them + running.set(false); LOG.info("Stopping"); scheduler.stop(); timeoutExecutor.sendStopSignal(); workerMonitorExecutor.sendStopSignal(); + forceUpdateExecutor.shutdown(); + asyncTaskExecutor.shutdown(); } public void join() { @@ -692,14 +717,29 @@ public void join() { for (WorkerThread worker : workerThreads) { worker.awaitTermination(); } + try { + if (!forceUpdateExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + LOG.warn("There are still pending tasks in forceUpdateExecutor"); + } + } catch (InterruptedException e) { + LOG.warn("interrupted while waiting for forceUpdateExecutor termination", e); + Thread.currentThread().interrupt(); + } + try { + if (!asyncTaskExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + LOG.warn("There are still pending tasks in asyncTaskExecutor"); + } + } catch (InterruptedException e) { + LOG.warn("interrupted while waiting for asyncTaskExecutor termination", e); + Thread.currentThread().interrupt(); + } // Destroy the Thread Group for the executors // TODO: Fix. #join is not place to destroy resources. try { threadGroup.destroy(); } catch (IllegalThreadStateException e) { - LOG.error("ThreadGroup {} contains running threads; {}: See STDOUT", this.threadGroup, - e.getMessage()); + LOG.error("ThreadGroup {} contains running threads; {}: See STDOUT", this.threadGroup, e); // This dumps list of threads on STDOUT. this.threadGroup.list(); } @@ -2055,6 +2095,13 @@ public IdLock getProcExecutionLock() { return procExecutionLock; } + /** + * Get a thread pool for executing some asynchronous tasks + */ + public ExecutorService getAsyncTaskExecutor() { + return asyncTaskExecutor; + } + // ========================================================================== // Worker Thread // ========================================================================== diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml index 1926e049cb9b..b2d9f79bbd08 100644 --- a/hbase-protocol-shaded/pom.xml +++ b/hbase-protocol-shaded/pom.xml @@ -34,7 +34,7 @@ - 3.21.12 + 3.24.3 + + + diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/processRest.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/processRest.jsp new file mode 100644 index 000000000000..2b2d35fbfb3f --- /dev/null +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/processRest.jsp @@ -0,0 +1,184 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.Date" + import="java.util.List" + import="javax.management.ObjectName" + import="java.lang.management.ManagementFactory" + import="java.lang.management.MemoryPoolMXBean" + import="java.lang.management.RuntimeMXBean" + import="java.lang.management.GarbageCollectorMXBean" + import="org.apache.hadoop.hbase.util.JSONMetricUtil" + import="org.apache.hadoop.hbase.procedure2.util.StringUtils" + import="org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix" +%> + +<% +RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); +ObjectName jvmMetrics = new ObjectName("Hadoop:service=HBase,name=JvmMetrics"); + +// There is always two of GC collectors +List gcBeans = JSONMetricUtil.getGcCollectorBeans(); +GarbageCollectorMXBean collector1 = null; +GarbageCollectorMXBean collector2 = null; +try { +collector1 = gcBeans.get(0); +collector2 = gcBeans.get(1); +} catch(IndexOutOfBoundsException e) {} +List mPools = JSONMetricUtil.getMemoryPools(); +pageContext.setAttribute("pageTitle", "Process info for PID: " + JSONMetricUtil.getProcessPID()); +%> + + + + + +

+
+
+

<%= JSONMetricUtil.getCommmand().split(" ")[0] %>

+
+
+ + + + + + + + + + + + + + +
StartedUptimePIDOwner
<%= new Date(runtimeBean.getStartTime()) %><%= StringUtils.humanTimeDiff(runtimeBean.getUptime()) %><%= JSONMetricUtil.getProcessPID() %><%= runtimeBean.getSystemProperties().get("user.name") %>
+
+
+
+
+

Threads

+
+
+ + + + + + + + + + + + + + + + + +
ThreadsNewThreadsRunableThreadsBlockedThreadsWaitingThreadsTimeWaitingThreadsTerminated
<%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsNew") %><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsRunnable")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsBlocked")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsWaiting")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsTimedWaiting")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsTerminated")%>
+
+
+
+
+

GC Collectors

+
+
+ <% if (gcBeans.size() == 2) { %> +
+ +
+
+ + + + + + + + + + + +
Collection CountCollection TimeLast duration
<%= collector1.getCollectionCount() %> <%= StringUtils.humanTimeDiff(collector1.getCollectionTime()) %> <%= StringUtils.humanTimeDiff(JSONMetricUtil.getLastGcDuration( + collector1.getObjectName())) %>
+
+
+ + + + + + + + + + + +
Collection CountCollection TimeLast duration
<%= collector2.getCollectionCount() %> <%= StringUtils.humanTimeDiff(collector2.getCollectionTime()) %> <%= StringUtils.humanTimeDiff(JSONMetricUtil.getLastGcDuration( + collector2.getObjectName())) %>
+
+
+
+ <%} else { %> +

Can not display GC Collector stats.

+ <%} %> + Total GC Collection time: <%= StringUtils.humanTimeDiff(collector1.getCollectionTime() + + collector2.getCollectionTime())%> +
+<% for(MemoryPoolMXBean mp:mPools) { +if(mp.getName().contains("Cache")) continue;%> +
+
+
+

<%= mp.getName() %>

+
+
+ + + + + + + + + + + + + + + + +
CommitedInitMaxUsedUtilization [%]
<%= TraditionalBinaryPrefix.long2String(mp.getUsage().getCommitted(), "B", 1) %><%= TraditionalBinaryPrefix.long2String(mp.getUsage().getInit(), "B", 1) %><%= TraditionalBinaryPrefix.long2String(mp.getUsage().getMax(), "B", 1) %><%= TraditionalBinaryPrefix.long2String(mp.getUsage().getUsed(), "B", 1) %><%= JSONMetricUtil.calcPercentage(mp.getUsage().getUsed(), + mp.getUsage().getCommitted()) %>
+
+<% } %> + + diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp index df8f0838d6cc..ce6725f283a7 100644 --- a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp @@ -18,70 +18,29 @@ */ --%> <%@ page contentType="text/html;charset=UTF-8" - import="org.apache.hadoop.conf.Configuration" - import="org.apache.hadoop.hbase.HBaseConfiguration" - import="org.apache.hadoop.hbase.rest.model.VersionModel" - import="org.apache.hadoop.hbase.util.VersionInfo" - import="java.util.Date"%> + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.rest.RESTServer" + import="org.apache.hadoop.hbase.rest.model.VersionModel" + import="org.apache.hadoop.hbase.util.VersionInfo" + import="java.util.Date"%> + <% -Configuration conf = (Configuration)getServletContext().getAttribute("hbase.conf"); -long startcode = conf.getLong("startcode", System.currentTimeMillis()); -String listenPort = conf.get("hbase.rest.port", "8080"); -%> - - - - - - HBase REST Server: <%= listenPort %> - - + Configuration conf = (Configuration) getServletContext().getAttribute("hbase.conf"); + long startcode = conf.getLong("startcode", System.currentTimeMillis()); - - - - + final RESTServer restServer = (RESTServer) getServletContext().getAttribute(RESTServer.REST_SERVER); + final String hostName = restServer.getServerName().getHostname(); + pageContext.setAttribute("pageTitle", "HBase REST Server" + hostName); +%> - -
-
-
- - HBase Logo -
-
- -
-
-
+ + +
-

RESTServer <%= listenPort %>

+

RESTServer <%= hostName %>

@@ -124,9 +83,6 @@ String listenPort = conf.get("hbase.rest.port", "8080");
- - - - - + + diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon index 0df4d2763b0b..e77318437e04 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon @@ -38,6 +38,7 @@ org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad; org.apache.hadoop.hbase.client.RegionReplicaUtil; org.apache.hadoop.hbase.regionserver.MetricsRegionWrapper; + org.apache.hadoop.util.StringUtils; org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; <%if (onlineRegions != null && onlineRegions.size() > 0) %> @@ -172,6 +173,7 @@ Bloom Size Data Locality Len Of Biggest Cell + % Cached @@ -237,6 +239,7 @@ <% bloomSizeStr %> <% load.getDataLocality() %> <% String.format("%,1d", lenOfBiggestCellInRegion) %> + <% StringUtils.formatPercent(load.getCurrentRegionCachedRatio(), 2) %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java index 291b38acb322..b2a0e7803624 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseRpcServicesBase.java @@ -310,6 +310,7 @@ public final GetBootstrapNodesResponse getBootstrapNodes(RpcController controlle } @Override + @QosPriority(priority = HConstants.ADMIN_QOS) public UpdateConfigurationResponse updateConfiguration(RpcController controller, UpdateConfigurationRequest request) throws ServiceException { try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java index 36f4f3addf5c..f64f2947ee96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java @@ -46,16 +46,19 @@ import org.apache.hadoop.hbase.conf.ConfigurationManager; import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.http.InfoServer; import org.apache.hadoop.hbase.io.util.MemorySizeUtil; import org.apache.hadoop.hbase.ipc.RpcServerInterface; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.namequeues.NamedQueueRecorder; import org.apache.hadoop.hbase.regionserver.ChunkCreator; import org.apache.hadoop.hbase.regionserver.HeapMemoryManager; import org.apache.hadoop.hbase.regionserver.MemStoreLAB; +import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost; import org.apache.hadoop.hbase.regionserver.ShutdownHook; import org.apache.hadoop.hbase.security.Superusers; import org.apache.hadoop.hbase.security.User; @@ -184,14 +187,14 @@ public abstract class HBaseServerBase> extends protected final NettyEventLoopGroupConfig eventLoopGroupConfig; - /** - * If running on Windows, do windows-specific setup. - */ - private static void setupWindows(final Configuration conf, ConfigurationManager cm) { + private void setupSignalHandlers() { if (!SystemUtils.IS_OS_WINDOWS) { HBasePlatformDependent.handle("HUP", (number, name) -> { - conf.reloadConfiguration(); - cm.notifyAllObservers(conf); + try { + updateConfiguration(); + } catch (IOException e) { + LOG.error("Problem while reloading configuration", e); + } }); } } @@ -276,7 +279,7 @@ public HBaseServerBase(Configuration conf, String name) throws IOException { new ZKWatcher(conf, getProcessName() + ":" + addr.getPort(), this, canCreateBaseZNode()); this.configurationManager = new ConfigurationManager(); - setupWindows(conf, configurationManager); + setupSignalHandlers(); initializeFileSystem(); @@ -614,11 +617,31 @@ public ConfigurationManager getConfigurationManager() { /** * Reload the configuration from disk. */ - public void updateConfiguration() { + public void updateConfiguration() throws IOException { LOG.info("Reloading the configuration from disk."); // Reload the configuration from disk. + preUpdateConfiguration(); conf.reloadConfiguration(); configurationManager.notifyAllObservers(conf); + postUpdateConfiguration(); + } + + private void preUpdateConfiguration() throws IOException { + CoprocessorHost coprocessorHost = getCoprocessorHost(); + if (coprocessorHost instanceof RegionServerCoprocessorHost) { + ((RegionServerCoprocessorHost) coprocessorHost).preUpdateConfiguration(conf); + } else if (coprocessorHost instanceof MasterCoprocessorHost) { + ((MasterCoprocessorHost) coprocessorHost).preUpdateConfiguration(conf); + } + } + + private void postUpdateConfiguration() throws IOException { + CoprocessorHost coprocessorHost = getCoprocessorHost(); + if (coprocessorHost instanceof RegionServerCoprocessorHost) { + ((RegionServerCoprocessorHost) coprocessorHost).postUpdateConfiguration(conf); + } else if (coprocessorHost instanceof MasterCoprocessorHost) { + ((MasterCoprocessorHost) coprocessorHost).postUpdateConfiguration(conf); + } } @Override @@ -626,6 +649,8 @@ public String toString() { return getServerName().toString(); } + protected abstract CoprocessorHost getCoprocessorHost(); + protected abstract boolean canCreateBaseZNode(); protected abstract String getProcessName(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java index 191910441409..144c01de874a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; import org.apache.hadoop.hbase.mob.MobFileCache; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -46,6 +47,8 @@ public class ClientSideRegionScanner extends AbstractClientScanner { private static final Logger LOG = LoggerFactory.getLogger(ClientSideRegionScanner.class); private HRegion region; + private MobFileCache mobFileCache; + private BlockCache blockCache; RegionScanner scanner; List values; boolean hasMore = true; @@ -69,12 +72,14 @@ public ClientSideRegionScanner(Configuration conf, FileSystem fs, Path rootDir, String.valueOf(HConstants.HBASE_CLIENT_SCANNER_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT)); // don't allow L2 bucket cache for non RS process to avoid unexpected disk usage. conf.unset(HConstants.BUCKET_CACHE_IOENGINE_KEY); - region.setBlockCache(BlockCacheFactory.createBlockCache(conf)); + blockCache = BlockCacheFactory.createBlockCache(conf); + region.setBlockCache(blockCache); // we won't initialize the MobFileCache when not running in RS process. so provided an // initialized cache. Consider the case: an CF was set from an mob to non-mob. if we only // initialize cache for MOB region, NPE from HMobStore will still happen. So Initialize the // cache for every region although it may hasn't any mob CF, BTW the cache is very light-weight. - region.setMobFileCache(new MobFileCache(conf)); + mobFileCache = new MobFileCache(conf); + region.setMobFileCache(mobFileCache); region.initialize(); // create an internal region scanner @@ -131,6 +136,19 @@ public void close() { LOG.warn("Exception while closing region", ex); } } + + // In typical region operation, RegionServerServices would handle the lifecycle of + // the MobFileCache and BlockCache. In ClientSideRegionScanner, we need to handle + // the lifecycle of these components ourselves to avoid resource leaks. + if (mobFileCache != null) { + mobFileCache.shutdown(); + mobFileCache = null; + } + + if (blockCache != null) { + blockCache.shutdown(); + blockCache = null; + } } HRegion getRegion() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java index 579da46af1c1..7225f92b7ff9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java @@ -64,7 +64,7 @@ private static AsyncClusterConnection createAsyncClusterConnection(Configuration */ public static AsyncClusterConnection createAsyncClusterConnection(Configuration conf, SocketAddress localAddress, User user) throws IOException { - return createAsyncClusterConnection(conf, ConnectionRegistryFactory.getRegistry(conf), + return createAsyncClusterConnection(conf, ConnectionRegistryFactory.getRegistry(conf, user), localAddress, user); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 820fef71fd07..d0e451508b43 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.MetaMutationAnnotation; @@ -1873,4 +1874,24 @@ default void preHasUserPermissions(ObserverContext default void postHasUserPermissions(ObserverContext ctx, String userName, List permissions) throws IOException { } + + /** + * Called before reloading the HMaster's {@link Configuration} from disk + * @param ctx the coprocessor instance's environment + * @param preReloadConf the {@link Configuration} in use prior to reload + * @throws IOException if you need to signal an IO error + */ + default void preUpdateMasterConfiguration(ObserverContext ctx, + Configuration preReloadConf) throws IOException { + } + + /** + * Called after reloading the HMaster's {@link Configuration} from disk + * @param ctx the coprocessor instance's environment + * @param postReloadConf the {@link Configuration} that was loaded + * @throws IOException if you need to signal an IO error + */ + default void postUpdateMasterConfiguration(ObserverContext ctx, + Configuration postReloadConf) throws IOException { + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java index 236667b4be7b..b6915ffaaeac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.coprocessor; import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CacheEvictionStats; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.replication.ReplicationEndpoint; @@ -169,4 +171,45 @@ default void postReplicationSinkBatchMutate( } + /** + * Called before clearing the block caches for one or more regions + * @param ctx the coprocessor instance's environment + * @throws IOException if you need to signal an IO error + */ + default void preClearRegionBlockCache(ObserverContext ctx) + throws IOException { + } + + /** + * Called after clearing the block caches for one or more regions + * @param ctx the coprocessor instance's environment + * @param stats statistics about the cache evictions that happened + * @throws IOException if you need to signal an IO error + */ + default void postClearRegionBlockCache(ObserverContext ctx, + CacheEvictionStats stats) throws IOException { + } + + /** + * Called before reloading the RegionServer's {@link Configuration} from disk + * @param ctx the coprocessor instance's environment + * @param preReloadConf the {@link Configuration} in use prior to reload + * @throws IOException if you need to signal an IO error + */ + default void preUpdateRegionServerConfiguration( + ObserverContext ctx, Configuration preReloadConf) + throws IOException { + } + + /** + * Called after reloading the RegionServer's {@link Configuration} from disk + * @param ctx the coprocessor instance's environment + * @param postReloadConf the {@link Configuration} that was loaded + * @throws IOException if you need to signal an IO error + */ + default void postUpdateRegionServerConfiguration( + ObserverContext ctx, Configuration postReloadConf) + throws IOException { + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/ErasureCodingUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/ErasureCodingUtils.java new file mode 100644 index 000000000000..6e3c1e9a7887 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/ErasureCodingUtils.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs; + +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.Private +public final class ErasureCodingUtils { + + private ErasureCodingUtils() { + } + + private static final Logger LOG = LoggerFactory.getLogger(ErasureCodingUtils.class); + + /** + * Runs checks against the FileSystem, verifying that HDFS is supported and the policy is + * available, enabled, and works with a simple write. + */ + public static void verifySupport(Configuration conf, String policy) throws HBaseIOException { + DistributedFileSystem dfs = getDfs(conf); + checkAvailable(dfs, policy); + + // Enable the policy on a test directory. Try writing ot it to ensure that HDFS allows it + // This acts as a safeguard against topology issues (not enough nodes for policy, etc) and + // anything else. This is otherwise hard to validate more directly. + Path globalTempDir = new Path(conf.get(HConstants.HBASE_DIR), HConstants.HBASE_TEMP_DIRECTORY); + Path currentTempDir = createTempDir(dfs, globalTempDir); + try { + setPolicy(dfs, currentTempDir, policy); + try (FSDataOutputStream out = dfs.create(new Path(currentTempDir, "test.out"))) { + out.writeUTF("Testing " + policy); + } + } catch (IOException e) { + throw new DoNotRetryIOException("Failed write test for EC policy. Check cause or logs", e); + } finally { + try { + dfs.delete(currentTempDir, true); + } catch (IOException e) { + LOG.warn("Failed to delete temp path for ec test", e); + } + } + } + + private static Path createTempDir(FileSystem fs, Path tempDir) throws HBaseIOException { + Path currentTempDir = new Path(tempDir, "ec-test-" + System.currentTimeMillis()); + try { + fs.mkdirs(currentTempDir); + fs.deleteOnExit(currentTempDir); + } catch (IOException e) { + throw new HBaseIOException("Failed to create test dir for EC write test", e); + } + return currentTempDir; + } + + private static void checkAvailable(DistributedFileSystem dfs, String policy) + throws HBaseIOException { + Collection policies; + try { + policies = dfs.getAllErasureCodingPolicies(); + } catch (IOException e) { + throw new HBaseIOException("Failed to check for Erasure Coding policy: " + policy, e); + } + for (ErasureCodingPolicyInfo policyInfo : policies) { + if (policyInfo.getPolicy().getName().equals(policy)) { + if (!policyInfo.isEnabled()) { + throw new DoNotRetryIOException("Cannot set Erasure Coding policy: " + policy + + ". The policy must be enabled, but has state " + policyInfo.getState()); + } + return; + } + } + throw new DoNotRetryIOException( + "Cannot set Erasure Coding policy: " + policy + ". Policy not found. Available policies are: " + + policies.stream().map(p -> p.getPolicy().getName()).collect(Collectors.joining(", "))); + } + + /** + * Check if EC policy is different between two descriptors + * @return true if a sync is necessary + */ + public static boolean needsSync(TableDescriptor oldDescriptor, TableDescriptor newDescriptor) { + String newPolicy = oldDescriptor.getErasureCodingPolicy(); + String oldPolicy = newDescriptor.getErasureCodingPolicy(); + return !Objects.equals(oldPolicy, newPolicy); + } + + /** + * Sync the EC policy state from the newDescriptor onto the FS for the table dir of the provided + * table descriptor. If the policy is null, we will remove erasure coding from the FS for the + * table dir. If it's non-null, we'll set it to that policy. + * @param newDescriptor descriptor containing the policy and table name + */ + public static void sync(FileSystem fs, Path rootDir, TableDescriptor newDescriptor) + throws IOException { + String newPolicy = newDescriptor.getErasureCodingPolicy(); + if (newPolicy == null) { + unsetPolicy(fs, rootDir, newDescriptor.getTableName()); + } else { + setPolicy(fs, rootDir, newDescriptor.getTableName(), newPolicy); + } + } + + /** + * Sets the EC policy on the table directory for the specified table + */ + public static void setPolicy(FileSystem fs, Path rootDir, TableName tableName, String policy) + throws IOException { + Path path = CommonFSUtils.getTableDir(rootDir, tableName); + setPolicy(fs, path, policy); + } + + /** + * Sets the EC policy on the path + */ + public static void setPolicy(FileSystem fs, Path path, String policy) throws IOException { + getDfs(fs).setErasureCodingPolicy(path, policy); + } + + /** + * Unsets any EC policy specified on the path. + */ + public static void unsetPolicy(FileSystem fs, Path rootDir, TableName tableName) + throws IOException { + DistributedFileSystem dfs = getDfs(fs); + Path path = CommonFSUtils.getTableDir(rootDir, tableName); + if (dfs.getErasureCodingPolicy(path) == null) { + LOG.warn("No EC policy set for path {}, nothing to unset", path); + return; + } + dfs.unsetErasureCodingPolicy(path); + } + + private static DistributedFileSystem getDfs(Configuration conf) throws HBaseIOException { + try { + return getDfs(FileSystem.get(conf)); + } catch (DoNotRetryIOException e) { + throw e; + } catch (IOException e) { + throw new HBaseIOException("Failed to get FileSystem from conf", e); + } + + } + + private static DistributedFileSystem getDfs(FileSystem fs) throws DoNotRetryIOException { + if (!(fs instanceof DistributedFileSystem)) { + throw new DoNotRetryIOException( + "Cannot manage Erasure Coding policy. Erasure Coding is only available on HDFS, but fs is " + + fs.getClass().getSimpleName()); + } + return (DistributedFileSystem) fs; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java index cb9dc84b94be..33eace47d632 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java @@ -19,17 +19,13 @@ import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.fs.CanUnbuffer; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; @@ -40,8 +36,6 @@ */ @InterfaceAudience.Private public class FSDataInputStreamWrapper implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(FSDataInputStreamWrapper.class); - private static final boolean isLogTraceEnabled = LOG.isTraceEnabled(); private final HFileSystem hfs; private final Path path; @@ -94,9 +88,6 @@ private static class ReadStatistics { long totalZeroCopyBytesRead; } - private Boolean instanceOfCanUnbuffer = null; - private CanUnbuffer unbuffer = null; - protected Path readerPath; public FSDataInputStreamWrapper(FileSystem fs, Path path) throws IOException { @@ -314,41 +305,22 @@ public HFileSystem getHfs() { * stream, the current socket will be closed and a new socket will be opened to serve the * requests. */ - @SuppressWarnings({ "rawtypes" }) public void unbuffer() { + // todo: it may make sense to always unbuffer both streams. we'd need to carefully + // research the usages to know if that is safe. for now just do the current. FSDataInputStream stream = this.getStream(this.shouldUseHBaseChecksum()); if (stream != null) { - InputStream wrappedStream = stream.getWrappedStream(); - // CanUnbuffer interface was added as part of HDFS-7694 and the fix is available in Hadoop - // 2.6.4+ and 2.7.1+ versions only so check whether the stream object implements the - // CanUnbuffer interface or not and based on that call the unbuffer api. - final Class streamClass = wrappedStream.getClass(); - if (this.instanceOfCanUnbuffer == null) { - // To ensure we compute whether the stream is instance of CanUnbuffer only once. - this.instanceOfCanUnbuffer = false; - if (wrappedStream instanceof CanUnbuffer) { - this.unbuffer = (CanUnbuffer) wrappedStream; - this.instanceOfCanUnbuffer = true; - } - } - if (this.instanceOfCanUnbuffer) { - try { - this.unbuffer.unbuffer(); - } catch (UnsupportedOperationException e) { - if (isLogTraceEnabled) { - LOG.trace("Failed to invoke 'unbuffer' method in class " + streamClass - + " . So there may be the stream does not support unbuffering.", e); - } - } - } else { - if (isLogTraceEnabled) { - LOG.trace("Failed to find 'unbuffer' method in class " + streamClass); - } - } + stream.unbuffer(); } } public Path getReaderPath() { return readerPath; } + + // For tests + void setShouldUseHBaseChecksum() { + useHBaseChecksumConfigured = true; + useHBaseChecksum = true; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/MetricsIO.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/MetricsIO.java index 58e6f7d01b71..4d2437d418bc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/MetricsIO.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/MetricsIO.java @@ -75,4 +75,8 @@ public void updateFsPreadTime(long t) { public void updateFsWriteTime(long t) { source.updateFsWriteTime(t); } + + public void incrSlowFsRead() { + source.incrSlowFsRead(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index a62ca853ca60..bed0194b1fab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -169,21 +169,16 @@ default boolean isMetaBlock(BlockType blockType) { /** * Notifies the cache implementation that the given file has been fully cached (all its blocks * made into the cache). - * @param fileName the file that has been completely cached. + * @param fileName the file that has been completely cached. + * @param totalBlockCount the total of blocks cached for this file. + * @param dataBlockCount number of DATA block type cached. + * @param size the size, in bytes, cached. */ default void notifyFileCachingCompleted(Path fileName, int totalBlockCount, int dataBlockCount, long size) { // noop } - /** - * Notifies the cache implementation that the given file had a block evicted - * @param fileName the file had a block evicted. - */ - default void notifyFileBlockEvicted(String fileName) { - // noop - } - /** * Checks whether there's enough space left in the cache to accommodate the passed block. This * method may not be overridden by all implementing classes. In such cases, the returned Optional @@ -245,4 +240,14 @@ default Optional getBlockSize(BlockCacheKey key) { default Optional>> getFullyCachedFiles() { return Optional.empty(); } + + /** + * Returns an Optional containing a map of regions and the percentage of how much of it has been + * cached so far. + * @return empty optional if this method is not supported, otherwise the returned optional + * contains a map of current regions caching percentage. + */ + default Optional> getRegionCachedInfo() { + return Optional.empty(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java index 1cfdc5868be7..bf22d38e373b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.yetus.audience.InterfaceAudience; @@ -31,6 +32,7 @@ public class BlockCacheKey implements HeapSize, java.io.Serializable { private final long offset; private BlockType blockType; private final boolean isPrimaryReplicaBlock; + private Path filePath; /** * Construct a new BlockCacheKey @@ -49,6 +51,14 @@ public BlockCacheKey(String hfileName, long offset, boolean isPrimaryReplica, this.blockType = blockType; } + public BlockCacheKey(Path hfilePath, long offset, boolean isPrimaryReplica, BlockType blockType) { + this.filePath = hfilePath; + this.isPrimaryReplicaBlock = isPrimaryReplica; + this.hfileName = hfilePath.getName(); + this.offset = offset; + this.blockType = blockType; + } + @Override public int hashCode() { return hfileName.hashCode() * 127 + (int) (offset ^ (offset >>> 32)); @@ -102,4 +112,9 @@ public BlockType getBlockType() { public void setBlockType(BlockType blockType) { this.blockType = blockType; } + + public Path getFilePath() { + return filePath; + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 4587eced6163..f89a6194cefb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -99,6 +99,12 @@ public class CacheConfig { public static final String BUCKETCACHE_PERSIST_INTERVAL_KEY = "hbase.bucketcache.persist.intervalinmillis"; + /** + * Configuration key to set the heap usage threshold limit once prefetch threads should be + * interrupted. + */ + public static final String PREFETCH_HEAP_USAGE_THRESHOLD = "hbase.rs.prefetchheapusage"; + // Defaults public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false; @@ -111,6 +117,7 @@ public class CacheConfig { public static final boolean DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE = false; public static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true; public static final long DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD = Long.MAX_VALUE; + public static final double DEFAULT_PREFETCH_HEAP_USAGE_THRESHOLD = 1d; /** * Whether blocks should be cached on read (default is on if there is a cache but this can be @@ -157,6 +164,8 @@ public class CacheConfig { private final ByteBuffAllocator byteBuffAllocator; + private final double heapUsageThreshold; + /** * Create a cache configuration using the specified configuration object and defaults for family * level settings. Only use if no column family context. @@ -201,6 +210,8 @@ public CacheConfig(Configuration conf, ColumnFamilyDescriptor family, BlockCache this.cacheCompactedDataOnWrite = conf.getBoolean(CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE); this.cacheCompactedDataOnWriteThreshold = getCacheCompactedBlocksOnWriteThreshold(conf); + this.heapUsageThreshold = + conf.getDouble(PREFETCH_HEAP_USAGE_THRESHOLD, DEFAULT_PREFETCH_HEAP_USAGE_THRESHOLD); this.blockCache = blockCache; this.byteBuffAllocator = byteBuffAllocator; } @@ -222,6 +233,7 @@ public CacheConfig(CacheConfig cacheConf) { this.dropBehindCompaction = cacheConf.dropBehindCompaction; this.blockCache = cacheConf.blockCache; this.byteBuffAllocator = cacheConf.byteBuffAllocator; + this.heapUsageThreshold = cacheConf.heapUsageThreshold; } private CacheConfig() { @@ -237,6 +249,7 @@ private CacheConfig() { this.dropBehindCompaction = false; this.blockCache = null; this.byteBuffAllocator = ByteBuffAllocator.HEAP; + this.heapUsageThreshold = DEFAULT_PREFETCH_HEAP_USAGE_THRESHOLD; } /** @@ -386,6 +399,17 @@ public boolean shouldReadBlockFromCache(BlockType blockType) { return false; } + /** + * Checks if the current heap usage is below the threshold configured by + * "hbase.rs.prefetchheapusage" (0.8 by default). + */ + public boolean isHeapUsageBelowThreshold() { + double total = Runtime.getRuntime().maxMemory(); + double available = Runtime.getRuntime().freeMemory(); + double usedRatio = 1d - (available / total); + return heapUsageThreshold > usedRatio; + } + /** * If we make sure the block could not be cached, we will not acquire the lock otherwise we will * acquire lock @@ -413,6 +437,10 @@ public ByteBuffAllocator getByteBuffAllocator() { return this.byteBuffAllocator; } + public double getHeapUsageThreshold() { + return heapUsageThreshold; + } + private long getCacheCompactedBlocksOnWriteThreshold(Configuration conf) { long cacheCompactedBlocksOnWriteThreshold = conf.getLong(CACHE_COMPACTED_BLOCKS_ON_WRITE_THRESHOLD_KEY, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 427f1771e669..d6692d2e2bf1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -439,6 +439,11 @@ public Optional>> getFullyCachedFiles() { return this.l2Cache.getFullyCachedFiles(); } + @Override + public Optional> getRegionCachedInfo() { + return l2Cache.getRegionCachedInfo(); + } + @Override public void setMaxSize(long size) { this.l1Cache.setMaxSize(size); @@ -467,12 +472,6 @@ public void notifyFileCachingCompleted(Path fileName, int totalBlockCount, int d } - @Override - public void notifyFileBlockEvicted(String fileName) { - l1Cache.notifyFileBlockEvicted(fileName); - l1Cache.notifyFileBlockEvicted(fileName); - } - @Override public Optional blockFitsIntoTheCache(HFileBlock block) { if (isMetaBlock(block.getBlockType())) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 207c99866511..84fe9387d6e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; +import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.CellSink; import org.apache.hadoop.hbase.regionserver.ShipperListener; import org.apache.hadoop.hbase.util.BloomFilterWriter; @@ -187,12 +188,16 @@ public static final long getChecksumFailuresCount() { return CHECKSUM_FAILURES.sum(); } - public static final void updateReadLatency(long latencyMillis, boolean pread) { + public static final void updateReadLatency(long latencyMillis, boolean pread, boolean tooSlow) { + RpcServer.getCurrentCall().ifPresent(call -> call.updateFsReadTime(latencyMillis)); if (pread) { MetricsIO.getInstance().updateFsPreadTime(latencyMillis); } else { MetricsIO.getInstance().updateFsReadTime(latencyMillis); } + if (tooSlow) { + MetricsIO.getInstance().incrSlowFsRead(); + } } public static final void updateWriteLatency(long latencyMillis) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index a3ead34730fb..47c20b691b4a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -1826,8 +1826,9 @@ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, int sizeWithoutChecksum = curBlock.getInt(Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX); curBlock.limit(sizeWithoutChecksum); long duration = EnvironmentEdgeManager.currentTime() - startTime; + boolean tooSlow = this.readWarnTime >= 0 && duration > this.readWarnTime; if (updateMetrics) { - HFile.updateReadLatency(duration, pread); + HFile.updateReadLatency(duration, pread, tooSlow); } // The onDiskBlock will become the headerAndDataBuffer for this block. // If nextBlockOnDiskSizeWithHeader is not zero, the onDiskBlock already @@ -1839,7 +1840,7 @@ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, hFileBlock.sanityCheckUncompressed(); } LOG.trace("Read {} in {} ms", hFileBlock, duration); - if (!LOG.isTraceEnabled() && this.readWarnTime >= 0 && duration > this.readWarnTime) { + if (!LOG.isTraceEnabled() && tooSlow) { LOG.warn("Read Block Slow: read {} cost {} ms, threshold = {} ms", hFileBlock, duration, this.readWarnTime); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java index 92f6a8169f32..6063ffe68891 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java @@ -106,13 +106,23 @@ public void run() { HFileBlock block = prefetchStreamReader.readBlock(offset, onDiskSizeOfNextBlock, /* cacheBlock= */true, /* pread= */false, false, false, null, null, true); try { - if (!cacheConf.isInMemory() && !cache.blockFitsIntoTheCache(block).orElse(true)) { - LOG.warn( - "Interrupting prefetch for file {} because block {} of size {} " - + "doesn't fit in the available cache space.", - path, cacheKey, block.getOnDiskSizeWithHeader()); - interrupted = true; - break; + if (!cacheConf.isInMemory()) { + if (!cache.blockFitsIntoTheCache(block).orElse(true)) { + LOG.warn( + "Interrupting prefetch for file {} because block {} of size {} " + + "doesn't fit in the available cache space.", + path, cacheKey, block.getOnDiskSizeWithHeader()); + interrupted = true; + break; + } + if (!cacheConf.isHeapUsageBelowThreshold()) { + LOG.warn( + "Interrupting prefetch because heap usage is above the threshold: {} " + + "configured via {}", + cacheConf.getHeapUsageThreshold(), CacheConfig.PREFETCH_HEAP_USAGE_THRESHOLD); + interrupted = true; + break; + } } onDiskSizeOfNextBlock = block.getNextBlockOnDiskSize(); offset += block.getOnDiskSizeWithHeader(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 9d431428f376..e0f27af71458 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -1290,7 +1290,7 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo // from doing). BlockCacheKey cacheKey = - new BlockCacheKey(name, dataBlockOffset, this.isPrimaryReplicaReader(), expectedBlockType); + new BlockCacheKey(path, dataBlockOffset, this.isPrimaryReplicaReader(), expectedBlockType); Attributes attributes = Attributes.of(BLOCK_CACHE_KEY_KEY, cacheKey.toString()); boolean cacheable = cacheBlock && cacheIfCompactionsOff(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 0d5104572605..57f71b31894e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -165,7 +165,7 @@ public class BucketCache implements BlockCache, HeapSize { * Map of region -> total size of the region prefetched on this region server. This is the total * size of hFiles for this region prefetched on this region server */ - final Map regionCachedSizeMap = new ConcurrentHashMap<>(); + final Map regionCachedSize = new ConcurrentHashMap<>(); private BucketCachePersister cachePersister; @@ -348,7 +348,7 @@ public BucketCache(String ioEngineName, long capacity, int blockSize, int[] buck fullyCachedFiles.clear(); backingMapValidated.set(true); bucketAllocator = new BucketAllocator(capacity, bucketSizes); - regionCachedSizeMap.clear(); + regionCachedSize.clear(); } } else { bucketAllocator = new BucketAllocator(capacity, bucketSizes); @@ -645,7 +645,7 @@ void blockEvicted(BlockCacheKey cacheKey, BucketEntry bucketEntry, boolean decre if (decrementBlockNumber) { this.blockNumber.decrement(); if (ioEngine.isPersistent()) { - removeFileFromPrefetch(cacheKey.getHfileName()); + fileNotFullyCached(cacheKey.getHfileName()); } } if (evictedByEvictionProcess) { @@ -656,6 +656,42 @@ void blockEvicted(BlockCacheKey cacheKey, BucketEntry bucketEntry, boolean decre } } + private void fileNotFullyCached(String hfileName) { + // Update the regionPrefetchedSizeMap before removing the file from prefetchCompleted + if (fullyCachedFiles.containsKey(hfileName)) { + Pair regionEntry = fullyCachedFiles.get(hfileName); + String regionEncodedName = regionEntry.getFirst(); + long filePrefetchSize = regionEntry.getSecond(); + LOG.debug("Removing file {} for region {}", hfileName, regionEncodedName); + regionCachedSize.computeIfPresent(regionEncodedName, (rn, pf) -> pf - filePrefetchSize); + // If all the blocks for a region are evicted from the cache, remove the entry for that region + if ( + regionCachedSize.containsKey(regionEncodedName) + && regionCachedSize.get(regionEncodedName) == 0 + ) { + regionCachedSize.remove(regionEncodedName); + } + } + fullyCachedFiles.remove(hfileName); + } + + public void fileCacheCompleted(Path filePath, long size) { + Pair pair = new Pair<>(); + // sets the region name + String regionName = filePath.getParent().getParent().getName(); + pair.setFirst(regionName); + pair.setSecond(size); + fullyCachedFiles.put(filePath.getName(), pair); + } + + private void updateRegionCachedSize(Path filePath, long cachedSize) { + if (filePath != null) { + String regionName = filePath.getParent().getParent().getName(); + regionCachedSize.merge(regionName, cachedSize, + (previousSize, newBlockSize) -> previousSize + newBlockSize); + } + } + /** * Free the {{@link BucketEntry} actually,which could only be invoked when the * {@link BucketEntry#refCnt} becoming 0. @@ -1087,6 +1123,7 @@ public void run() { protected void putIntoBackingMap(BlockCacheKey key, BucketEntry bucketEntry) { BucketEntry previousEntry = backingMap.put(key, bucketEntry); blocksByHFile.add(key); + updateRegionCachedSize(key.getFilePath(), bucketEntry.getLength()); if (previousEntry != null && previousEntry != bucketEntry) { previousEntry.withWriteLock(offsetLock, () -> { blockEvicted(key, previousEntry, false, false); @@ -1308,8 +1345,9 @@ public boolean isCachePersistent() { return ioEngine.isPersistent() && persistencePath != null; } - public Map getRegionCachedInfo() { - return Collections.unmodifiableMap(regionCachedSizeMap); + @Override + public Optional> getRegionCachedInfo() { + return Optional.of(Collections.unmodifiableMap(regionCachedSize)); } /** @@ -1350,17 +1388,17 @@ private void retrieveFromFile(int[] bucketSizes) throws IOException { } private void updateRegionSizeMapWhileRetrievingFromFile() { - // Update the regionCachedSizeMap with the region size while restarting the region server + // Update the regionCachedSize with the region size while restarting the region server if (LOG.isDebugEnabled()) { LOG.debug("Updating region size map after retrieving cached file list"); dumpPrefetchList(); } - regionCachedSizeMap.clear(); + regionCachedSize.clear(); fullyCachedFiles.forEach((hFileName, hFileSize) -> { // Get the region name for each file String regionEncodedName = hFileSize.getFirst(); long cachedFileSize = hFileSize.getSecond(); - regionCachedSizeMap.merge(regionEncodedName, cachedFileSize, + regionCachedSize.merge(regionEncodedName, cachedFileSize, (oldpf, fileSize) -> oldpf + fileSize); }); } @@ -1519,7 +1557,7 @@ private void disableCache() { this.backingMap.clear(); this.blocksByHFile.clear(); this.fullyCachedFiles.clear(); - this.regionCachedSizeMap.clear(); + this.regionCachedSize.clear(); } } @@ -1619,7 +1657,7 @@ protected String getAlgorithm() { */ @Override public int evictBlocksByHfileName(String hfileName) { - removeFileFromPrefetch(hfileName); + fileNotFullyCached(hfileName); Set keySet = blocksByHFile.subSet(new BlockCacheKey(hfileName, Long.MIN_VALUE), true, new BlockCacheKey(hfileName, Long.MAX_VALUE), true); @@ -2028,35 +2066,6 @@ public static Optional getBucketCacheFromCacheConfig(CacheConfig ca return Optional.empty(); } - private void removeFileFromPrefetch(String hfileName) { - // Update the regionPrefetchedSizeMap before removing the file from prefetchCompleted - if (fullyCachedFiles.containsKey(hfileName)) { - Pair regionEntry = fullyCachedFiles.get(hfileName); - String regionEncodedName = regionEntry.getFirst(); - long filePrefetchSize = regionEntry.getSecond(); - LOG.debug("Removing file {} for region {}", hfileName, regionEncodedName); - regionCachedSizeMap.computeIfPresent(regionEncodedName, (rn, pf) -> pf - filePrefetchSize); - // If all the blocks for a region are evicted from the cache, remove the entry for that region - if ( - regionCachedSizeMap.containsKey(regionEncodedName) - && regionCachedSizeMap.get(regionEncodedName) == 0 - ) { - regionCachedSizeMap.remove(regionEncodedName); - } - } - fullyCachedFiles.remove(hfileName); - } - - public void fileCacheCompleted(Path filePath, long size) { - Pair pair = new Pair<>(); - // sets the region name - String regionName = filePath.getParent().getParent().getName(); - pair.setFirst(regionName); - pair.setSecond(size); - fullyCachedFiles.put(filePath.getName(), pair); - regionCachedSizeMap.merge(regionName, size, (oldpf, fileSize) -> oldpf + fileSize); - } - @Override public void notifyFileCachingCompleted(Path fileName, int totalBlockCount, int dataBlockCount, long size) { @@ -2111,11 +2120,6 @@ public void notifyFileCachingCompleted(Path fileName, int totalBlockCount, int d } } - @Override - public void notifyFileBlockEvicted(String fileName) { - fullyCachedFiles.remove(fileName); - } - @Override public Optional blockFitsIntoTheCache(HFileBlock block) { long currentUsed = bucketAllocator.getUsedSize(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/DelegatingRpcScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingRpcScheduler.java similarity index 86% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/DelegatingRpcScheduler.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingRpcScheduler.java index f8ac4a1bb9a1..52bd43cb7f62 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/DelegatingRpcScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingRpcScheduler.java @@ -17,6 +17,17 @@ */ package org.apache.hadoop.hbase.ipc; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; + +/** + * Users of the hbase.region.server.rpc.scheduler.factory.class customization config can return an + * implementation which extends this class in order to minimize impact of breaking interface + * changes. + */ +@InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX }) +@InterfaceStability.Evolving public class DelegatingRpcScheduler extends RpcScheduler { protected RpcScheduler delegate; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java index 722ee1d28c91..1d93fbd0f668 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java @@ -17,17 +17,22 @@ */ package org.apache.hadoop.hbase.ipc; +import static org.apache.hadoop.hbase.io.crypto.tls.X509Util.DEFAULT_HBASE_SERVER_NETTY_TLS_WRAP_SIZE; import static org.apache.hadoop.hbase.io.crypto.tls.X509Util.HBASE_SERVER_NETTY_TLS_ENABLED; import static org.apache.hadoop.hbase.io.crypto.tls.X509Util.HBASE_SERVER_NETTY_TLS_SUPPORTPLAINTEXT; +import static org.apache.hadoop.hbase.io.crypto.tls.X509Util.HBASE_SERVER_NETTY_TLS_WRAP_SIZE; import static org.apache.hadoop.hbase.io.crypto.tls.X509Util.TLS_CONFIG_REVERSE_DNS_LOOKUP_ENABLED; import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.SSLPeerUnverifiedException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HBaseServerBase; @@ -58,7 +63,6 @@ import org.apache.hbase.thirdparty.io.netty.channel.WriteBufferWaterMark; import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup; import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup; -import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder; import org.apache.hbase.thirdparty.io.netty.handler.ssl.OptionalSslHandler; import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslContext; import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler; @@ -162,13 +166,15 @@ protected void initChannel(Channel ch) throws Exception { ch.config().setWriteBufferWaterMark(writeBufferWaterMark); ch.config().setAllocator(channelAllocator); ChannelPipeline pipeline = ch.pipeline(); - FixedLengthFrameDecoder preambleDecoder = new FixedLengthFrameDecoder(6); - preambleDecoder.setSingleDecode(true); + + NettyServerRpcConnection conn = createNettyServerRpcConnection(ch); + if (conf.getBoolean(HBASE_SERVER_NETTY_TLS_ENABLED, false)) { - initSSL(pipeline, conf.getBoolean(HBASE_SERVER_NETTY_TLS_SUPPORTPLAINTEXT, true)); + initSSL(pipeline, conn, conf.getBoolean(HBASE_SERVER_NETTY_TLS_SUPPORTPLAINTEXT, true)); } - NettyServerRpcConnection conn = createNettyServerRpcConnection(ch); - pipeline.addLast(NettyRpcServerPreambleHandler.DECODER_NAME, preambleDecoder) + pipeline + .addLast(NettyRpcServerPreambleHandler.DECODER_NAME, + NettyRpcServerPreambleHandler.createDecoder()) .addLast(new NettyRpcServerPreambleHandler(NettyRpcServer.this, conn)) // We need NettyRpcServerResponseEncoder here because NettyRpcServerPreambleHandler may // send RpcResponse to client. @@ -376,7 +382,7 @@ public int getNumOpenConnections() { return allChannels.size(); } - private void initSSL(ChannelPipeline p, boolean supportPlaintext) + private void initSSL(ChannelPipeline p, NettyServerRpcConnection conn, boolean supportPlaintext) throws X509Exception, IOException { SslContext nettySslContext = getSslContext(); @@ -408,11 +414,44 @@ private void initSSL(ChannelPipeline p, boolean supportPlaintext) sslHandler = nettySslContext.newHandler(p.channel().alloc()); } + sslHandler.setWrapDataSize( + conf.getInt(HBASE_SERVER_NETTY_TLS_WRAP_SIZE, DEFAULT_HBASE_SERVER_NETTY_TLS_WRAP_SIZE)); + + sslHandler.handshakeFuture() + .addListener(future -> sslHandshakeCompleteHandler(conn, sslHandler, remoteAddress)); + p.addLast("ssl", sslHandler); LOG.debug("SSL handler added for channel: {}", p.channel()); } } + static void sslHandshakeCompleteHandler(NettyServerRpcConnection conn, SslHandler sslHandler, + SocketAddress remoteAddress) { + try { + Certificate[] certificates = sslHandler.engine().getSession().getPeerCertificates(); + if (certificates != null && certificates.length > 0) { + X509Certificate[] x509Certificates = new X509Certificate[certificates.length]; + for (int i = 0; i < x509Certificates.length; i++) { + x509Certificates[i] = (X509Certificate) certificates[i]; + } + conn.clientCertificateChain = x509Certificates; + } else if (sslHandler.engine().getNeedClientAuth()) { + LOG.debug( + "Could not get peer certificate on TLS connection from {}, although one is required", + remoteAddress); + } + } catch (SSLPeerUnverifiedException e) { + if (sslHandler.engine().getNeedClientAuth()) { + LOG.debug( + "Could not get peer certificate on TLS connection from {}, although one is required", + remoteAddress, e); + } + } catch (Exception e) { + LOG.debug("Unexpected error getting peer certificate for TLS connection from {}", + remoteAddress, e); + } + } + SslContext getSslContext() throws X509Exception, IOException { SslContext result = sslContextForServer.get(); if (result == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServerPreambleHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServerPreambleHandler.java index b79a67f986e8..5aa77e0e8ace 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServerPreambleHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServerPreambleHandler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.ipc; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.ipc.ServerRpcConnection.PreambleResponse; import org.apache.hadoop.hbase.util.NettyFutureUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -25,6 +26,7 @@ import org.apache.hbase.thirdparty.io.netty.channel.ChannelHandlerContext; import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline; import org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler; +import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder; import org.apache.hbase.thirdparty.io.netty.handler.codec.LengthFieldBasedFrameDecoder; /** @@ -38,21 +40,44 @@ class NettyRpcServerPreambleHandler extends SimpleChannelInboundHandler private final NettyRpcServer rpcServer; private final NettyServerRpcConnection conn; + private boolean processPreambleError; public NettyRpcServerPreambleHandler(NettyRpcServer rpcServer, NettyServerRpcConnection conn) { this.rpcServer = rpcServer; this.conn = conn; } + static FixedLengthFrameDecoder createDecoder() { + FixedLengthFrameDecoder preambleDecoder = new FixedLengthFrameDecoder(6); + preambleDecoder.setSingleDecode(true); + return preambleDecoder; + } + @Override protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception { + if (processPreambleError) { + // if we failed to process preamble, we will close the connection immediately, but it is + // possible that we have already received some bytes after the 'preamble' so when closing, the + // netty framework will still pass them here. So we set a flag here to just skip processing + // these broken messages. + return; + } ByteBuffer buf = ByteBuffer.allocate(msg.readableBytes()); msg.readBytes(buf); buf.flip(); - if (!conn.processPreamble(buf)) { + PreambleResponse resp = conn.processPreamble(buf); + if (resp == PreambleResponse.CLOSE) { + processPreambleError = true; conn.close(); return; } + if (resp == PreambleResponse.CONTINUE) { + // we use a single decode decoder, so here we need to replace it with a new one so it will + // decode a new preamble header again + ctx.pipeline().replace(DECODER_NAME, DECODER_NAME, createDecoder()); + return; + } + // resp == PreambleResponse.SUCCEED ChannelPipeline p = ctx.pipeline(); if (conn.useSasl) { LengthFieldBasedFrameDecoder decoder = diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java index 260d6e1a9803..2d06aa7c47af 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java @@ -132,4 +132,8 @@ public interface RpcCall extends RpcCallContext { /** Returns A short string format of this call without possibly lengthy params */ String toShortString(); + + void updateFsReadTime(long latencyMillis); + + long getFsReadTime(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java index 4f299b4a85d2..43432324579b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.ipc; import java.net.InetAddress; +import java.security.cert.X509Certificate; import java.util.Optional; import org.apache.hadoop.hbase.security.User; import org.apache.yetus.audience.InterfaceAudience; @@ -60,6 +61,13 @@ default Optional getRequestUserName() { return getRequestUser().map(User::getShortName); } + /** + * Returns the TLS certificate(s) that the client presented to this HBase server when making its + * connection. TLS is orthogonal to Kerberos, so this is unrelated to + * {@link RpcCallContext#getRequestUser()}. Both, one, or neither may be present. + */ + Optional getClientCertificateChain(); + /** Returns Address of remote client in this call */ InetAddress getRemoteAddress(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index f48183d0ee68..0876a1fd55f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -443,14 +444,17 @@ public Pair call(RpcCall call, MonitoredRPCHandler status) int totalTime = (int) (endTime - receiveTime); if (LOG.isTraceEnabled()) { LOG.trace( - "{}, response: {}, receiveTime: {}, queueTime: {}, processingTime: {}, totalTime: {}", + "{}, response: {}, receiveTime: {}, queueTime: {}, processingTime: {}, " + + "totalTime: {}, fsReadTime: {}", CurCall.get().toString(), TextFormat.shortDebugString(result), - CurCall.get().getReceiveTime(), qTime, processingTime, totalTime); + CurCall.get().getReceiveTime(), qTime, processingTime, totalTime, + CurCall.get().getFsReadTime()); } // Use the raw request call size for now. long requestSize = call.getSize(); long responseSize = result.getSerializedSize(); long responseBlockSize = call.getBlockBytesScanned(); + long fsReadTime = call.getFsReadTime(); if (call.isClientCellBlockSupported()) { // Include the payload size in HBaseRpcController responseSize += call.getResponseCellSize(); @@ -471,13 +475,13 @@ public Pair call(RpcCall call, MonitoredRPCHandler status) // note that large responses will often also be slow. logResponse(param, md.getName(), md.getName() + "(" + param.getClass().getName() + ")", tooLarge, tooSlow, status.getClient(), startTime, processingTime, qTime, responseSize, - responseBlockSize, userName); + responseBlockSize, fsReadTime, userName); if (this.namedQueueRecorder != null && this.isOnlineLogProviderEnabled) { // send logs to ring buffer owned by slowLogRecorder final String className = server == null ? StringUtils.EMPTY : server.getClass().getSimpleName(); this.namedQueueRecorder.addRecord(new RpcLogDetails(call, param, status.getClient(), - responseSize, responseBlockSize, className, tooSlow, tooLarge)); + responseSize, responseBlockSize, fsReadTime, className, tooSlow, tooLarge)); } } return new Pair<>(result, controller.cellScanner()); @@ -521,7 +525,7 @@ public Pair call(RpcCall call, MonitoredRPCHandler status) */ void logResponse(Message param, String methodName, String call, boolean tooLarge, boolean tooSlow, String clientAddress, long startTime, int processingTime, int qTime, long responseSize, - long blockBytesScanned, String userName) { + long blockBytesScanned, long fsReadTime, String userName) { final String className = server == null ? StringUtils.EMPTY : server.getClass().getSimpleName(); // base information that is reported regardless of type of call Map responseInfo = new HashMap<>(); @@ -530,6 +534,7 @@ void logResponse(Message param, String methodName, String call, boolean tooLarge responseInfo.put("queuetimems", qTime); responseInfo.put("responsesize", responseSize); responseInfo.put("blockbytesscanned", blockBytesScanned); + responseInfo.put("fsreadtime", fsReadTime); responseInfo.put("client", clientAddress); responseInfo.put("class", className); responseInfo.put("method", methodName); @@ -790,7 +795,10 @@ public static Optional getRequestUserName() { return getRequestUser().map(User::getShortName); } - /** Returns Address of remote client if a request is ongoing, else null */ + /** + * Returns the address of the remote client associated with the current RPC request or not present + * if no address is set. + */ public static Optional getRemoteAddress() { return getCurrentCall().map(RpcCall::getRemoteAddress); } @@ -878,4 +886,10 @@ public void setNamedQueueRecorder(NamedQueueRecorder namedQueueRecorder) { protected boolean needAuthorization() { return authorize; } + + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public List getServices() { + return services; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java index ed688977b963..25d153c068aa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.InetAddress; import java.nio.ByteBuffer; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -95,10 +96,12 @@ public abstract class ServerCall implements RpcCa protected final User user; protected final InetAddress remoteAddress; + protected final X509Certificate[] clientCertificateChain; protected RpcCallback rpcCallback; private long responseCellSize = 0; private long responseBlockSize = 0; + private long fsReadTimeMillis = 0; // cumulative size of serialized exceptions private long exceptionSize = 0; private final boolean retryImmediatelySupported; @@ -134,9 +137,11 @@ public abstract class ServerCall implements RpcCa if (connection != null) { this.user = connection.user; this.retryImmediatelySupported = connection.retryImmediatelySupported; + this.clientCertificateChain = connection.clientCertificateChain; } else { this.user = null; this.retryImmediatelySupported = false; + this.clientCertificateChain = null; } this.remoteAddress = remoteAddress; this.timeout = timeout; @@ -498,6 +503,11 @@ public Optional getRequestUser() { return Optional.ofNullable(user); } + @Override + public Optional getClientCertificateChain() { + return Optional.ofNullable(clientCertificateChain); + } + @Override public InetAddress getRemoteAddress() { return remoteAddress; @@ -567,4 +577,14 @@ public int getRemotePort() { public synchronized BufferChain getResponse() { return response; } + + @Override + public void updateFsReadTime(long latencyMillis) { + fsReadTimeMillis += latencyMillis; + } + + @Override + public long getFsReadTime() { + return fsReadTimeMillis; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java index 695f1e7050c4..be97ad582c37 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java @@ -31,6 +31,7 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; +import java.security.cert.X509Certificate; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -40,6 +41,7 @@ import org.apache.commons.crypto.random.CryptoRandomFactory; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.client.ConnectionRegistryEndpoint; import org.apache.hadoop.hbase.client.VersionInfoUtil; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.io.ByteBufferOutputStream; @@ -56,6 +58,7 @@ import org.apache.hadoop.hbase.security.provider.SaslServerAuthenticationProviders; import org.apache.hadoop.hbase.security.provider.SimpleSaslServerAuthenticationProvider; import org.apache.hadoop.hbase.trace.TraceUtil; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.IntWritable; @@ -86,6 +89,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ResponseHeader; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.UserInformation; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetConnectionRegistryResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.TracingProtos.RPCTInfo; /** Reads calls from a connection and queues them for handling. */ @@ -133,6 +137,7 @@ abstract class ServerRpcConnection implements Closeable { protected User user = null; protected UserGroupInformation ugi = null; protected SaslServerAuthenticationProviders saslProviders = null; + protected X509Certificate[] clientCertificateChain = null; public ServerRpcConnection(RpcServer rpcServer) { this.rpcServer = rpcServer; @@ -686,10 +691,32 @@ private void doBadPreambleHandling(String msg) throws IOException { } private void doBadPreambleHandling(String msg, Exception e) throws IOException { - RpcServer.LOG.warn(msg); + RpcServer.LOG.warn(msg, e); doRespond(getErrorResponse(msg, e)); } + private boolean doConnectionRegistryResponse() throws IOException { + if (!(rpcServer.server instanceof ConnectionRegistryEndpoint)) { + // should be in tests or some scenarios where we should not reach here + return false; + } + // on backup masters, this request may be blocked since we need to fetch it from filesystem, + // but since it is just backup master, it is not a critical problem + String clusterId = ((ConnectionRegistryEndpoint) rpcServer.server).getClusterId(); + RpcServer.LOG.debug("Response connection registry, clusterId = '{}'", clusterId); + if (clusterId == null) { + // should be in tests or some scenarios where we should not reach here + return false; + } + GetConnectionRegistryResponse resp = + GetConnectionRegistryResponse.newBuilder().setClusterId(clusterId).build(); + ResponseHeader header = ResponseHeader.newBuilder().setCallId(-1).build(); + ByteBuffer buf = ServerCall.createHeaderAndMessageBytes(resp, header, 0, null); + BufferChain bufChain = new BufferChain(buf); + doRespond(() -> bufChain); + return true; + } + protected final void callCleanupIfNeeded() { if (callCleanup != null) { callCleanup.run(); @@ -697,30 +724,42 @@ protected final void callCleanupIfNeeded() { } } - protected final boolean processPreamble(ByteBuffer preambleBuffer) throws IOException { - assert preambleBuffer.remaining() == 6; - for (int i = 0; i < RPC_HEADER.length; i++) { - if (RPC_HEADER[i] != preambleBuffer.get()) { - doBadPreambleHandling( - "Expected HEADER=" + Bytes.toStringBinary(RPC_HEADER) + " but received HEADER=" - + Bytes.toStringBinary(preambleBuffer.array(), 0, RPC_HEADER.length) + " from " - + toString()); - return false; - } - } - int version = preambleBuffer.get() & 0xFF; - byte authbyte = preambleBuffer.get(); + protected enum PreambleResponse { + SUCCEED, // successfully processed the rpc preamble header + CONTINUE, // the preamble header is for other purpose, wait for the rpc preamble header + CLOSE // close the rpc connection + } + protected final PreambleResponse processPreamble(ByteBuffer preambleBuffer) throws IOException { + assert preambleBuffer.remaining() == 6; + if ( + ByteBufferUtils.equals(preambleBuffer, preambleBuffer.position(), 6, + RpcClient.REGISTRY_PREAMBLE_HEADER, 0, 6) && doConnectionRegistryResponse() + ) { + return PreambleResponse.CLOSE; + } + if (!ByteBufferUtils.equals(preambleBuffer, preambleBuffer.position(), 4, RPC_HEADER, 0, 4)) { + doBadPreambleHandling( + "Expected HEADER=" + Bytes.toStringBinary(RPC_HEADER) + " but received HEADER=" + + Bytes.toStringBinary( + ByteBufferUtils.toBytes(preambleBuffer, preambleBuffer.position(), RPC_HEADER.length), + 0, RPC_HEADER.length) + + " from " + toString()); + return PreambleResponse.CLOSE; + } + int version = preambleBuffer.get(preambleBuffer.position() + 4) & 0xFF; + byte authByte = preambleBuffer.get(preambleBuffer.position() + 5); if (version != RpcServer.CURRENT_VERSION) { - String msg = getFatalConnectionString(version, authbyte); + String msg = getFatalConnectionString(version, authByte); doBadPreambleHandling(msg, new WrongVersionException(msg)); - return false; + return PreambleResponse.CLOSE; } - this.provider = this.saslProviders.selectProvider(authbyte); + + this.provider = this.saslProviders.selectProvider(authByte); if (this.provider == null) { - String msg = getFatalConnectionString(version, authbyte); + String msg = getFatalConnectionString(version, authByte); doBadPreambleHandling(msg, new BadAuthException(msg)); - return false; + return PreambleResponse.CLOSE; } // TODO this is a wart while simple auth'n doesn't go through sasl. if (this.rpcServer.isSecurityEnabled && isSimpleAuthentication()) { @@ -730,7 +769,7 @@ protected final boolean processPreamble(ByteBuffer preambleBuffer) throws IOExce } else { AccessDeniedException ae = new AccessDeniedException("Authentication is required"); doRespond(getErrorResponse(ae.getMessage(), ae)); - return false; + return PreambleResponse.CLOSE; } } if (!this.rpcServer.isSecurityEnabled && !isSimpleAuthentication()) { @@ -743,7 +782,7 @@ protected final boolean processPreamble(ByteBuffer preambleBuffer) throws IOExce skipInitialSaslHandshake = true; } useSasl = !(provider instanceof SimpleSaslServerAuthenticationProvider); - return true; + return PreambleResponse.SUCCEED; } boolean isSimpleAuthentication() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleServerRpcConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleServerRpcConnection.java index ac705d7a26fa..9e90a7a31339 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleServerRpcConnection.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleServerRpcConnection.java @@ -137,12 +137,21 @@ private int readPreamble() throws IOException { return count; } preambleBuffer.flip(); - if (!processPreamble(preambleBuffer)) { - return -1; + PreambleResponse resp = processPreamble(preambleBuffer); + switch (resp) { + case SUCCEED: + preambleBuffer = null; // do not need it anymore + connectionPreambleRead = true; + return count; + case CONTINUE: + // wait for the next preamble header + preambleBuffer.reset(); + return count; + case CLOSE: + return -1; + default: + throw new IllegalArgumentException("Unknown preamble response: " + resp); } - preambleBuffer = null; // do not need it anymore - connectionPreambleRead = true; - return count; } private int read4Bytes() throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 0dca3a0111e3..88b82f01069e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -458,6 +458,7 @@ public class HMaster extends HBaseServerBase implements Maste private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier; private QuotaObserverChore quotaObserverChore; private SnapshotQuotaObserverChore snapshotQuotaChore; + private OldWALsDirSizeChore oldWALsDirSizeChore; private ProcedureExecutor procedureExecutor; private ProcedureStore procedureStore; @@ -723,6 +724,11 @@ public MasterRpcServices getMasterRpcServices() { return rpcServices; } + @Override + protected MasterCoprocessorHost getCoprocessorHost() { + return getMasterCoprocessorHost(); + } + public boolean balanceSwitch(final boolean b) throws IOException { return getMasterRpcServices().switchBalancer(b, BalanceSwitchMode.ASYNC); } @@ -1362,6 +1368,10 @@ private void finishActiveMasterInitialization() throws IOException, InterruptedE this.rollingUpgradeChore = new RollingUpgradeChore(this); getChoreService().scheduleChore(rollingUpgradeChore); + + this.oldWALsDirSizeChore = new OldWALsDirSizeChore(this); + getChoreService().scheduleChore(this.oldWALsDirSizeChore); + status.markComplete("Progress after master initialized complete"); } @@ -1894,6 +1904,7 @@ protected void stopChores() { shutdownChore(hbckChore); shutdownChore(regionsRecoveryChore); shutdownChore(rollingUpgradeChore); + shutdownChore(oldWALsDirSizeChore); } /** Returns Get remote side's InetAddress */ @@ -3336,6 +3347,18 @@ public void setInitialized(boolean isInitialized) { procedureExecutor.getEnvironment().setEventReady(initialized, isInitialized); } + /** + * Mainly used in procedure related tests, where we will restart ProcedureExecutor and + * AssignmentManager, but we do not want to restart master(to speed up the test), so we need to + * disable rpc for a while otherwise some critical rpc requests such as + * reportRegionStateTransition could fail and cause region server to abort. + */ + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public void setServiceStarted(boolean started) { + this.serviceStarted = started; + } + @Override public ProcedureEvent getInitializedEvent() { return initialized; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 3af69b362609..e3d269973f8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -2114,4 +2114,22 @@ public void call(MasterObserver observer) throws IOException { } }); } + + public void preUpdateConfiguration(Configuration preReloadConf) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.preUpdateMasterConfiguration(this, preReloadConf); + } + }); + } + + public void postUpdateConfiguration(Configuration postReloadConf) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.postUpdateMasterConfiguration(this, postReloadConf); + } + }); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 6d330d6eb791..1da8e03d179e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -2714,6 +2714,7 @@ public MasterProtos.AssignsResponse assigns(RpcController controller, MasterProtos.AssignsResponse.Builder responseBuilder = MasterProtos.AssignsResponse.newBuilder(); final boolean override = request.getOverride(); + final boolean force = request.getForce(); LOG.info("{} assigns, override={}", server.getClientIdAuditPrefix(), override); for (HBaseProtos.RegionSpecifier rs : request.getRegionList()) { final RegionInfo info = getRegionInfo(rs); @@ -2721,7 +2722,7 @@ public MasterProtos.AssignsResponse assigns(RpcController controller, LOG.info("Unknown region {}", rs); continue; } - responseBuilder.addPid(Optional.ofNullable(am.createOneAssignProcedure(info, override)) + responseBuilder.addPid(Optional.ofNullable(am.createOneAssignProcedure(info, override, force)) .map(pe::submitProcedure).orElse(Procedure.NO_PROC_ID)); } return responseBuilder.build(); @@ -2741,6 +2742,7 @@ public MasterProtos.UnassignsResponse unassigns(RpcController controller, MasterProtos.UnassignsResponse.Builder responseBuilder = MasterProtos.UnassignsResponse.newBuilder(); final boolean override = request.getOverride(); + final boolean force = request.getForce(); LOG.info("{} unassigns, override={}", server.getClientIdAuditPrefix(), override); for (HBaseProtos.RegionSpecifier rs : request.getRegionList()) { final RegionInfo info = getRegionInfo(rs); @@ -2748,8 +2750,9 @@ public MasterProtos.UnassignsResponse unassigns(RpcController controller, LOG.info("Unknown region {}", rs); continue; } - responseBuilder.addPid(Optional.ofNullable(am.createOneUnassignProcedure(info, override)) - .map(pe::submitProcedure).orElse(Procedure.NO_PROC_ID)); + responseBuilder + .addPid(Optional.ofNullable(am.createOneUnassignProcedure(info, override, force)) + .map(pe::submitProcedure).orElse(Procedure.NO_PROC_ID)); } return responseBuilder.build(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java index aca04a8ac83a..a2c929cb79d4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java @@ -89,6 +89,9 @@ public boolean accept(Path p) { // create the split log lock private final Lock splitLogLock = new ReentrantLock(); + // old WALs directory size in bytes + private long oldWALsDirSize; + /** * Superceded by {@link SplitWALManager}; i.e. procedure-based WAL splitting rather than 'classic' * zk-coordinated WAL splitting. @@ -114,6 +117,7 @@ public MasterWalManager(Configuration conf, FileSystem fs, Path rootDir, MasterS this.services = services; this.splitLogManager = new SplitLogManager(services, conf); this.oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + this.oldWALsDirSize = 0; } public void stop() { @@ -134,6 +138,14 @@ Path getOldLogDir() { return this.oldLogDir; } + public void updateOldWALsDirSize() throws IOException { + this.oldWALsDirSize = fs.getContentSummary(this.oldLogDir).getLength(); + } + + public long getOldWALsDirSize() { + return this.oldWALsDirSize; + } + public FileSystem getFileSystem() { return this.fs; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 923c663807f1..ff6f5b8e5df8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -238,4 +238,12 @@ public PairOfSameType getRegionCounts() { return new PairOfSameType<>(0, 0); } } + + @Override + public long getOldWALsDirSize() { + if (master == null || !master.isInitialized()) { + return 0; + } + return master.getMasterWalManager().getOldWALsDirSize(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/OldWALsDirSizeChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/OldWALsDirSizeChore.java new file mode 100644 index 000000000000..b2f0622b7d28 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/OldWALsDirSizeChore.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This chore is used to update the 'oldWALsDirSize' variable in {@link MasterWalManager} through + * the {@link MasterWalManager#updateOldWALsDirSize()} method. + */ +@InterfaceAudience.Private +public class OldWALsDirSizeChore extends ScheduledChore { + private static final Logger LOG = LoggerFactory.getLogger(OldWALsDirSizeChore.class); + + private final MasterServices master; + + public OldWALsDirSizeChore(MasterServices master) { + super(master.getServerName() + "-OldWALsDirSizeChore", master, + master.getConfiguration().getInt(HConstants.HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD, + HConstants.DEFAULT_HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD)); + this.master = master; + } + + @Override + protected void chore() { + try { + this.master.getMasterWalManager().updateOldWALsDirSize(); + } catch (IOException e) { + LOG.error("Got exception while trying to update the old WALs Directory size counter: " + + e.getMessage(), e); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java index 71ca500a045e..a09f6689a5c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java @@ -135,7 +135,7 @@ public void upgrade(Set deadServersFromPE, Set liveServe .forEach(s -> LOG.error("{} has no matching ServerCrashProcedure", s)); // create ServerNode for all possible live servers from wal directory and master local region liveServersBeforeRestart - .forEach(sn -> server.getAssignmentManager().getRegionStates().getOrCreateServer(sn)); + .forEach(sn -> server.getAssignmentManager().getRegionStates().createServer(sn)); ServerManager serverManager = server.getServerManager(); synchronized (this) { Set liveServers = regionServers; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 6a169beb53bf..2afd48c58df5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -426,6 +426,7 @@ public ServerName findServerWithSameHostnamePortWithLock(final ServerName server void recordNewServerWithLock(final ServerName serverName, final ServerMetrics sl) { LOG.info("Registering regionserver=" + serverName); this.onlineServers.put(serverName, sl); + master.getAssignmentManager().getRegionStates().createServer(serverName); } public ConcurrentNavigableMap getFlushedSequenceIdByRegion() { @@ -603,6 +604,10 @@ synchronized long expireServer(final ServerName serverName, boolean force) { } LOG.info("Processing expiration of " + serverName + " on " + this.master.getServerName()); long pid = master.getAssignmentManager().submitServerCrash(serverName, true, force); + if (pid == Procedure.NO_PROC_ID) { + // skip later processing as we failed to submit SCP + return Procedure.NO_PROC_ID; + } storage.expired(serverName); // Tell our listeners that a server was removed if (!this.listeners.isEmpty()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 804757959d5c..9cee9f87ce2f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -82,6 +83,7 @@ import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; @@ -331,6 +333,12 @@ public void start() throws IOException, KeeperException { regionNode.getProcedure().stateLoaded(this, regionNode); } if (regionLocation != null) { + // TODO: this could lead to some orphan server state nodes, as it is possible that the + // region server is already dead and its SCP has already finished but we have + // persisted an opening state on this region server. Finally the TRSP will assign the + // region to another region server, so it will not cause critical problems, just waste + // some memory as no one will try to cleanup these orphan server state nodes. + regionStates.createServer(regionLocation); regionStates.addRegionToServer(regionNode); } if (RegionReplicaUtil.isDefaultReplica(regionInfo.getReplicaId())) { @@ -757,11 +765,14 @@ private void preTransitCheck(RegionStateNode regionNode, RegionState.State[] exp * @param override If false, check RegionState is appropriate for assign; if not throw exception. */ private TransitRegionStateProcedure createAssignProcedure(RegionInfo regionInfo, ServerName sn, - boolean override) throws IOException { + boolean override, boolean force) throws IOException { RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(regionInfo); regionNode.lock(); try { if (override) { + if (!force) { + preTransitCheck(regionNode, STATES_EXPECTED_ON_ASSIGN); + } if (regionNode.getProcedure() != null) { regionNode.unsetProcedure(regionNode.getProcedure()); } @@ -779,7 +790,7 @@ private TransitRegionStateProcedure createAssignProcedure(RegionInfo regionInfo, /** * Create an assign TransitRegionStateProcedure. Does NO checking of RegionState. Presumes * appriopriate state ripe for assign. - * @see #createAssignProcedure(RegionInfo, ServerName, boolean) + * @see #createAssignProcedure(RegionInfo, ServerName, boolean, boolean) */ private TransitRegionStateProcedure createAssignProcedure(RegionStateNode regionNode, ServerName targetServer) { @@ -793,7 +804,7 @@ private TransitRegionStateProcedure createAssignProcedure(RegionStateNode region } public long assign(RegionInfo regionInfo, ServerName sn) throws IOException { - TransitRegionStateProcedure proc = createAssignProcedure(regionInfo, sn, false); + TransitRegionStateProcedure proc = createAssignProcedure(regionInfo, sn, false, false); ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); return proc.getProcId(); } @@ -809,7 +820,7 @@ public long assign(RegionInfo regionInfo) throws IOException { */ public Future assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException { return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), - createAssignProcedure(regionInfo, sn, false)); + createAssignProcedure(regionInfo, sn, false, false)); } /** @@ -953,10 +964,11 @@ static int compare(TransitRegionStateProcedure left, TransitRegionStateProcedure * method is called from HBCK2. * @return an assign or null */ - public TransitRegionStateProcedure createOneAssignProcedure(RegionInfo ri, boolean override) { + public TransitRegionStateProcedure createOneAssignProcedure(RegionInfo ri, boolean override, + boolean force) { TransitRegionStateProcedure trsp = null; try { - trsp = createAssignProcedure(ri, null, override); + trsp = createAssignProcedure(ri, null, override, force); } catch (IOException ioe) { LOG.info( "Failed {} assign, override={}" @@ -970,12 +982,16 @@ public TransitRegionStateProcedure createOneAssignProcedure(RegionInfo ri, boole * Create one TransitRegionStateProcedure to unassign a region. This method is called from HBCK2. * @return an unassign or null */ - public TransitRegionStateProcedure createOneUnassignProcedure(RegionInfo ri, boolean override) { + public TransitRegionStateProcedure createOneUnassignProcedure(RegionInfo ri, boolean override, + boolean force) { RegionStateNode regionNode = regionStates.getOrCreateRegionStateNode(ri); TransitRegionStateProcedure trsp = null; regionNode.lock(); try { if (override) { + if (!force) { + preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE); + } if (regionNode.getProcedure() != null) { regionNode.unsetProcedure(regionNode.getProcedure()); } @@ -1108,7 +1124,7 @@ public void deleteTable(final TableName tableName) throws IOException { // RS Region Transition Report helpers // ============================================================================================ private void reportRegionStateTransition(ReportRegionStateTransitionResponse.Builder builder, - ServerName serverName, List transitionList) throws IOException { + ServerStateNode serverNode, List transitionList) throws IOException { for (RegionStateTransition transition : transitionList) { switch (transition.getTransitionCode()) { case OPENED: @@ -1118,7 +1134,7 @@ private void reportRegionStateTransition(ReportRegionStateTransitionResponse.Bui final RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); long procId = transition.getProcIdCount() > 0 ? transition.getProcId(0) : Procedure.NO_PROC_ID; - updateRegionTransition(serverName, transition.getTransitionCode(), hri, + updateRegionTransition(serverNode, transition.getTransitionCode(), hri, transition.hasOpenSeqNum() ? transition.getOpenSeqNum() : HConstants.NO_SEQNUM, procId); break; case READY_TO_SPLIT: @@ -1128,7 +1144,7 @@ private void reportRegionStateTransition(ReportRegionStateTransitionResponse.Bui final RegionInfo parent = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); final RegionInfo splitA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); final RegionInfo splitB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); - updateRegionSplitTransition(serverName, transition.getTransitionCode(), parent, splitA, + updateRegionSplitTransition(serverNode, transition.getTransitionCode(), parent, splitA, splitB); break; case READY_TO_MERGE: @@ -1138,7 +1154,7 @@ private void reportRegionStateTransition(ReportRegionStateTransitionResponse.Bui final RegionInfo merged = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); final RegionInfo mergeA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); final RegionInfo mergeB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); - updateRegionMergeTransition(serverName, transition.getTransitionCode(), merged, mergeA, + updateRegionMergeTransition(serverNode, transition.getTransitionCode(), merged, mergeA, mergeB); break; } @@ -1150,7 +1166,12 @@ public ReportRegionStateTransitionResponse reportRegionStateTransition( ReportRegionStateTransitionResponse.Builder builder = ReportRegionStateTransitionResponse.newBuilder(); ServerName serverName = ProtobufUtil.toServerName(req.getServer()); - ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); + ServerStateNode serverNode = regionStates.getServerNode(serverName); + if (serverNode == null) { + LOG.warn("No server node for {}", serverName); + builder.setErrorMessage("No server node for " + serverName); + return builder.build(); + } // here we have to acquire a read lock instead of a simple exclusive lock. This is because that // we should not block other reportRegionStateTransition call from the same region server. This // is not only about performance, but also to prevent dead lock. Think of the meta region is @@ -1163,7 +1184,7 @@ public ReportRegionStateTransitionResponse reportRegionStateTransition( // above in submitServerCrash method and HBASE-21508 for more details. if (serverNode.isInState(ServerState.ONLINE)) { try { - reportRegionStateTransition(builder, serverName, req.getTransitionList()); + reportRegionStateTransition(builder, serverNode, req.getTransitionList()); } catch (PleaseHoldException e) { LOG.trace("Failed transition ", e); throw e; @@ -1185,7 +1206,7 @@ public ReportRegionStateTransitionResponse reportRegionStateTransition( return builder.build(); } - private void updateRegionTransition(ServerName serverName, TransitionCode state, + private void updateRegionTransition(ServerStateNode serverNode, TransitionCode state, RegionInfo regionInfo, long seqId, long procId) throws IOException { checkMetaLoaded(regionInfo); @@ -1193,13 +1214,12 @@ private void updateRegionTransition(ServerName serverName, TransitionCode state, if (regionNode == null) { // the table/region is gone. maybe a delete, split, merge throw new UnexpectedStateException(String.format( - "Server %s was trying to transition region %s to %s. but Region is not known.", serverName, - regionInfo, state)); + "Server %s was trying to transition region %s to %s. but Region is not known.", + serverNode.getServerName(), regionInfo, state)); } - LOG.trace("Update region transition serverName={} region={} regionState={}", serverName, - regionNode, state); + LOG.trace("Update region transition serverName={} region={} regionState={}", + serverNode.getServerName(), regionNode, state); - ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); regionNode.lock(); try { if (!reportTransition(regionNode, serverNode, state, seqId, procId)) { @@ -1216,8 +1236,8 @@ private void updateRegionTransition(ServerName serverName, TransitionCode state, ) { LOG.info("RegionServer {} {}", state, regionNode.getRegionInfo().getEncodedName()); } else { - LOG.warn("No matching procedure found for {} transition on {} to {}", serverName, - regionNode, state); + LOG.warn("No matching procedure found for {} transition on {} to {}", + serverNode.getServerName(), regionNode, state); } } } finally { @@ -1237,8 +1257,9 @@ private boolean reportTransition(RegionStateNode regionNode, ServerStateNode ser return true; } - private void updateRegionSplitTransition(final ServerName serverName, final TransitionCode state, - final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB) throws IOException { + private void updateRegionSplitTransition(final ServerStateNode serverNode, + final TransitionCode state, final RegionInfo parent, final RegionInfo hriA, + final RegionInfo hriB) throws IOException { checkMetaLoaded(parent); if (state != TransitionCode.READY_TO_SPLIT) { @@ -1262,8 +1283,8 @@ private void updateRegionSplitTransition(final ServerName serverName, final Tran // Submit the Split procedure final byte[] splitKey = hriB.getStartKey(); if (LOG.isDebugEnabled()) { - LOG.debug("Split request from " + serverName + ", parent=" + parent + " splitKey=" - + Bytes.toStringBinary(splitKey)); + LOG.debug("Split request from {}, parent={}, splitKey={}", serverNode.getServerName(), parent, + Bytes.toStringBinary(splitKey)); } // Processing this report happens asynchronously from other activities which can mutate // the region state. For example, a split procedure may already be running for this parent. @@ -1277,21 +1298,22 @@ private void updateRegionSplitTransition(final ServerName serverName, final Tran if (parentState != null && parentState.isOpened()) { master.getMasterProcedureExecutor().submitProcedure(createSplitProcedure(parent, splitKey)); } else { - LOG.info("Ignoring split request from " + serverName + ", parent=" + parent - + " because parent is unknown or not open"); + LOG.info("Ignoring split request from {}, parent={} because parent is unknown or not open", + serverNode.getServerName(), parent); return; } // If the RS is < 2.0 throw an exception to abort the operation, we are handling the split - if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) { + if (master.getServerManager().getVersionNumber(serverNode.getServerName()) < 0x0200000) { throw new UnsupportedOperationException( String.format("Split handled by the master: " + "parent=%s hriA=%s hriB=%s", parent.getShortNameToLog(), hriA, hriB)); } } - private void updateRegionMergeTransition(final ServerName serverName, final TransitionCode state, - final RegionInfo merged, final RegionInfo hriA, final RegionInfo hriB) throws IOException { + private void updateRegionMergeTransition(final ServerStateNode serverNode, + final TransitionCode state, final RegionInfo merged, final RegionInfo hriA, + final RegionInfo hriB) throws IOException { checkMetaLoaded(merged); if (state != TransitionCode.READY_TO_MERGE) { @@ -1313,7 +1335,7 @@ private void updateRegionMergeTransition(final ServerName serverName, final Tran master.getMasterProcedureExecutor().submitProcedure(createMergeProcedure(hriA, hriB)); // If the RS is < 2.0 throw an exception to abort the operation, we are handling the merge - if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) { + if (master.getServerManager().getVersionNumber(serverNode.getServerName()) < 0x0200000) { throw new UnsupportedOperationException( String.format("Merge not handled yet: regionState=%s merged=%s hriA=%s hriB=%s", state, merged, hriA, hriB)); @@ -1343,12 +1365,19 @@ public void reportOnlineRegions(ServerName serverName, Set regionNames) regionNames.stream().map(Bytes::toStringBinary).collect(Collectors.toList())); } - ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); - synchronized (serverNode) { + ServerStateNode serverNode = regionStates.getServerNode(serverName); + if (serverNode == null) { + LOG.warn("Got a report from server {} where its server node is null", serverName); + return; + } + serverNode.readLock().lock(); + try { if (!serverNode.isInState(ServerState.ONLINE)) { - LOG.warn("Got a report from a server result in state " + serverNode.getState()); + LOG.warn("Got a report from a server result in state {}", serverNode); return; } + } finally { + serverNode.readLock().unlock(); } // Track the regionserver reported online regions in memory. @@ -1754,6 +1783,12 @@ public void visitRegionState(Result result, final RegionInfo regionInfo, final S localState.matches(State.OPEN, State.OPENING, State.CLOSING, State.SPLITTING, State.MERGING) ) { assert regionLocation != null : "found null region location for " + regionNode; + // TODO: this could lead to some orphan server state nodes, as it is possible that the + // region server is already dead and its SCP has already finished but we have + // persisted an opening state on this region server. Finally the TRSP will assign the + // region to another region server, so it will not cause critical problems, just waste + // some memory as no one will try to cleanup these orphan server state nodes. + regionStates.createServer(regionLocation); regionStates.addRegionToServer(regionNode); } else if (localState == State.OFFLINE || regionInfo.isOffline()) { regionStates.addToOfflineRegions(regionNode); @@ -1839,7 +1874,17 @@ public long submitServerCrash(ServerName serverName, boolean shouldSplitWal, boo synchronized (rsReports) { rsReports.remove(serverName); } + if (serverNode == null) { + if (force) { + LOG.info("Force adding ServerCrashProcedure for {} when server node is null", serverName); + } else { + // for normal case, do not schedule SCP if ServerStateNode is null + LOG.warn("Skip adding ServerCrashProcedure for {} because server node is null", serverName); + return Procedure.NO_PROC_ID; + } + } + ProcedureExecutor procExec = this.master.getMasterProcedureExecutor(); // We hold the write lock here for fencing on reportRegionStateTransition. Once we set the // server state to CRASHED, we will no longer accept the reportRegionStateTransition call from // this server. This is used to simplify the implementation for TRSP and SCP, where we can make @@ -1847,44 +1892,42 @@ public long submitServerCrash(ServerName serverName, boolean shouldSplitWal, boo if (serverNode != null) { serverNode.writeLock().lock(); } - boolean carryingMeta; - long pid; try { - ProcedureExecutor procExec = this.master.getMasterProcedureExecutor(); - carryingMeta = isCarryingMeta(serverName); - if (!force && serverNode != null && !serverNode.isInState(ServerState.ONLINE)) { - LOG.info("Skip adding ServerCrashProcedure for {} (meta={}) -- running?", serverNode, - carryingMeta); - return Procedure.NO_PROC_ID; - } else { - MasterProcedureEnv mpe = procExec.getEnvironment(); - // If serverNode == null, then 'Unknown Server'. Schedule HBCKSCP instead. - // HBCKSCP scours Master in-memory state AND hbase;meta for references to - // serverName just-in-case. An SCP that is scheduled when the server is - // 'Unknown' probably originated externally with HBCK2 fix-it tool. - ServerState oldState = null; - if (serverNode != null) { - oldState = serverNode.getState(); - serverNode.setState(ServerState.CRASHED); - } + boolean carryingMeta = isCarryingMeta(serverName); + if (serverNode != null && !serverNode.isInState(ServerState.ONLINE)) { if (force) { - pid = procExec.submitProcedure( - new HBCKServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta)); + LOG.info("Force adding ServerCrashProcedure for {} (meta={}) when state is not {}", + serverNode, carryingMeta, ServerState.ONLINE); } else { - pid = procExec.submitProcedure( - new ServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta)); + LOG.info("Skip adding ServerCrashProcedure for {} (meta={}) when state is not {}", + serverNode, carryingMeta, ServerState.ONLINE); + return Procedure.NO_PROC_ID; } - LOG.info("Scheduled ServerCrashProcedure pid={} for {} (carryingMeta={}){}.", pid, - serverName, carryingMeta, - serverNode == null ? "" : " " + serverNode.toString() + ", oldState=" + oldState); } + MasterProcedureEnv mpe = procExec.getEnvironment(); + // If serverNode == null, then 'Unknown Server'. Schedule HBCKSCP instead. + // HBCKSCP scours Master in-memory state AND hbase;meta for references to + // serverName just-in-case. An SCP that is scheduled when the server is + // 'Unknown' probably originated externally with HBCK2 fix-it tool. + ServerState oldState = null; + if (serverNode != null) { + oldState = serverNode.getState(); + serverNode.setState(ServerState.CRASHED); + } + ServerCrashProcedure scp = force + ? new HBCKServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta) + : new ServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta); + long pid = procExec.submitProcedure(scp); + LOG.info("Scheduled ServerCrashProcedure pid={} for {} (carryingMeta={}){}.", pid, serverName, + carryingMeta, + serverNode == null ? "" : " " + serverNode.toString() + ", oldState=" + oldState); + return pid; } finally { if (serverNode != null) { serverNode.writeLock().unlock(); } } - return pid; } public void offlineRegion(final RegionInfo regionInfo) { @@ -1989,71 +2032,88 @@ public RegionInfo getRegionInfo(final String encodedRegionName) { // Should only be called in TransitRegionStateProcedure(and related procedures), as the locking // and pre-assumptions are very tricky. // ============================================================================================ - private void transitStateAndUpdate(RegionStateNode regionNode, RegionState.State newState, - RegionState.State... expectedStates) throws IOException { + private CompletableFuture transitStateAndUpdate(RegionStateNode regionNode, + RegionState.State newState, RegionState.State... expectedStates) { RegionState.State state = regionNode.getState(); - regionNode.transitionState(newState, expectedStates); - boolean succ = false; try { - regionStateStore.updateRegionLocation(regionNode); - succ = true; - } finally { - if (!succ) { + regionNode.transitionState(newState, expectedStates); + } catch (UnexpectedStateException e) { + return FutureUtils.failedFuture(e); + } + CompletableFuture future = regionStateStore.updateRegionLocation(regionNode); + FutureUtils.addListener(future, (r, e) -> { + if (e != null) { // revert regionNode.setState(state); } - } + }); + return future; } // should be called within the synchronized block of RegionStateNode - void regionOpening(RegionStateNode regionNode) throws IOException { + CompletableFuture regionOpening(RegionStateNode regionNode) { // As in SCP, for performance reason, there is no TRSP attached with this region, we will not // update the region state, which means that the region could be in any state when we want to // assign it after a RS crash. So here we do not pass the expectedStates parameter. - transitStateAndUpdate(regionNode, State.OPENING); - regionStates.addRegionToServer(regionNode); - // update the operation count metrics - metrics.incrementOperationCounter(); + return transitStateAndUpdate(regionNode, State.OPENING).thenAccept(r -> { + ServerStateNode serverNode = regionStates.getServerNode(regionNode.getRegionLocation()); + // Here the server node could be null. For example, we want to assign the region to a given + // region server and it crashes, and it is the region server which holds hbase:meta, then the + // above transitStateAndUpdate call will never succeed until we finishes the SCP for it. But + // after the SCP finishes, the server node will be removed, so when we arrive there, the + // server + // node will be null. This is not a big problem if we skip adding it, as later we will fail to + // execute the remote procedure on the region server and then try to assign to another region + // server + if (serverNode != null) { + serverNode.addRegion(regionNode); + } + // update the operation count metrics + metrics.incrementOperationCounter(); + }); } // should be called under the RegionStateNode lock // The parameter 'giveUp' means whether we will try to open the region again, if it is true, then // we will persist the FAILED_OPEN state into hbase:meta. - void regionFailedOpen(RegionStateNode regionNode, boolean giveUp) throws IOException { + CompletableFuture regionFailedOpen(RegionStateNode regionNode, boolean giveUp) { RegionState.State state = regionNode.getState(); ServerName regionLocation = regionNode.getRegionLocation(); - if (giveUp) { - regionNode.setState(State.FAILED_OPEN); - regionNode.setRegionLocation(null); - boolean succ = false; - try { - regionStateStore.updateRegionLocation(regionNode); - succ = true; - } finally { - if (!succ) { - // revert - regionNode.setState(state); - regionNode.setRegionLocation(regionLocation); - } + if (!giveUp) { + if (regionLocation != null) { + regionStates.removeRegionFromServer(regionLocation, regionNode); } + return CompletableFuture.completedFuture(null); } - if (regionLocation != null) { - regionStates.removeRegionFromServer(regionLocation, regionNode); - } + regionNode.setState(State.FAILED_OPEN); + regionNode.setRegionLocation(null); + CompletableFuture future = regionStateStore.updateRegionLocation(regionNode); + FutureUtils.addListener(future, (r, e) -> { + if (e == null) { + if (regionLocation != null) { + regionStates.removeRegionFromServer(regionLocation, regionNode); + } + } else { + // revert + regionNode.setState(state); + regionNode.setRegionLocation(regionLocation); + } + }); + return future; } // should be called under the RegionStateNode lock - void regionClosing(RegionStateNode regionNode) throws IOException { - transitStateAndUpdate(regionNode, State.CLOSING, STATES_EXPECTED_ON_CLOSING); - - RegionInfo hri = regionNode.getRegionInfo(); - // Set meta has not initialized early. so people trying to create/edit tables will wait - if (isMetaRegion(hri)) { - setMetaAssigned(hri, false); - } - regionStates.addRegionToServer(regionNode); - // update the operation count metrics - metrics.incrementOperationCounter(); + CompletableFuture regionClosing(RegionStateNode regionNode) { + return transitStateAndUpdate(regionNode, State.CLOSING, STATES_EXPECTED_ON_CLOSING) + .thenAccept(r -> { + RegionInfo hri = regionNode.getRegionInfo(); + // Set meta has not initialized early. so people trying to create/edit tables will wait + if (isMetaRegion(hri)) { + setMetaAssigned(hri, false); + } + // update the operation count metrics + metrics.incrementOperationCounter(); + }); } // for open and close, they will first be persist to the procedure store in @@ -2062,7 +2122,8 @@ void regionClosing(RegionStateNode regionNode) throws IOException { // RegionRemoteProcedureBase is woken up, we will persist the RegionStateNode to hbase:meta. // should be called under the RegionStateNode lock - void regionOpenedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOException { + void regionOpenedWithoutPersistingToMeta(RegionStateNode regionNode) + throws UnexpectedStateException { regionNode.transitionState(State.OPEN, STATES_EXPECTED_ON_OPEN); RegionInfo regionInfo = regionNode.getRegionInfo(); regionStates.addRegionToServer(regionNode); @@ -2070,7 +2131,8 @@ void regionOpenedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOEx } // should be called under the RegionStateNode lock - void regionClosedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOException { + void regionClosedWithoutPersistingToMeta(RegionStateNode regionNode) + throws UnexpectedStateException { ServerName regionLocation = regionNode.getRegionLocation(); regionNode.transitionState(State.CLOSED, STATES_EXPECTED_ON_CLOSED); regionNode.setRegionLocation(null); @@ -2080,40 +2142,41 @@ void regionClosedWithoutPersistingToMeta(RegionStateNode regionNode) throws IOEx } } + // should be called under the RegionStateNode lock + CompletableFuture persistToMeta(RegionStateNode regionNode) { + return regionStateStore.updateRegionLocation(regionNode).thenAccept(r -> { + RegionInfo regionInfo = regionNode.getRegionInfo(); + if (isMetaRegion(regionInfo) && regionNode.getState() == State.OPEN) { + // Usually we'd set a table ENABLED at this stage but hbase:meta is ALWAYs enabled, it + // can't be disabled -- so skip the RPC (besides... enabled is managed by TableStateManager + // which is backed by hbase:meta... Avoid setting ENABLED to avoid having to update state + // on table that contains state. + setMetaAssigned(regionInfo, true); + } + }); + } + // should be called under the RegionStateNode lock // for SCP - public void regionClosedAbnormally(RegionStateNode regionNode) throws IOException { + public CompletableFuture regionClosedAbnormally(RegionStateNode regionNode) { RegionState.State state = regionNode.getState(); ServerName regionLocation = regionNode.getRegionLocation(); - regionNode.transitionState(State.ABNORMALLY_CLOSED); + regionNode.setState(State.ABNORMALLY_CLOSED); regionNode.setRegionLocation(null); - boolean succ = false; - try { - regionStateStore.updateRegionLocation(regionNode); - succ = true; - } finally { - if (!succ) { + CompletableFuture future = regionStateStore.updateRegionLocation(regionNode); + FutureUtils.addListener(future, (r, e) -> { + if (e == null) { + if (regionLocation != null) { + regionNode.setLastHost(regionLocation); + regionStates.removeRegionFromServer(regionLocation, regionNode); + } + } else { // revert regionNode.setState(state); regionNode.setRegionLocation(regionLocation); } - } - if (regionLocation != null) { - regionNode.setLastHost(regionLocation); - regionStates.removeRegionFromServer(regionLocation, regionNode); - } - } - - void persistToMeta(RegionStateNode regionNode) throws IOException { - regionStateStore.updateRegionLocation(regionNode); - RegionInfo regionInfo = regionNode.getRegionInfo(); - if (isMetaRegion(regionInfo) && regionNode.getState() == State.OPEN) { - // Usually we'd set a table ENABLED at this stage but hbase:meta is ALWAYs enabled, it - // can't be disabled -- so skip the RPC (besides... enabled is managed by TableStateManager - // which is backed by hbase:meta... Avoid setting ENABLED to avoid having to update state - // on table that contains state. - setMetaAssigned(regionInfo, true); - } + }); + return future; } // ============================================================================================ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index c0b47b0bc246..7d4ec71d35b1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -442,7 +442,7 @@ protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) { private boolean prepareMergeRegion(final MasterProcedureEnv env) throws IOException { // Fail if we are taking snapshot for the given table TableName tn = regionsToMerge[0].getTable(); - if (env.getMasterServices().getSnapshotManager().isTakingSnapshot(tn)) { + if (env.getMasterServices().getSnapshotManager().isTableTakingAnySnapshot(tn)) { throw new MergeRegionException("Skip merging regions " + RegionInfo.getShortNameToLog(regionsToMerge) + ", because we are snapshotting " + tn); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java index 6b6da9e33965..41466fce9549 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -29,6 +30,7 @@ import org.apache.hadoop.hbase.procedure2.FailedRemoteDispatchException; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureEvent; +import org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureUtil; @@ -45,6 +47,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseState; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; /** @@ -73,6 +76,8 @@ public abstract class RegionRemoteProcedureBase extends Procedure future; + protected RegionRemoteProcedureBase() { } @@ -179,7 +184,20 @@ protected abstract void updateTransitionWithoutPersistingToMeta(MasterProcedureE // A bit strange but the procedure store will throw RuntimeException if we can not persist the // state, so upper layer should take care of this... private void persistAndWake(MasterProcedureEnv env, RegionStateNode regionNode) { - env.getMasterServices().getMasterProcedureExecutor().getStore().update(this); + // The synchronization here is to guard with ProcedureExecutor.executeRollback, as here we will + // not hold the procedure execution lock, but we should not persist a procedure in ROLLEDBACK + // state to the procedure store. + // The ProcedureStore.update must be inside the lock, so here the check for procedure state and + // update could be atomic. In ProcedureExecutor.cleanupAfterRollbackOneStep, we will set the + // state to ROLLEDBACK, which will hold the same lock too as the Procedure.setState method is + // synchronized. This is the key to keep us safe. + synchronized (this) { + if (getState() == ProcedureState.ROLLEDBACK) { + LOG.warn("Procedure {} has already been rolled back, skip persistent", this); + return; + } + env.getMasterServices().getMasterProcedureExecutor().getStore().update(this); + } regionNode.getProcedureEvent().wake(env.getProcedureScheduler()); } @@ -268,17 +286,26 @@ private void unattach(MasterProcedureEnv env) { getParent(env).unattachRemoteProc(this); } + private CompletableFuture getFuture() { + return future; + } + + private void setFuture(CompletableFuture f) { + future = f; + } + @Override protected Procedure[] execute(MasterProcedureEnv env) throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException { RegionStateNode regionNode = getRegionNode(env); - regionNode.lock(); + if (!regionNode.isLockedBy(this)) { + regionNode.lock(this, () -> ProcedureFutureUtil.wakeUp(this, env)); + } try { switch (state) { case REGION_REMOTE_PROCEDURE_DISPATCH: { // The code which wakes us up also needs to lock the RSN so here we do not need to - // synchronize - // on the event. + // synchronize on the event. ProcedureEvent event = regionNode.getProcedureEvent(); try { env.getRemoteDispatcher().addOperationToNode(targetServer, this); @@ -294,16 +321,29 @@ protected Procedure[] execute(MasterProcedureEnv env) throw new ProcedureSuspendedException(); } case REGION_REMOTE_PROCEDURE_REPORT_SUCCEED: - env.getAssignmentManager().persistToMeta(regionNode); - unattach(env); + if ( + ProcedureFutureUtil.checkFuture(this, this::getFuture, this::setFuture, + () -> unattach(env)) + ) { + return null; + } + ProcedureFutureUtil.suspendIfNecessary(this, this::setFuture, + env.getAssignmentManager().persistToMeta(regionNode), env, () -> unattach(env)); return null; case REGION_REMOTE_PROCEDURE_DISPATCH_FAIL: // the remote call is failed so we do not need to change the region state, just return. unattach(env); return null; case REGION_REMOTE_PROCEDURE_SERVER_CRASH: - env.getAssignmentManager().regionClosedAbnormally(regionNode); - unattach(env); + if ( + ProcedureFutureUtil.checkFuture(this, this::getFuture, this::setFuture, + () -> unattach(env)) + ) { + return null; + } + ProcedureFutureUtil.suspendIfNecessary(this, this::setFuture, + env.getAssignmentManager().regionClosedAbnormally(regionNode), env, + () -> unattach(env)); return null; default: throw new IllegalStateException("Unknown state: " + state); @@ -314,12 +354,11 @@ protected Procedure[] execute(MasterProcedureEnv env) } long backoff = retryCounter.getBackoffTimeAndIncrementAttempts(); LOG.warn("Failed updating meta, suspend {}secs {}; {};", backoff / 1000, this, regionNode, e); - setTimeout(Math.toIntExact(backoff)); - setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT); - skipPersistence(); - throw new ProcedureSuspendedException(); + throw suspend(Math.toIntExact(backoff), true); } finally { - regionNode.unlock(); + if (future == null) { + regionNode.unlock(this); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java index 91c0222facd1..abfe8b404354 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java @@ -19,8 +19,6 @@ import java.util.Arrays; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -30,7 +28,9 @@ import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionState.State; +import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureEvent; +import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -75,7 +75,7 @@ public AssignmentProcedureEvent(final RegionInfo regionInfo) { } } - final Lock lock = new ReentrantLock(); + private final RegionStateNodeLock lock; private final RegionInfo regionInfo; private final ProcedureEvent event; private final ConcurrentMap ritMap; @@ -106,6 +106,7 @@ public AssignmentProcedureEvent(final RegionInfo regionInfo) { this.regionInfo = regionInfo; this.event = new AssignmentProcedureEvent(regionInfo); this.ritMap = ritMap; + this.lock = new RegionStateNodeLock(regionInfo); } /** @@ -319,15 +320,70 @@ public void checkOnline() throws DoNotRetryRegionException { } } + // The below 3 methods are for normal locking operation, where the thread owner is the current + // thread. Typically you just need to use these 3 methods, and use try..finally to release the + // lock in the finally block + /** + * @see RegionStateNodeLock#lock() + */ public void lock() { lock.lock(); } + /** + * @see RegionStateNodeLock#tryLock() + */ public boolean tryLock() { return lock.tryLock(); } + /** + * @see RegionStateNodeLock#unlock() + */ public void unlock() { lock.unlock(); } + + // The below 3 methods are for locking region state node when executing procedures, where we may + // do some time consuming work under the lock, for example, updating meta. As we may suspend the + // procedure while holding the lock and then release it when the procedure is back, in another + // thread, so we need to use the procedure itself as owner, instead of the current thread. You can + // see the usage in TRSP, SCP, and RegionRemoteProcedureBase for more details. + // Notice that, this does not mean you must use these 3 methods when locking region state node in + // procedure, you are free to use the above 3 methods if you do not want to hold the lock when + // suspending the procedure. + /** + * @see RegionStateNodeLock#lock(Procedure, Runnable) + */ + public void lock(Procedure proc, Runnable wakeUp) throws ProcedureSuspendedException { + lock.lock(proc, wakeUp); + } + + /** + * @see RegionStateNodeLock#tryLock(Procedure) + */ + public boolean tryLock(Procedure proc) { + return lock.tryLock(proc); + } + + /** + * @see RegionStateNodeLock#unlock(Procedure) + */ + public void unlock(Procedure proc) { + lock.unlock(proc); + } + + /** + * @see RegionStateNodeLock#isLocked() + */ + boolean isLocked() { + return lock.isLocked(); + } + + /** + * @see RegionStateNodeLock#isLockedBy(Object) + */ + public boolean isLockedBy(Procedure proc) { + return lock.isLockedBy(proc); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNodeLock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNodeLock.java new file mode 100644 index 000000000000..55ad61360ca8 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNodeLock.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.assignment; + +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil; +import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * A lock implementation which supports unlock by another thread. + *

+ * This is because we need to hold region state node lock while updating region state to meta(for + * keeping consistency), so it is better to yield the procedure to release the procedure worker. But + * after waking up the procedure, we may use another procedure worker to execute the procedure, + * which means we need to unlock by another thread. + *

+ * For locking by procedure, we will also suspend the procedure if the lock is not ready, and + * schedule it again when lock is ready. This is very important to not block the PEWorker as we may + * hold the lock when updating meta, which could take a lot of time. + *

+ * Please see HBASE-28196 for more details. + */ +@InterfaceAudience.Private +class RegionStateNodeLock { + + // for better logging message + private final RegionInfo regionInfo; + + private final Lock lock = new ReentrantLock(); + + private final Queue waitingQueue = new ArrayDeque<>(); + + private Object owner; + + private int count; + + /** + * This is for abstraction the common lock/unlock logic for both Thread and Procedure. + */ + private interface QueueEntry { + + /** + * A thread, or a procedure. + */ + Object getOwner(); + + /** + * Called when we can not hold the lock and should wait. For thread, you should wait on a + * condition, and for procedure, a ProcedureSuspendedException should be thrown. + */ + void await() throws ProcedureSuspendedException; + + /** + * Called when it is your turn to get the lock. For thread, just delegate the call to + * condition's signal method, and for procedure, you should call the {@code wakeUp} action, to + * add the procedure back to procedure scheduler. + */ + void signal(); + } + + RegionStateNodeLock(RegionInfo regionInfo) { + this.regionInfo = regionInfo; + } + + private void lock0(QueueEntry entry) throws ProcedureSuspendedException { + lock.lock(); + try { + for (;;) { + if (owner == null) { + owner = entry.getOwner(); + count = 1; + return; + } + if (owner == entry.getOwner()) { + count++; + return; + } + waitingQueue.add(entry); + entry.await(); + } + } finally { + lock.unlock(); + } + } + + private boolean tryLock0(Object lockBy) { + if (!lock.tryLock()) { + return false; + } + try { + if (owner == null) { + owner = lockBy; + count = 1; + return true; + } + if (owner == lockBy) { + count++; + return true; + } + return false; + } finally { + lock.unlock(); + } + } + + private void unlock0(Object unlockBy) { + lock.lock(); + try { + if (owner == null) { + throw new IllegalMonitorStateException("RegionStateNode " + regionInfo + " is not locked"); + } + if (owner != unlockBy) { + throw new IllegalMonitorStateException("RegionStateNode " + regionInfo + " is locked by " + + owner + ", can not be unlocked by " + unlockBy); + } + count--; + if (count == 0) { + owner = null; + QueueEntry entry = waitingQueue.poll(); + if (entry != null) { + entry.signal(); + } + } + } finally { + lock.unlock(); + } + } + + /** + * Normal lock, will set the current thread as owner. Typically you should use try...finally to + * call unlock in the finally block. + */ + void lock() { + Thread currentThread = Thread.currentThread(); + try { + lock0(new QueueEntry() { + + private Condition cond; + + @Override + public void signal() { + cond.signal(); + } + + @Override + public Object getOwner() { + return currentThread; + } + + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "WA_AWAIT_NOT_IN_LOOP", + justification = "Loop is in the caller method") + @Override + public void await() { + if (cond == null) { + cond = lock.newCondition(); + } + cond.awaitUninterruptibly(); + } + }); + } catch (ProcedureSuspendedException e) { + // should not happen + throw new AssertionError(e); + } + } + + /** + * Normal tryLock, will set the current thread as owner. Typically you should use try...finally to + * call unlock in the finally block. + */ + boolean tryLock() { + return tryLock0(Thread.currentThread()); + } + + /** + * Normal unLock, will use the current thread as owner. Typically you should use try...finally to + * call unlock in the finally block. + */ + void unlock() { + unlock0(Thread.currentThread()); + } + + /** + * Lock by a procedure. You can release the lock in another thread. + *

+ * When the procedure can not get the lock immediately, a ProcedureSuspendedException will be + * thrown to suspend the procedure. And when we want to wake up a procedure, we will call the + * {@code wakeUp} action. Usually in the {@code wakeUp} action you should add the procedure back + * to procedure scheduler. + */ + void lock(Procedure proc, Runnable wakeUp) throws ProcedureSuspendedException { + lock0(new QueueEntry() { + + @Override + public Object getOwner() { + return proc; + } + + @Override + public void await() throws ProcedureSuspendedException { + ProcedureFutureUtil.suspend(proc); + } + + @Override + public void signal() { + // Here we need to set the owner to the procedure directly. + // For thread, we just block inside the lock0 method, so after signal we will continue and + // get the lock + // For procedure, the waking up here is actually a reschedule of the procedure to + // ProcedureExecutor, so here we need to set the owner first, and it is the procedure's duty + // to make sure that it has already hold the lock so do not need to call lock again, usually + // this should be done by calling isLockedBy method below + assert owner == null; + assert count == 0; + owner = proc; + count = 1; + wakeUp.run(); + } + }); + } + + /** + * TryLock by a procedure. You can release the lock in another thread. + */ + boolean tryLock(Procedure proc) { + return tryLock0(proc); + } + + /** + * Unlock by a procedure. You do not need to call this method in the same thread with lock. + */ + void unlock(Procedure proc) { + unlock0(proc); + } + + /** + * Check whether the lock is locked by someone. + */ + boolean isLocked() { + lock.lock(); + try { + return owner != null; + } finally { + lock.unlock(); + } + } + + /** + * Check whether the lock is locked by the given {@code lockBy}. + */ + boolean isLockedBy(Object lockBy) { + lock.lock(); + try { + return owner == lockBy; + } finally { + lock.unlock(); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index 3561e0cd055b..4d506365f238 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -176,7 +176,7 @@ public static void visitMetaEntry(final RegionStateVisitor visitor, final Result } } - void updateRegionLocation(RegionStateNode regionStateNode) throws IOException { + private Put generateUpdateRegionLocationPut(RegionStateNode regionStateNode) throws IOException { long time = EnvironmentEdgeManager.currentTime(); long openSeqNum = regionStateNode.getState() == State.OPEN ? regionStateNode.getOpenSeqNum() @@ -221,11 +221,34 @@ && hasGlobalReplicationScope(regionInfo.getTable()) .setTimestamp(put.getTimestamp()).setType(Cell.Type.Put).setValue(Bytes.toBytes(state.name())) .build()); LOG.info(info.toString()); - updateRegionLocation(regionInfo, state, put); + return put; + } + + CompletableFuture updateRegionLocation(RegionStateNode regionStateNode) { + Put put; + try { + put = generateUpdateRegionLocationPut(regionStateNode); + } catch (IOException e) { + return FutureUtils.failedFuture(e); + } + RegionInfo regionInfo = regionStateNode.getRegionInfo(); + State state = regionStateNode.getState(); + CompletableFuture future = updateRegionLocation(regionInfo, state, put); if (regionInfo.isMetaRegion() && regionInfo.isFirst()) { // mirror the meta location to zookeeper - mirrorMetaLocation(regionInfo, regionLocation, state); + // we store meta location in master local region which means the above method is + // synchronous(we just wrap the result with a CompletableFuture to make it look like + // asynchronous), so it is OK to just call this method directly here + assert future.isDone(); + if (!future.isCompletedExceptionally()) { + try { + mirrorMetaLocation(regionInfo, regionStateNode.getRegionLocation(), state); + } catch (IOException e) { + return FutureUtils.failedFuture(e); + } + } } + return future; } private void mirrorMetaLocation(RegionInfo regionInfo, ServerName serverName, State state) @@ -249,25 +272,31 @@ private void removeMirrorMetaLocation(int oldReplicaCount, int newReplicaCount) } } - private void updateRegionLocation(RegionInfo regionInfo, State state, Put put) - throws IOException { - try { - if (regionInfo.isMetaRegion()) { + private CompletableFuture updateRegionLocation(RegionInfo regionInfo, State state, + Put put) { + CompletableFuture future; + if (regionInfo.isMetaRegion()) { + try { masterRegion.update(r -> r.put(put)); - } else { - try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) { - table.put(put); - } + future = CompletableFuture.completedFuture(null); + } catch (Exception e) { + future = FutureUtils.failedFuture(e); } - } catch (IOException e) { - // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host! - // In tests we abort the Master! - String msg = String.format("FAILED persisting region=%s state=%s", - regionInfo.getShortNameToLog(), state); - LOG.error(msg, e); - master.abort(msg, e); - throw e; + } else { + AsyncTable table = master.getAsyncConnection().getTable(TableName.META_TABLE_NAME); + future = table.put(put); } + FutureUtils.addListener(future, (r, e) -> { + if (e != null) { + // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host! + // In tests we abort the Master! + String msg = String.format("FAILED persisting region=%s state=%s", + regionInfo.getShortNameToLog(), state); + LOG.error(msg, e); + master.abort(msg, e); + } + }); + return future; } private long getOpenSeqNumForParentRegion(RegionInfo region) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java index a0f0ba9ab809..b3de79c18264 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java @@ -405,7 +405,7 @@ private boolean include(final RegionStateNode node, final boolean offline) { // ============================================================================================ private void setServerState(ServerName serverName, ServerState state) { - ServerStateNode serverNode = getOrCreateServer(serverName); + ServerStateNode serverNode = getServerNode(serverName); synchronized (serverNode) { serverNode.setState(state); } @@ -746,18 +746,16 @@ public List getRegionFailedOpen() { // ========================================================================== /** - * Be judicious calling this method. Do it on server register ONLY otherwise you could mess up - * online server accounting. TOOD: Review usage and convert to {@link #getServerNode(ServerName)} - * where we can. + * Create the ServerStateNode when registering a new region server */ - public ServerStateNode getOrCreateServer(final ServerName serverName) { - return serverMap.computeIfAbsent(serverName, key -> new ServerStateNode(key)); + public void createServer(ServerName serverName) { + serverMap.computeIfAbsent(serverName, key -> new ServerStateNode(key)); } /** * Called by SCP at end of successful processing. */ - public void removeServer(final ServerName serverName) { + public void removeServer(ServerName serverName) { serverMap.remove(serverName); } @@ -776,17 +774,24 @@ public double getAverageLoad() { return numServers == 0 ? 0.0 : (double) totalLoad / (double) numServers; } - public ServerStateNode addRegionToServer(final RegionStateNode regionNode) { - ServerStateNode serverNode = getOrCreateServer(regionNode.getRegionLocation()); + public void addRegionToServer(final RegionStateNode regionNode) { + ServerStateNode serverNode = getServerNode(regionNode.getRegionLocation()); serverNode.addRegion(regionNode); - return serverNode; } - public ServerStateNode removeRegionFromServer(final ServerName serverName, + public void removeRegionFromServer(final ServerName serverName, final RegionStateNode regionNode) { - ServerStateNode serverNode = getOrCreateServer(serverName); - serverNode.removeRegion(regionNode); - return serverNode; + ServerStateNode serverNode = getServerNode(serverName); + // here the server node could be null. For example, if there is already a TRSP for a region and + // at the same time, the target server is crashed and there is a SCP. The SCP will interrupt the + // TRSP and the TRSP will first set the region as abnormally closed and remove it from the + // server node. But here, this TRSP is not a child procedure of the SCP, so it is possible that + // the SCP finishes, thus removes the server node for this region server, before the TRSP wakes + // up and enter here to remove the region node from the server node, then we will get a null + // server node here. + if (serverNode != null) { + serverNode.removeRegion(regionNode); + } } // ========================================================================== diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerStateNode.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerStateNode.java index 3bde01ad5f57..06eea30370f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerStateNode.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerStateNode.java @@ -35,6 +35,7 @@ public class ServerStateNode implements Comparable { private final Set regions; private final ServerName serverName; + // the lock here is for fencing SCP and TRSP, so not all operations need to hold this lock private final ReadWriteLock lock = new ReentrantReadWriteLock(); private volatile ServerState state = ServerState.ONLINE; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index a0118cbd7b05..2e2182b25d29 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -505,7 +505,8 @@ private byte[] getSplitRow() { public boolean prepareSplitRegion(final MasterProcedureEnv env) throws IOException { // Fail if we are taking snapshot for the given table if ( - env.getMasterServices().getSnapshotManager().isTakingSnapshot(getParentRegion().getTable()) + env.getMasterServices().getSnapshotManager() + .isTableTakingAnySnapshot(getParentRegion().getTable()) ) { setFailure(new IOException("Skip splitting region " + getParentRegion().getShortNameToLog() + ", because we are taking snapshot for the table " + getParentRegion().getTable())); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java index 81397915647d..8dfc08a5de89 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java @@ -24,6 +24,7 @@ import edu.umd.cs.findbugs.annotations.Nullable; import java.io.IOException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.ServerName; @@ -38,11 +39,13 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureUtil; import org.apache.hadoop.hbase.procedure2.ProcedureYieldException; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.hadoop.hbase.util.RetryCounter; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -137,6 +140,8 @@ public class TransitRegionStateProcedure private long forceRetainmentTotalWait; + private CompletableFuture future; + public TransitRegionStateProcedure() { } @@ -268,21 +273,54 @@ private void queueAssign(MasterProcedureEnv env, RegionStateNode regionNode) } } - private void openRegion(MasterProcedureEnv env, RegionStateNode regionNode) throws IOException { + private CompletableFuture getFuture() { + return future; + } + + private void setFuture(CompletableFuture f) { + future = f; + } + + private void openRegionAfterUpdatingMeta(ServerName loc) { + addChildProcedure(new OpenRegionProcedure(this, getRegion(), loc)); + setNextState(RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_OPENED); + } + + private void openRegion(MasterProcedureEnv env, RegionStateNode regionNode) + throws IOException, ProcedureSuspendedException { ServerName loc = regionNode.getRegionLocation(); + if ( + ProcedureFutureUtil.checkFuture(this, this::getFuture, this::setFuture, + () -> openRegionAfterUpdatingMeta(loc)) + ) { + return; + } if (loc == null || BOGUS_SERVER_NAME.equals(loc)) { LOG.warn("No location specified for {}, jump back to state {} to get one", getRegion(), RegionStateTransitionState.REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE); setNextState(RegionStateTransitionState.REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE); throw new HBaseIOException("Failed to open region, the location is null or bogus."); } - env.getAssignmentManager().regionOpening(regionNode); - addChildProcedure(new OpenRegionProcedure(this, getRegion(), loc)); - setNextState(RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_OPENED); + ProcedureFutureUtil.suspendIfNecessary(this, this::setFuture, + env.getAssignmentManager().regionOpening(regionNode), env, + () -> openRegionAfterUpdatingMeta(loc)); + } + + private void regionFailedOpenAfterUpdatingMeta(MasterProcedureEnv env, + RegionStateNode regionNode) { + setFailure(getClass().getSimpleName(), new RetriesExhaustedException( + "Max attempts " + env.getAssignmentManager().getAssignMaxAttempts() + " exceeded")); + regionNode.unsetProcedure(this); } private Flow confirmOpened(MasterProcedureEnv env, RegionStateNode regionNode) - throws IOException { + throws IOException, ProcedureSuspendedException { + if ( + ProcedureFutureUtil.checkFuture(this, this::getFuture, this::setFuture, + () -> regionFailedOpenAfterUpdatingMeta(env, regionNode)) + ) { + return Flow.NO_MORE_STATE; + } if (regionNode.isInState(State.OPEN)) { retryCounter = null; if (lastState == RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_OPENED) { @@ -306,14 +344,16 @@ private Flow confirmOpened(MasterProcedureEnv env, RegionStateNode regionNode) LOG.info("Retry={} of max={}; {}; {}", retries, maxAttempts, this, regionNode.toShortString()); if (retries >= maxAttempts) { - env.getAssignmentManager().regionFailedOpen(regionNode, true); - setFailure(getClass().getSimpleName(), new RetriesExhaustedException( - "Max attempts " + env.getAssignmentManager().getAssignMaxAttempts() + " exceeded")); - regionNode.unsetProcedure(this); + ProcedureFutureUtil.suspendIfNecessary(this, this::setFuture, + env.getAssignmentManager().regionFailedOpen(regionNode, true), env, + () -> regionFailedOpenAfterUpdatingMeta(env, regionNode)); return Flow.NO_MORE_STATE; } - env.getAssignmentManager().regionFailedOpen(regionNode, false); + // if not giving up, we will not update meta, so the returned CompletableFuture should be a fake + // one, which should have been completed already + CompletableFuture future = env.getAssignmentManager().regionFailedOpen(regionNode, false); + assert future.isDone(); // we failed to assign the region, force a new plan forceNewPlan = true; regionNode.setRegionLocation(null); @@ -329,17 +369,29 @@ private Flow confirmOpened(MasterProcedureEnv env, RegionStateNode regionNode) } } - private void closeRegion(MasterProcedureEnv env, RegionStateNode regionNode) throws IOException { + private void closeRegionAfterUpdatingMeta(RegionStateNode regionNode) { + CloseRegionProcedure closeProc = isSplit + ? new CloseRegionProcedure(this, getRegion(), regionNode.getRegionLocation(), assignCandidate, + true) + : new CloseRegionProcedure(this, getRegion(), regionNode.getRegionLocation(), assignCandidate, + evictCache); + addChildProcedure(closeProc); + setNextState(RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_CLOSED); + } + + private void closeRegion(MasterProcedureEnv env, RegionStateNode regionNode) + throws IOException, ProcedureSuspendedException { + if ( + ProcedureFutureUtil.checkFuture(this, this::getFuture, this::setFuture, + () -> closeRegionAfterUpdatingMeta(regionNode)) + ) { + return; + } if (regionNode.isInState(State.OPEN, State.CLOSING, State.MERGING, State.SPLITTING)) { // this is the normal case - env.getAssignmentManager().regionClosing(regionNode); - CloseRegionProcedure closeProc = isSplit - ? new CloseRegionProcedure(this, getRegion(), regionNode.getRegionLocation(), - assignCandidate, true) - : new CloseRegionProcedure(this, getRegion(), regionNode.getRegionLocation(), - assignCandidate, evictCache); - addChildProcedure(closeProc); - setNextState(RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_CLOSED); + ProcedureFutureUtil.suspendIfNecessary(this, this::setFuture, + env.getAssignmentManager().regionClosing(regionNode), env, + () -> closeRegionAfterUpdatingMeta(regionNode)); } else { forceNewPlan = true; regionNode.setRegionLocation(null); @@ -393,11 +445,16 @@ protected Procedure[] execute(MasterProcedureEnv env) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { RegionStateNode regionNode = env.getAssignmentManager().getRegionStates().getOrCreateRegionStateNode(getRegion()); - regionNode.lock(); + if (!regionNode.isLockedBy(this)) { + regionNode.lock(this, () -> ProcedureFutureUtil.wakeUp(this, env)); + } try { return super.execute(env); } finally { - regionNode.unlock(); + if (future == null) { + // release the lock if there is no pending updating meta operation + regionNode.unlock(this); + } } } @@ -452,10 +509,7 @@ protected Flow executeFromState(MasterProcedureEnv env, RegionStateTransitionSta "Failed transition, suspend {}secs {}; {}; waiting on rectified condition fixed " + "by other Procedure or operator intervention", backoff / 1000, this, regionNode.toShortString(), e); - setTimeout(Math.toIntExact(backoff)); - setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT); - skipPersistence(); - throw new ProcedureSuspendedException(); + throw suspend(Math.toIntExact(backoff), true); } } @@ -492,15 +546,25 @@ public void reportTransition(MasterProcedureEnv env, RegionStateNode regionNode, } // Should be called with RegionStateNode locked - public void serverCrashed(MasterProcedureEnv env, RegionStateNode regionNode, - ServerName serverName, boolean forceNewPlan) throws IOException { + public CompletableFuture serverCrashed(MasterProcedureEnv env, RegionStateNode regionNode, + ServerName serverName, boolean forceNewPlan) { this.forceNewPlan = forceNewPlan; if (remoteProc != null) { // this means we are waiting for the sub procedure, so wake it up - remoteProc.serverCrashed(env, regionNode, serverName); + try { + remoteProc.serverCrashed(env, regionNode, serverName); + } catch (Exception e) { + return FutureUtils.failedFuture(e); + } + return CompletableFuture.completedFuture(null); } else { - // we are in RUNNING state, just update the region state, and we will process it later. - env.getAssignmentManager().regionClosedAbnormally(regionNode); + if (regionNode.isInState(State.ABNORMALLY_CLOSED)) { + // should be a retry, where we have already changed the region state to abnormally closed + return CompletableFuture.completedFuture(null); + } else { + // we are in RUNNING state, just update the region state, and we will process it later. + return env.getAssignmentManager().regionClosedAbnormally(regionNode); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockManager.java index 7fab616449c2..eae9c7dcc1b7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockManager.java @@ -108,15 +108,6 @@ public MasterLock(final RegionInfo[] regionInfos, final String description) { this.description = description; } - /** - * Acquire the lock, waiting indefinitely until the lock is released or the thread is - * interrupted. - * @throws InterruptedException If current thread is interrupted while waiting for the lock - */ - public boolean acquire() throws InterruptedException { - return tryAcquire(0); - } - /** * Acquire the lock within a wait time. * @param timeoutMs The maximum time (in milliseconds) to wait for the lock, 0 to wait diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index 1a9c93b82c03..a2c4cd40c9c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; +import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.MetricsSnapshot; @@ -155,6 +156,16 @@ protected Flow executeFromState(final MasterProcedureEnv env, final CloneSnapsho updateTableDescriptorWithSFT(); newRegions = createFilesystemLayout(env, tableDescriptor, newRegions); env.getMasterServices().getTableDescriptors().update(tableDescriptor, true); + if (tableDescriptor.getErasureCodingPolicy() != null) { + setNextState(CloneSnapshotState.CLONE_SNAPSHOT_SET_ERASURE_CODING_POLICY); + } else { + setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ADD_TO_META); + } + break; + case CLONE_SNAPSHOT_SET_ERASURE_CODING_POLICY: + ErasureCodingUtils.setPolicy(env.getMasterFileSystem().getFileSystem(), + env.getMasterFileSystem().getRootDir(), getTableName(), + tableDescriptor.getErasureCodingPolicy()); setNextState(CloneSnapshotState.CLONE_SNAPSHOT_ADD_TO_META); break; case CLONE_SNAPSHOT_ADD_TO_META: diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 34e1f3d5bfc3..533b6fffcc43 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; @@ -98,6 +99,16 @@ protected Flow executeFromState(final MasterProcedureEnv env, final CreateTableS DeleteTableProcedure.deleteFromFs(env, getTableName(), newRegions, true); newRegions = createFsLayout(env, tableDescriptor, newRegions); env.getMasterServices().getTableDescriptors().update(tableDescriptor, true); + if (tableDescriptor.getErasureCodingPolicy() != null) { + setNextState(CreateTableState.CREATE_TABLE_SET_ERASURE_CODING_POLICY); + } else { + setNextState(CreateTableState.CREATE_TABLE_ADD_TO_META); + } + break; + case CREATE_TABLE_SET_ERASURE_CODING_POLICY: + ErasureCodingUtils.setPolicy(env.getMasterFileSystem().getFileSystem(), + env.getMasterFileSystem().getRootDir(), getTableName(), + tableDescriptor.getErasureCodingPolicy()); setNextState(CreateTableState.CREATE_TABLE_ADD_TO_META); break; case CREATE_TABLE_ADD_TO_META: diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java index 67f0442b618a..88f7e652cbff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java @@ -88,6 +88,11 @@ protected Procedure[] execute(MasterProcedureEnv env) RegionStates regionStates = env.getAssignmentManager().getRegionStates(); RegionStateNode regionNode = regionStates.getRegionStateNode(region); + if (regionNode == null) { + LOG.debug("Region {} is not in region states, it is very likely that it has been cleared by" + + " other procedures such as merge or split, so skip {}. See HBASE-28226", region, this); + return null; + } regionNode.lock(); try { if (!regionNode.isInState(State.OPEN) || regionNode.isInTransition()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java index bd8f95973ebc..43d69361c2d2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/HBCKServerCrashProcedure.java @@ -105,6 +105,8 @@ List getRegionsOnCrashedServer(MasterProcedureEnv env) { LOG.warn("Failed scan of hbase:meta for 'Unknown Servers'", ioe); return ris; } + // create the server state node too + env.getAssignmentManager().getRegionStates().createServer(getServerName()); LOG.info("Found {} mentions of {} in hbase:meta of OPEN/OPENING Regions: {}", visitor.getReassigns().size(), getServerName(), visitor.getReassigns().stream() .map(RegionInfo::getEncodedName).collect(Collectors.joining(","))); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java index f7f4146bd0d5..218d3096d8df 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.procedure; import java.io.IOException; +import java.util.concurrent.ExecutorService; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -86,6 +87,13 @@ public MasterProcedureEnv(final MasterServices master, this.remoteDispatcher = remoteDispatcher; } + /** + * Get a thread pool for executing some asynchronous tasks + */ + public ExecutorService getAsyncTaskExecutor() { + return master.getMasterProcedureExecutor().getAsyncTaskExecutor(); + } + public User getRequestUser() { return RpcServer.getRequestUser().orElse(Superusers.getSystemUser()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index ff0d7d2cc94b..9a52dbd079dd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -17,6 +17,11 @@ */ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_DEFAULT; +import static org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY; +import static org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_DISABLED; +import static org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_KEY; + import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -26,6 +31,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ConcurrentTableModificationException; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseIOException; @@ -36,6 +42,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; @@ -108,6 +115,7 @@ protected void preflightChecks(MasterProcedureEnv env, Boolean enabled) throws H } } } + if (!reopenRegions) { if (this.unmodifiedTableDescriptor == null) { throw new HBaseIOException( @@ -213,13 +221,22 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS postModify(env, state); if (reopenRegions) { setNextState(ModifyTableState.MODIFY_TABLE_REOPEN_ALL_REGIONS); - } else { - return Flow.NO_MORE_STATE; - } + } else + if (ErasureCodingUtils.needsSync(unmodifiedTableDescriptor, modifiedTableDescriptor)) { + setNextState(ModifyTableState.MODIFY_TABLE_SYNC_ERASURE_CODING_POLICY); + } else { + return Flow.NO_MORE_STATE; + } break; case MODIFY_TABLE_REOPEN_ALL_REGIONS: if (isTableEnabled(env)) { - addChildProcedure(new ReopenTableRegionsProcedure(getTableName())); + Configuration conf = env.getMasterConfiguration(); + long backoffMillis = conf.getLong(PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, + PROGRESSIVE_BATCH_BACKOFF_MILLIS_DEFAULT); + int batchSizeMax = + conf.getInt(PROGRESSIVE_BATCH_SIZE_MAX_KEY, PROGRESSIVE_BATCH_SIZE_MAX_DISABLED); + addChildProcedure( + new ReopenTableRegionsProcedure(getTableName(), backoffMillis, batchSizeMax)); } setNextState(ModifyTableState.MODIFY_TABLE_ASSIGN_NEW_REPLICAS); break; @@ -233,12 +250,24 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS } if (deleteColumnFamilyInModify) { setNextState(ModifyTableState.MODIFY_TABLE_DELETE_FS_LAYOUT); - } else { - return Flow.NO_MORE_STATE; - } + } else + if (ErasureCodingUtils.needsSync(unmodifiedTableDescriptor, modifiedTableDescriptor)) { + setNextState(ModifyTableState.MODIFY_TABLE_SYNC_ERASURE_CODING_POLICY); + } else { + return Flow.NO_MORE_STATE; + } break; case MODIFY_TABLE_DELETE_FS_LAYOUT: deleteFromFs(env, unmodifiedTableDescriptor, modifiedTableDescriptor); + if (ErasureCodingUtils.needsSync(unmodifiedTableDescriptor, modifiedTableDescriptor)) { + setNextState(ModifyTableState.MODIFY_TABLE_SYNC_ERASURE_CODING_POLICY); + break; + } else { + return Flow.NO_MORE_STATE; + } + case MODIFY_TABLE_SYNC_ERASURE_CODING_POLICY: + ErasureCodingUtils.sync(env.getMasterFileSystem().getFileSystem(), + env.getMasterFileSystem().getRootDir(), modifiedTableDescriptor); return Flow.NO_MORE_STATE; default: throw new UnsupportedOperationException("unhandled state=" + state); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java index 4efb1768b0ce..353636e6ddd6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master.procedure; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -53,6 +54,17 @@ public class ReopenTableRegionsProcedure private static final Logger LOG = LoggerFactory.getLogger(ReopenTableRegionsProcedure.class); + public static final String PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY = + "hbase.reopen.table.regions.progressive.batch.backoff.ms"; + public static final long PROGRESSIVE_BATCH_BACKOFF_MILLIS_DEFAULT = 0L; + public static final String PROGRESSIVE_BATCH_SIZE_MAX_KEY = + "hbase.reopen.table.regions.progressive.batch.size.max"; + public static final int PROGRESSIVE_BATCH_SIZE_MAX_DISABLED = -1; + private static final int PROGRESSIVE_BATCH_SIZE_MAX_DEFAULT_VALUE = Integer.MAX_VALUE; + + // this minimum prevents a max which would break this procedure + private static final int MINIMUM_BATCH_SIZE_MAX = 1; + private TableName tableName; // Specify specific regions of a table to reopen. @@ -61,20 +73,46 @@ public class ReopenTableRegionsProcedure private List regions = Collections.emptyList(); + private List currentRegionBatch = Collections.emptyList(); + private RetryCounter retryCounter; + private long reopenBatchBackoffMillis; + private int reopenBatchSize; + private int reopenBatchSizeMax; + private long regionsReopened = 0; + private long batchesProcessed = 0; + public ReopenTableRegionsProcedure() { - regionNames = Collections.emptyList(); + this(null); } public ReopenTableRegionsProcedure(TableName tableName) { - this.tableName = tableName; - this.regionNames = Collections.emptyList(); + this(tableName, Collections.emptyList()); } public ReopenTableRegionsProcedure(final TableName tableName, final List regionNames) { + this(tableName, regionNames, PROGRESSIVE_BATCH_BACKOFF_MILLIS_DEFAULT, + PROGRESSIVE_BATCH_SIZE_MAX_DISABLED); + } + + public ReopenTableRegionsProcedure(final TableName tableName, long reopenBatchBackoffMillis, + int reopenBatchSizeMax) { + this(tableName, Collections.emptyList(), reopenBatchBackoffMillis, reopenBatchSizeMax); + } + + public ReopenTableRegionsProcedure(final TableName tableName, final List regionNames, + long reopenBatchBackoffMillis, int reopenBatchSizeMax) { this.tableName = tableName; this.regionNames = regionNames; + this.reopenBatchBackoffMillis = reopenBatchBackoffMillis; + if (reopenBatchSizeMax == PROGRESSIVE_BATCH_SIZE_MAX_DISABLED) { + this.reopenBatchSize = Integer.MAX_VALUE; + this.reopenBatchSizeMax = Integer.MAX_VALUE; + } else { + this.reopenBatchSize = 1; + this.reopenBatchSizeMax = Math.max(reopenBatchSizeMax, MINIMUM_BATCH_SIZE_MAX); + } } @Override @@ -87,6 +125,30 @@ public TableOperationType getTableOperationType() { return TableOperationType.REGION_EDIT; } + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public long getRegionsReopened() { + return regionsReopened; + } + + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public long getBatchesProcessed() { + return batchesProcessed; + } + + @RestrictedApi(explanation = "Should only be called internally or in tests", link = "", + allowedOnPath = ".*(/src/test/.*|ReopenTableRegionsProcedure).java") + protected int progressBatchSize() { + int previousBatchSize = reopenBatchSize; + reopenBatchSize = Math.min(reopenBatchSizeMax, 2 * reopenBatchSize); + if (reopenBatchSize < previousBatchSize) { + // the batch size should never decrease. this must be overflow, so just use max + reopenBatchSize = reopenBatchSizeMax; + } + return reopenBatchSize; + } + private boolean canSchedule(MasterProcedureEnv env, HRegionLocation loc) { if (loc.getSeqNum() < 0) { return false; @@ -114,7 +176,13 @@ protected Flow executeFromState(MasterProcedureEnv env, ReopenTableRegionsState setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_REOPEN_REGIONS); return Flow.HAS_MORE_STATE; case REOPEN_TABLE_REGIONS_REOPEN_REGIONS: - for (HRegionLocation loc : regions) { + // if we didn't finish reopening the last batch yet, let's keep trying until we do. + // at that point, the batch will be empty and we can generate a new batch + if (!regions.isEmpty() && currentRegionBatch.isEmpty()) { + currentRegionBatch = regions.stream().limit(reopenBatchSize).collect(Collectors.toList()); + batchesProcessed++; + } + for (HRegionLocation loc : currentRegionBatch) { RegionStateNode regionNode = env.getAssignmentManager().getRegionStates().getRegionStateNode(loc.getRegion()); // this possible, maybe the region has already been merged or split, see HBASE-20921 @@ -133,39 +201,72 @@ protected Flow executeFromState(MasterProcedureEnv env, ReopenTableRegionsState regionNode.unlock(); } addChildProcedure(proc); + regionsReopened++; } setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_CONFIRM_REOPENED); return Flow.HAS_MORE_STATE; case REOPEN_TABLE_REGIONS_CONFIRM_REOPENED: - regions = regions.stream().map(env.getAssignmentManager().getRegionStates()::checkReopened) - .filter(l -> l != null).collect(Collectors.toList()); + // update region lists based on what's been reopened + regions = filterReopened(env, regions); + currentRegionBatch = filterReopened(env, currentRegionBatch); + + // existing batch didn't fully reopen, so try to resolve that first. + // since this is a retry, don't do the batch backoff + if (!currentRegionBatch.isEmpty()) { + return reopenIfSchedulable(env, currentRegionBatch, false); + } + if (regions.isEmpty()) { return Flow.NO_MORE_STATE; } - if (regions.stream().anyMatch(loc -> canSchedule(env, loc))) { - retryCounter = null; - setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_REOPEN_REGIONS); - return Flow.HAS_MORE_STATE; - } - // We can not schedule TRSP for all the regions need to reopen, wait for a while and retry - // again. - if (retryCounter == null) { - retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration()); - } - long backoff = retryCounter.getBackoffTimeAndIncrementAttempts(); - LOG.info( - "There are still {} region(s) which need to be reopened for table {} are in " - + "OPENING state, suspend {}secs and try again later", - regions.size(), tableName, backoff / 1000); - setTimeout(Math.toIntExact(backoff)); - setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT); - skipPersistence(); - throw new ProcedureSuspendedException(); + + // current batch is finished, schedule more regions + return reopenIfSchedulable(env, regions, true); default: throw new UnsupportedOperationException("unhandled state=" + state); } } + private List filterReopened(MasterProcedureEnv env, + List regionsToCheck) { + return regionsToCheck.stream().map(env.getAssignmentManager().getRegionStates()::checkReopened) + .filter(l -> l != null).collect(Collectors.toList()); + } + + private Flow reopenIfSchedulable(MasterProcedureEnv env, List regionsToReopen, + boolean shouldBatchBackoff) throws ProcedureSuspendedException { + if (regionsToReopen.stream().anyMatch(loc -> canSchedule(env, loc))) { + retryCounter = null; + setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_REOPEN_REGIONS); + if (shouldBatchBackoff && reopenBatchBackoffMillis > 0) { + progressBatchSize(); + setBackoffState(reopenBatchBackoffMillis); + throw new ProcedureSuspendedException(); + } else { + return Flow.HAS_MORE_STATE; + } + } + + // We can not schedule TRSP for all the regions need to reopen, wait for a while and retry + // again. + if (retryCounter == null) { + retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration()); + } + long backoffMillis = retryCounter.getBackoffTimeAndIncrementAttempts(); + LOG.info( + "There are still {} region(s) which need to be reopened for table {}. {} are in " + + "OPENING state, suspend {}secs and try again later", + regions.size(), tableName, currentRegionBatch.size(), backoffMillis / 1000); + setBackoffState(backoffMillis); + throw new ProcedureSuspendedException(); + } + + private void setBackoffState(long millis) { + setTimeout(Math.toIntExact(millis)); + setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT); + skipPersistence(); + } + private List getRegionLocationsForReopen(List tableRegionsForReopen) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java index bc2f41c13910..e16b33741065 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.favored.FavoredNodesManager; +import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.MetricsSnapshot; import org.apache.hadoop.hbase.master.RegionState; @@ -70,6 +71,7 @@ public class RestoreSnapshotProcedure extends AbstractStateMachineTableProcedure { private static final Logger LOG = LoggerFactory.getLogger(RestoreSnapshotProcedure.class); + private TableDescriptor oldTableDescriptor; private TableDescriptor modifiedTableDescriptor; private List regionsToRestore = null; private List regionsToRemove = null; @@ -94,18 +96,25 @@ public RestoreSnapshotProcedure(final MasterProcedureEnv env, this(env, tableDescriptor, snapshot, false); } + public RestoreSnapshotProcedure(final MasterProcedureEnv env, + final TableDescriptor tableDescriptor, final SnapshotDescription snapshot, + final boolean restoreAcl) throws HBaseIOException { + this(env, tableDescriptor, tableDescriptor, snapshot, restoreAcl); + } + /** * Constructor - * @param env MasterProcedureEnv - * @param tableDescriptor the table to operate on - * @param snapshot snapshot to restore from + * @param env MasterProcedureEnv + * @param modifiedTableDescriptor the table to operate on + * @param snapshot snapshot to restore from */ public RestoreSnapshotProcedure(final MasterProcedureEnv env, - final TableDescriptor tableDescriptor, final SnapshotDescription snapshot, - final boolean restoreAcl) throws HBaseIOException { + final TableDescriptor oldTableDescriptor, final TableDescriptor modifiedTableDescriptor, + final SnapshotDescription snapshot, final boolean restoreAcl) throws HBaseIOException { super(env); + this.oldTableDescriptor = oldTableDescriptor; // This is the new schema we are going to write out as this modification. - this.modifiedTableDescriptor = tableDescriptor; + this.modifiedTableDescriptor = modifiedTableDescriptor; preflightChecks(env, null/* Table can be online when restore is called? */); // Snapshot information this.snapshot = snapshot; @@ -143,6 +152,18 @@ protected Flow executeFromState(final MasterProcedureEnv env, final RestoreSnaps break; case RESTORE_SNAPSHOT_UPDATE_TABLE_DESCRIPTOR: updateTableDescriptor(env); + // for restore, table dir already exists. sync EC if necessary before doing the real + // restore. this may be useful in certain restore scenarios where a user is explicitly + // trying to disable EC for some reason as part of the restore. + if (ErasureCodingUtils.needsSync(oldTableDescriptor, modifiedTableDescriptor)) { + setNextState(RestoreSnapshotState.RESTORE_SNAPSHOT_SYNC_ERASURE_CODING_POLICY); + } else { + setNextState(RestoreSnapshotState.RESTORE_SNAPSHOT_WRITE_FS_LAYOUT); + } + break; + case RESTORE_SNAPSHOT_SYNC_ERASURE_CODING_POLICY: + ErasureCodingUtils.sync(env.getMasterFileSystem().getFileSystem(), + env.getMasterFileSystem().getRootDir(), modifiedTableDescriptor); setNextState(RestoreSnapshotState.RESTORE_SNAPSHOT_WRITE_FS_LAYOUT); break; case RESTORE_SNAPSHOT_WRITE_FS_LAYOUT: @@ -239,7 +260,8 @@ protected void serializeStateData(ProcedureStateSerializer serializer) throws IO RestoreSnapshotStateData.Builder restoreSnapshotMsg = RestoreSnapshotStateData.newBuilder() .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())).setSnapshot(this.snapshot) - .setModifiedTableSchema(ProtobufUtil.toTableSchema(modifiedTableDescriptor)); + .setModifiedTableSchema(ProtobufUtil.toTableSchema(modifiedTableDescriptor)) + .setOldTableSchema(ProtobufUtil.toTableSchema(oldTableDescriptor)); if (regionsToRestore != null) { for (RegionInfo hri : regionsToRestore) { @@ -281,6 +303,7 @@ protected void deserializeStateData(ProcedureStateSerializer serializer) throws serializer.deserialize(RestoreSnapshotStateData.class); setUser(MasterProcedureUtil.toUserInfo(restoreSnapshotMsg.getUserInfo())); snapshot = restoreSnapshotMsg.getSnapshot(); + oldTableDescriptor = ProtobufUtil.toTableDescriptor(restoreSnapshotMsg.getOldTableSchema()); modifiedTableDescriptor = ProtobufUtil.toTableDescriptor(restoreSnapshotMsg.getModifiedTableSchema()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java index 97976756d828..cdf13064e24a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CompletableFuture; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -41,6 +42,7 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; @@ -107,6 +109,10 @@ public class ServerCrashProcedure extends // progress will not update the state because the actual state is overwritten by its next state private ServerCrashState currentRunningState = getInitialState(); + private CompletableFuture updateMetaFuture; + + private int processedRegions = 0; + /** * Call this constructor queuing up a Procedure. * @param serverName Name of the crashed server. @@ -532,6 +538,14 @@ protected boolean isMatchingRegionLocation(RegionStateNode rsn) { return this.serverName.equals(rsn.getRegionLocation()); } + private CompletableFuture getUpdateMetaFuture() { + return updateMetaFuture; + } + + private void setUpdateMetaFuture(CompletableFuture f) { + updateMetaFuture = f; + } + /** * Assign the regions on the crashed RS to other Rses. *

@@ -542,14 +556,36 @@ protected boolean isMatchingRegionLocation(RegionStateNode rsn) { * We will also check whether the table for a region is enabled, if not, we will skip assigning * it. */ - private void assignRegions(MasterProcedureEnv env, List regions) throws IOException { + private void assignRegions(MasterProcedureEnv env, List regions) + throws IOException, ProcedureSuspendedException { AssignmentManager am = env.getMasterServices().getAssignmentManager(); boolean retainAssignment = env.getMasterConfiguration().getBoolean(MASTER_SCP_RETAIN_ASSIGNMENT, DEFAULT_MASTER_SCP_RETAIN_ASSIGNMENT); - for (RegionInfo region : regions) { + // Since we may suspend in the middle of this loop, so here we use processedRegions to record + // the progress, so next time we can locate the correct region + // We do not need to persist the processedRegions when serializing the procedure, as when master + // restarts, the sub procedure list will be cleared when rescheduling this SCP again, so we need + // to start from beginning. + for (int n = regions.size(); processedRegions < n; processedRegions++) { + RegionInfo region = regions.get(processedRegions); RegionStateNode regionNode = am.getRegionStates().getOrCreateRegionStateNode(region); - regionNode.lock(); + // There are two possible ways where we have already hold the lock here + // 1. We have already hold the lock and we suspend while updating meta, so after being woken + // up, we should skip lock again. + // 2. We suspend the procedure while trying to hold the lock, and finally it is our turn to + // hold the lock and we schedule the procedure again, this time we should have already hold + // the lock, so we do not need to lock again + if (!regionNode.isLockedBy(this)) { + regionNode.lock(this, () -> ProcedureFutureUtil.wakeUp(this, env)); + } try { + if ( + ProcedureFutureUtil.checkFuture(this, this::getUpdateMetaFuture, + this::setUpdateMetaFuture, () -> { + }) + ) { + continue; + } // This is possible, as when a server is dead, TRSP will fail to schedule a RemoteProcedure // and then try to assign the region to a new RS. And before it has updated the region // location to the new RS, we may have already called the am.getRegionsOnServer so we will @@ -572,8 +608,10 @@ private void assignRegions(MasterProcedureEnv env, List regions) thr } if (regionNode.getProcedure() != null) { LOG.info("{} found RIT {}; {}", this, regionNode.getProcedure(), regionNode); - regionNode.getProcedure().serverCrashed(env, regionNode, getServerName(), - !retainAssignment); + ProcedureFutureUtil.suspendIfNecessary(this, this::setUpdateMetaFuture, regionNode + .getProcedure().serverCrashed(env, regionNode, getServerName(), !retainAssignment), env, + () -> { + }); continue; } if ( @@ -583,7 +621,9 @@ private void assignRegions(MasterProcedureEnv env, List regions) thr // We need to change the state here otherwise the TRSP scheduled by DTP will try to // close the region from a dead server and will never succeed. Please see HBASE-23636 // for more details. - env.getAssignmentManager().regionClosedAbnormally(regionNode); + ProcedureFutureUtil.suspendIfNecessary(this, this::setUpdateMetaFuture, + env.getAssignmentManager().regionClosedAbnormally(regionNode), env, () -> { + }); LOG.info("{} found table disabling for region {}, set it state to ABNORMALLY_CLOSED.", this, regionNode); continue; @@ -599,11 +639,20 @@ private void assignRegions(MasterProcedureEnv env, List regions) thr TransitRegionStateProcedure proc = TransitRegionStateProcedure.assign(env, region, !retainAssignment, null); regionNode.setProcedure(proc); + // It is OK to still use addChildProcedure even if we suspend in the middle of this loop, as + // the subProcList will only be cleared when we successfully returned from the + // executeFromState method. This means we will submit all the TRSPs after we successfully + // finished this loop addChildProcedure(proc); } finally { - regionNode.unlock(); + if (updateMetaFuture == null) { + regionNode.unlock(this); + } } } + // we will call this method two times if the region server carries meta, so we need to reset it + // to 0 after successfully finished the above loop + processedRegions = 0; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java index 5e907c1681ac..83722d6c1dca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java @@ -110,8 +110,8 @@ assert getRegion().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID || isFailed() private void deleteRegionFromFileSystem(final MasterProcedureEnv env) throws IOException { RegionStateNode regionNode = env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion()); + regionNode.lock(); try { - regionNode.lock(); final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName()); HRegionFileSystem.deleteRegionFromFileSystem(env.getMasterConfiguration(), @@ -210,10 +210,10 @@ public TableOperationType getTableOperationType() { private TransitRegionStateProcedure createUnAssignProcedures(MasterProcedureEnv env) throws IOException { - return env.getAssignmentManager().createOneUnassignProcedure(getRegion(), true); + return env.getAssignmentManager().createOneUnassignProcedure(getRegion(), true, true); } private TransitRegionStateProcedure createAssignProcedures(MasterProcedureEnv env) { - return env.getAssignmentManager().createOneAssignProcedure(getRegion(), true); + return env.getAssignmentManager().createOneAssignProcedure(getRegion(), true, true); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java index cd4cca0f9186..b3b01f675c7f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java @@ -52,7 +52,7 @@ public abstract class AbstractPeerProcedure extends AbstractPeerNoLockPr // The sleep interval when waiting table to be enabled or disabled. protected static final int SLEEP_INTERVAL_MS = 1000; - // used to keep compatible with old client where we can only returns after updateStorage. + // used to keep compatible with old client where we can only return after updateStorage. protected ProcedurePrepareLatch latch; protected AbstractPeerProcedure() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java index c88d613e5260..cff1b3879360 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.master.procedure.GlobalProcedureInterface; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface; +import org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureUtil; @@ -43,8 +44,6 @@ import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration; -import org.apache.hadoop.hbase.util.FutureUtils; -import org.apache.hadoop.hbase.util.IdLock; import org.apache.hadoop.hbase.util.RetryCounter; import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -73,7 +72,7 @@ public class MigrateReplicationQueueFromZkToTableProcedure private List disabledPeerIds; - private CompletableFuture future; + private CompletableFuture future; private ExecutorService executor; @@ -84,6 +83,14 @@ public String getGlobalId() { return getClass().getSimpleName(); } + private CompletableFuture getFuture() { + return future; + } + + private void setFuture(CompletableFuture f) { + future = f; + } + private ProcedureSuspendedException suspend(Configuration conf, LongConsumer backoffConsumer) throws ProcedureSuspendedException { if (retryCounter == null) { @@ -153,6 +160,12 @@ private void waitUntilNoPeerProcedure(MasterProcedureEnv env) throws ProcedureSu LOG.info("No pending peer procedures found, continue..."); } + private void finishMigartion() { + shutdownExecutorService(); + setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING); + resetRetry(); + } + @Override protected Flow executeFromState(MasterProcedureEnv env, MigrateReplicationQueueFromZkToTableState state) @@ -195,52 +208,23 @@ protected Flow executeFromState(MasterProcedureEnv env, setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE); return Flow.HAS_MORE_STATE; case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE: - if (future != null) { - // should have finished when we arrive here - assert future.isDone(); - try { - future.get(); - } catch (Exception e) { - future = null; - throw suspend(env.getMasterConfiguration(), - backoff -> LOG.warn("failed to migrate queue data, sleep {} secs and retry later", - backoff / 1000, e)); + try { + if ( + ProcedureFutureUtil.checkFuture(this, this::getFuture, this::setFuture, + this::finishMigartion) + ) { + return Flow.HAS_MORE_STATE; } - shutdownExecutorService(); - setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING); - resetRetry(); - return Flow.HAS_MORE_STATE; + ProcedureFutureUtil.suspendIfNecessary(this, this::setFuture, + env.getReplicationPeerManager() + .migrateQueuesFromZk(env.getMasterServices().getZooKeeper(), getExecutorService()), + env, this::finishMigartion); + } catch (IOException e) { + throw suspend(env.getMasterConfiguration(), + backoff -> LOG.warn("failed to migrate queue data, sleep {} secs and retry later", + backoff / 1000, e)); } - future = env.getReplicationPeerManager() - .migrateQueuesFromZk(env.getMasterServices().getZooKeeper(), getExecutorService()); - FutureUtils.addListener(future, (r, e) -> { - // should acquire procedure execution lock to make sure that the procedure executor has - // finished putting this procedure to the WAITING_TIMEOUT state, otherwise there could be - // race and cause unexpected result - IdLock procLock = - env.getMasterServices().getMasterProcedureExecutor().getProcExecutionLock(); - IdLock.Entry lockEntry; - try { - lockEntry = procLock.getLockEntry(getProcId()); - } catch (IOException ioe) { - LOG.error("Error while acquiring execution lock for procedure {}" - + " when trying to wake it up, aborting...", this, ioe); - env.getMasterServices().abort("Can not acquire procedure execution lock", e); - return; - } - try { - setTimeoutFailure(env); - } finally { - procLock.releaseLockEntry(lockEntry); - } - }); - // here we set timeout to -1 so the ProcedureExecutor will not schedule a Timer for us - setTimeout(-1); - setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT); - // skip persistence is a must now since when restarting, if the procedure is in - // WAITING_TIMEOUT state and has -1 as timeout, it will block there forever... - skipPersistence(); - throw new ProcedureSuspendedException(); + return Flow.HAS_MORE_STATE; case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING: long rsWithLowerVersion = env.getMasterServices().getServerManager().getOnlineServers().values().stream() diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java index 988c519f781d..322b5bb7fc78 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java @@ -797,7 +797,7 @@ private CompletableFuture runAsync(ExceptionalRunnable task, ExecutorService /** * Submit the migration tasks to the given {@code executor}. */ - CompletableFuture migrateQueuesFromZk(ZKWatcher zookeeper, ExecutorService executor) { + CompletableFuture migrateQueuesFromZk(ZKWatcher zookeeper, ExecutorService executor) { // the replication queue table creation is asynchronous and will be triggered by addPeer, so // here we need to manually initialize it since we will not call addPeer. try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index ed7ef583ec52..14c7b4925de2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -1086,9 +1086,10 @@ private synchronized long restoreSnapshot(final SnapshotDescription snapshot, } try { + TableDescriptor oldDescriptor = master.getTableDescriptors().get(tableName); long procId = master.getMasterProcedureExecutor().submitProcedure( new RestoreSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(), - tableDescriptor, snapshot, restoreAcl), + oldDescriptor, tableDescriptor, snapshot, restoreAcl), nonceKey); this.restoreTableToProcIdMap.put(tableName, procId); return procId; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java index 58ecaca09ec6..f746adf0b89e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java @@ -17,12 +17,16 @@ */ package org.apache.hadoop.hbase.master.snapshot; +import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_RPC_TIMEOUT; +import static org.apache.hadoop.hbase.HConstants.HBASE_RPC_TIMEOUT_KEY; + import java.io.IOException; import java.util.List; import java.util.concurrent.CancellationException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -65,6 +69,8 @@ public abstract class TakeSnapshotHandler extends EventHandler implements SnapshotSentinel, ForeignExceptionSnare { private static final Logger LOG = LoggerFactory.getLogger(TakeSnapshotHandler.class); + public static final String HBASE_SNAPSHOT_MASTER_LOCK_ACQUIRE_TIMEOUT = + "hbase.snapshot.master.lock.acquire.timeout"; private volatile boolean finished; @@ -85,6 +91,13 @@ public abstract class TakeSnapshotHandler extends EventHandler protected final TableName snapshotTable; protected final SnapshotManifest snapshotManifest; protected final SnapshotManager snapshotManager; + /** + * Snapshot creation requires table lock. If any region of the table is in transition, table lock + * cannot be acquired by LockProcedure and hence snapshot creation could hang for potentially very + * long time. This timeout will ensure snapshot creation fails-fast by waiting for only given + * timeout. + */ + private final long lockAcquireTimeoutMs; protected TableDescriptor htd; @@ -129,6 +142,8 @@ public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices ma "Taking " + snapshot.getType() + " snapshot on table: " + snapshotTable, false, true); this.snapshotManifest = SnapshotManifest.create(conf, rootFs, workingDir, snapshot, monitor, status); + this.lockAcquireTimeoutMs = conf.getLong(HBASE_SNAPSHOT_MASTER_LOCK_ACQUIRE_TIMEOUT, + conf.getLong(HBASE_RPC_TIMEOUT_KEY, DEFAULT_HBASE_RPC_TIMEOUT)); } private TableDescriptor loadTableDescriptor() throws IOException { @@ -147,12 +162,16 @@ private TableDescriptor loadTableDescriptor() throws IOException { public TakeSnapshotHandler prepare() throws Exception { super.prepare(); // after this, you should ensure to release this lock in case of exceptions - this.tableLock.acquire(); - try { - this.htd = loadTableDescriptor(); // check that .tableinfo is present - } catch (Exception e) { - this.tableLock.release(); - throw e; + if (this.tableLock.tryAcquire(this.lockAcquireTimeoutMs)) { + try { + this.htd = loadTableDescriptor(); // check that .tableinfo is present + } catch (Exception e) { + this.tableLock.release(); + throw e; + } + } else { + LOG.error("Master lock could not be acquired in {} ms", lockAcquireTimeoutMs); + throw new DoNotRetryIOException("Master lock could not be acquired"); } return this; } @@ -176,7 +195,12 @@ public void process() { tableLockToRelease = master.getLockManager().createMasterLock(snapshotTable, LockType.SHARED, this.getClass().getName() + ": take snapshot " + snapshot.getName()); tableLock.release(); - tableLockToRelease.acquire(); + boolean isTableLockAcquired = tableLockToRelease.tryAcquire(this.lockAcquireTimeoutMs); + if (!isTableLockAcquired) { + LOG.error("Could not acquire shared lock on table {} in {} ms", snapshotTable, + lockAcquireTimeoutMs); + throw new IOException("Could not acquire shared lock on table " + snapshotTable); + } } // If regions move after this meta scan, the region specific snapshot should fail, triggering // an external exception that gets captured here. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java index e04d67a0aaaf..60f0f126ab60 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java @@ -280,6 +280,7 @@ public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, Table calendar.set(Calendar.HOUR_OF_DAY, 0); calendar.set(Calendar.MINUTE, 0); calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MILLISECOND, 0); Date expireDate = calendar.getTime(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/RpcLogDetails.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/RpcLogDetails.java index 235d82302d64..263fff66a738 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/RpcLogDetails.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/RpcLogDetails.java @@ -42,6 +42,7 @@ public class RpcLogDetails extends NamedQueuePayload { private final String clientAddress; private final long responseSize; private final long blockBytesScanned; + private final long fsReadTime; private final String className; private final boolean isSlowLog; private final boolean isLargeLog; @@ -49,12 +50,14 @@ public class RpcLogDetails extends NamedQueuePayload { private final Map requestAttributes; public RpcLogDetails(RpcCall rpcCall, Message param, String clientAddress, long responseSize, - long blockBytesScanned, String className, boolean isSlowLog, boolean isLargeLog) { + long blockBytesScanned, long fsReadTime, String className, boolean isSlowLog, + boolean isLargeLog) { super(SLOW_LOG_EVENT); this.rpcCall = rpcCall; this.clientAddress = clientAddress; this.responseSize = responseSize; this.blockBytesScanned = blockBytesScanned; + this.fsReadTime = fsReadTime; this.className = className; this.isSlowLog = isSlowLog; this.isLargeLog = isLargeLog; @@ -92,6 +95,10 @@ public long getBlockBytesScanned() { return blockBytesScanned; } + public long getFsReadTime() { + return fsReadTime; + } + public String getClassName() { return className; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/SlowLogQueueService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/SlowLogQueueService.java index fb29b8563ef7..ea4e286bf43f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/SlowLogQueueService.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/SlowLogQueueService.java @@ -124,6 +124,7 @@ public void consumeEventFromDisruptor(NamedQueuePayload namedQueuePayload) { final String clientAddress = rpcLogDetails.getClientAddress(); final long responseSize = rpcLogDetails.getResponseSize(); final long blockBytesScanned = rpcLogDetails.getBlockBytesScanned(); + final long fsReadTime = rpcLogDetails.getFsReadTime(); final String className = rpcLogDetails.getClassName(); final TooSlowLog.SlowLogPayload.Type type = getLogType(rpcLogDetails); if (type == null) { @@ -168,7 +169,8 @@ public void consumeEventFromDisruptor(NamedQueuePayload namedQueuePayload) { .setProcessingTime(processingTime).setQueueTime(qTime) .setRegionName(slowLogParams != null ? slowLogParams.getRegionName() : StringUtils.EMPTY) .setResponseSize(responseSize).setBlockBytesScanned(blockBytesScanned) - .setServerClass(className).setStartTime(startTime).setType(type).setUserName(userName) + .setFsReadTime(fsReadTime).setServerClass(className).setStartTime(startTime).setType(type) + .setUserName(userName) .addAllRequestAttribute(buildNameBytesPairs(rpcLogDetails.getRequestAttributes())) .addAllConnectionAttribute(buildNameBytesPairs(rpcLogDetails.getConnectionAttributes())); if (slowLogParams != null && slowLogParams.getScan() != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFutureUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFutureUtil.java new file mode 100644 index 000000000000..b606ce963960 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFutureUtil.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.procedure2; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; +import java.util.function.Supplier; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.hadoop.hbase.util.IdLock; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A helper class for switching procedure out(yielding) while it is doing some time consuming + * operation, such as updating meta, where we can get a {@link CompletableFuture} about the + * operation. + */ +@InterfaceAudience.Private +public final class ProcedureFutureUtil { + + private static final Logger LOG = LoggerFactory.getLogger(ProcedureFutureUtil.class); + + private ProcedureFutureUtil() { + } + + public static boolean checkFuture(Procedure proc, Supplier> getFuture, + Consumer> setFuture, Runnable actionAfterDone) throws IOException { + CompletableFuture future = getFuture.get(); + if (future == null) { + return false; + } + // reset future + setFuture.accept(null); + FutureUtils.get(future); + actionAfterDone.run(); + return true; + } + + public static void suspendIfNecessary(Procedure proc, + Consumer> setFuture, CompletableFuture future, + MasterProcedureEnv env, Runnable actionAfterDone) + throws IOException, ProcedureSuspendedException { + MutableBoolean completed = new MutableBoolean(false); + Thread currentThread = Thread.currentThread(); + // This is for testing. In ProcedureTestingUtility, we will restart a ProcedureExecutor and + // reuse it, for performance, so we need to make sure that all the procedure have been stopped. + // But here, the callback of this future is not executed in a PEWorker, so in ProcedureExecutor + // we have no way to stop it. So here, we will get the asyncTaskExecutor first, in the PEWorker + // thread, where the ProcedureExecutor should have not been stopped yet, then when calling the + // callback, if the ProcedureExecutor have already been stopped and restarted, the + // asyncTaskExecutor will also be shutdown so we can not add anything back to the scheduler. + ExecutorService asyncTaskExecutor = env.getAsyncTaskExecutor(); + FutureUtils.addListener(future, (r, e) -> { + if (Thread.currentThread() == currentThread) { + LOG.debug("The future has completed while adding callback, give up suspending procedure {}", + proc); + // this means the future has already been completed, as we call the callback directly while + // calling addListener, so here we just set completed to true without doing anything + completed.setTrue(); + return; + } + LOG.debug("Going to wake up procedure {} because future has completed", proc); + // This callback may be called inside netty's event loop, so we should not block it for a long + // time. The worker executor will hold the execution lock while executing the procedure, and + // we may persist the procedure state inside the lock, which is a time consuming operation. + // And what makes things worse is that, we persist procedure state to master local region, + // where the AsyncFSWAL implementation will use the same netty's event loop for dealing with + // I/O, which could even cause dead lock. + asyncTaskExecutor.execute(() -> wakeUp(proc, env)); + }); + if (completed.getValue()) { + FutureUtils.get(future); + actionAfterDone.run(); + } else { + // suspend the procedure + setFuture.accept(future); + proc.skipPersistence(); + suspend(proc); + } + } + + public static void suspend(Procedure proc) throws ProcedureSuspendedException { + proc.skipPersistence(); + throw new ProcedureSuspendedException(); + } + + public static void wakeUp(Procedure proc, MasterProcedureEnv env) { + // should acquire procedure execution lock to make sure that the procedure executor has + // finished putting this procedure to the WAITING_TIMEOUT state, otherwise there could be + // race and cause unexpected result + IdLock procLock = env.getMasterServices().getMasterProcedureExecutor().getProcExecutionLock(); + IdLock.Entry lockEntry; + try { + lockEntry = procLock.getLockEntry(proc.getProcId()); + } catch (IOException e) { + LOG.error("Error while acquiring execution lock for procedure {}" + + " when trying to wake it up, aborting...", proc, e); + env.getMasterServices().abort("Can not acquire procedure execution lock", e); + return; + } + try { + env.getProcedureScheduler().addFront(proc); + } finally { + procLock.releaseLockEntry(lockEntry); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java index ddf804243ed8..4b89e18a8021 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java @@ -22,6 +22,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.ipc.RpcCall; +import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -49,9 +51,15 @@ public class DefaultOperationQuota implements OperationQuota { protected long readDiff = 0; protected long writeCapacityUnitDiff = 0; protected long readCapacityUnitDiff = 0; + private boolean useResultSizeBytes; + private long blockSizeBytes; - public DefaultOperationQuota(final Configuration conf, final QuotaLimiter... limiters) { + public DefaultOperationQuota(final Configuration conf, final int blockSizeBytes, + final QuotaLimiter... limiters) { this(conf, Arrays.asList(limiters)); + this.useResultSizeBytes = + conf.getBoolean(OperationQuota.USE_RESULT_SIZE_BYTES, USE_RESULT_SIZE_BYTES_DEFAULT); + this.blockSizeBytes = blockSizeBytes; } /** @@ -94,8 +102,17 @@ public void checkQuota(int numWrites, int numReads, int numScans) throws RpcThro public void close() { // Adjust the quota consumed for the specified operation writeDiff = operationSize[OperationType.MUTATE.ordinal()] - writeConsumed; - readDiff = operationSize[OperationType.GET.ordinal()] - + operationSize[OperationType.SCAN.ordinal()] - readConsumed; + + long resultSize = + operationSize[OperationType.GET.ordinal()] + operationSize[OperationType.SCAN.ordinal()]; + if (useResultSizeBytes) { + readDiff = resultSize - readConsumed; + } else { + long blockBytesScanned = + RpcServer.getCurrentCall().map(RpcCall::getBlockBytesScanned).orElse(0L); + readDiff = Math.max(blockBytesScanned, resultSize) - readConsumed; + } + writeCapacityUnitDiff = calculateWriteCapacityUnitDiff(operationSize[OperationType.MUTATE.ordinal()], writeConsumed); readCapacityUnitDiff = calculateReadCapacityUnitDiff( @@ -140,8 +157,15 @@ public void addMutation(final Mutation mutation) { */ protected void updateEstimateConsumeQuota(int numWrites, int numReads, int numScans) { writeConsumed = estimateConsume(OperationType.MUTATE, numWrites, 100); - readConsumed = estimateConsume(OperationType.GET, numReads, 100); - readConsumed += estimateConsume(OperationType.SCAN, numScans, 1000); + + if (useResultSizeBytes) { + readConsumed = estimateConsume(OperationType.GET, numReads, 100); + readConsumed += estimateConsume(OperationType.SCAN, numScans, 1000); + } else { + // assume 1 block required for reads. this is probably a low estimate, which is okay + readConsumed = numReads > 0 ? blockSizeBytes : 0; + readConsumed += numScans > 0 ? blockSizeBytes : 0; + } writeCapacityUnitConsumed = calculateWriteCapacityUnit(writeConsumed); readCapacityUnitConsumed = calculateReadCapacityUnit(readConsumed); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java index 1b7200f5f22f..1788e550f22a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java @@ -40,9 +40,9 @@ public class ExceedOperationQuota extends DefaultOperationQuota { private static final Logger LOG = LoggerFactory.getLogger(ExceedOperationQuota.class); private QuotaLimiter regionServerLimiter; - public ExceedOperationQuota(final Configuration conf, QuotaLimiter regionServerLimiter, - final QuotaLimiter... limiters) { - super(conf, limiters); + public ExceedOperationQuota(final Configuration conf, int blockSizeBytes, + QuotaLimiter regionServerLimiter, final QuotaLimiter... limiters) { + super(conf, blockSizeBytes, limiters); this.regionServerLimiter = regionServerLimiter; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/OperationQuota.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/OperationQuota.java index aaae64b6184a..ffc3cd50825c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/OperationQuota.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/OperationQuota.java @@ -32,9 +32,20 @@ public interface OperationQuota { public enum OperationType { MUTATE, GET, - SCAN + SCAN, + CHECK_AND_MUTATE } + /** + * If false, the default, then IO based throttles will consume read availability based on the + * block bytes scanned by the given request. If true then IO based throttles will use result size + * rather than block bytes scanned. Using block bytes scanned should be preferable to using result + * size, because otherwise access patterns like heavily filtered scans may be able to produce a + * significant and effectively un-throttled workload. + */ + String USE_RESULT_SIZE_BYTES = "hbase.quota.use.result.size.bytes"; + boolean USE_RESULT_SIZE_BYTES_DEFAULT = false; + /** * Checks if it is possible to execute the specified operation. The quota will be estimated based * on the number of operations to perform and the average size accumulated during time. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 0a57b9fd8f8f..67b2aecc5448 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -137,7 +137,8 @@ public QuotaLimiter getUserLimiter(final UserGroupInformation ugi, final TableNa * @return the quota info associated to specified user */ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { - return computeIfAbsent(userQuotaCache, getQuotaUserName(ugi), UserQuotaState::new, + return computeIfAbsent(userQuotaCache, getQuotaUserName(ugi), + () -> QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration()), this::triggerCacheRefresh); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index f9fa1a95c0cc..44357c88d2dc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.DoNotRetryIOException; @@ -49,7 +50,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TimeUnit; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.QuotaScope; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle; @@ -73,6 +77,26 @@ public class QuotaUtil extends QuotaTableUtil { // the default one write capacity unit is 1024 bytes (1KB) public static final long DEFAULT_WRITE_CAPACITY_UNIT = 1024; + /* + * The below defaults, if configured, will be applied to otherwise unthrottled users. For example, + * set `hbase.quota.default.user.machine.read.size` to `1048576` in your hbase-site.xml to ensure + * that any given user may not query more than 1mb per second from any given machine, unless + * explicitly permitted by a persisted quota. All of these defaults use TimeUnit.SECONDS and + * QuotaScope.MACHINE. + */ + public static final String QUOTA_DEFAULT_USER_MACHINE_READ_NUM = + "hbase.quota.default.user.machine.read.num"; + public static final String QUOTA_DEFAULT_USER_MACHINE_READ_SIZE = + "hbase.quota.default.user.machine.read.size"; + public static final String QUOTA_DEFAULT_USER_MACHINE_REQUEST_NUM = + "hbase.quota.default.user.machine.request.num"; + public static final String QUOTA_DEFAULT_USER_MACHINE_REQUEST_SIZE = + "hbase.quota.default.user.machine.request.size"; + public static final String QUOTA_DEFAULT_USER_MACHINE_WRITE_NUM = + "hbase.quota.default.user.machine.write.num"; + public static final String QUOTA_DEFAULT_USER_MACHINE_WRITE_SIZE = + "hbase.quota.default.user.machine.write.size"; + /** Table descriptor for Quota internal table */ public static final TableDescriptor QUOTA_TABLE_DESC = TableDescriptorBuilder.newBuilder(QUOTA_TABLE_NAME) @@ -154,6 +178,31 @@ public static void deleteRegionServerQuota(final Connection connection, final St deleteQuotas(connection, getRegionServerRowKey(regionServer)); } + public static OperationQuota.OperationType getQuotaOperationType(ClientProtos.Action action, + boolean hasCondition) { + if (action.hasMutation()) { + return getQuotaOperationType(action.getMutation(), hasCondition); + } + return OperationQuota.OperationType.GET; + } + + public static OperationQuota.OperationType + getQuotaOperationType(ClientProtos.MutateRequest mutateRequest) { + return getQuotaOperationType(mutateRequest.getMutation(), mutateRequest.hasCondition()); + } + + private static OperationQuota.OperationType + getQuotaOperationType(ClientProtos.MutationProto mutationProto, boolean hasCondition) { + ClientProtos.MutationProto.MutationType mutationType = mutationProto.getMutateType(); + if ( + hasCondition || mutationType == ClientProtos.MutationProto.MutationType.APPEND + || mutationType == ClientProtos.MutationProto.MutationType.INCREMENT + ) { + return OperationQuota.OperationType.CHECK_AND_MUTATE; + } + return OperationQuota.OperationType.MUTATE; + } + protected static void switchExceedThrottleQuota(final Connection connection, boolean exceedThrottleQuotaEnabled) throws IOException { if (exceedThrottleQuotaEnabled) { @@ -284,10 +333,14 @@ public static Map fetchUserQuotas(final Connection conne assert isUserRowKey(key); String user = getUserFromRowKey(key); + if (results[i].isEmpty()) { + userQuotas.put(user, buildDefaultUserQuotaState(connection.getConfiguration())); + continue; + } + final UserQuotaState quotaInfo = new UserQuotaState(nowTs); userQuotas.put(user, quotaInfo); - if (results[i].isEmpty()) continue; assert Bytes.equals(key, results[i].getRow()); try { @@ -321,6 +374,38 @@ public void visitUserQuotas(String userName, Quotas quotas) { return userQuotas; } + protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf) { + QuotaProtos.Throttle.Builder throttleBuilder = QuotaProtos.Throttle.newBuilder(); + + buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_READ_NUM) + .ifPresent(throttleBuilder::setReadNum); + buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_READ_SIZE) + .ifPresent(throttleBuilder::setReadSize); + buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_REQUEST_NUM) + .ifPresent(throttleBuilder::setReqNum); + buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_REQUEST_SIZE) + .ifPresent(throttleBuilder::setReqSize); + buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_WRITE_NUM) + .ifPresent(throttleBuilder::setWriteNum); + buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_WRITE_SIZE) + .ifPresent(throttleBuilder::setWriteSize); + + UserQuotaState state = new UserQuotaState(); + QuotaProtos.Quotas defaultQuotas = + QuotaProtos.Quotas.newBuilder().setThrottle(throttleBuilder.build()).build(); + state.setQuotas(defaultQuotas); + return state; + } + + private static Optional buildDefaultTimedQuota(Configuration conf, String key) { + int defaultSoftLimit = conf.getInt(key, -1); + if (defaultSoftLimit == -1) { + return Optional.empty(); + } + return Optional.of(ProtobufUtil.toTimedQuota(defaultSoftLimit, + java.util.concurrent.TimeUnit.SECONDS, org.apache.hadoop.hbase.quotas.QuotaScope.MACHINE)); + } + public static Map fetchTableQuotas(final Connection connection, final List gets, Map tableMachineFactors) throws IOException { return fetchGlobalQuotas("table", connection, gets, new KeyFromRow() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 4b09c0308f9e..3c72c662887b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Optional; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.ipc.RpcScheduler; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.Region; @@ -113,7 +114,8 @@ QuotaCache getQuotaCache() { * @param table the table where the operation will be executed * @return the OperationQuota */ - public OperationQuota getQuota(final UserGroupInformation ugi, final TableName table) { + public OperationQuota getQuota(final UserGroupInformation ugi, final TableName table, + final int blockSizeBytes) { if (isQuotaEnabled() && !table.isSystemTable() && isRpcThrottleEnabled()) { UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi); QuotaLimiter userLimiter = userQuotaState.getTableLimiter(table); @@ -123,7 +125,8 @@ public OperationQuota getQuota(final UserGroupInformation ugi, final TableName t LOG.trace("get quota for ugi=" + ugi + " table=" + table + " userLimiter=" + userLimiter); } if (!useNoop) { - return new DefaultOperationQuota(this.rsServices.getConfiguration(), userLimiter); + return new DefaultOperationQuota(this.rsServices.getConfiguration(), blockSizeBytes, + userLimiter); } } else { QuotaLimiter nsLimiter = quotaCache.getNamespaceLimiter(table.getNamespaceAsString()); @@ -139,11 +142,11 @@ public OperationQuota getQuota(final UserGroupInformation ugi, final TableName t } if (!useNoop) { if (exceedThrottleQuotaEnabled) { - return new ExceedOperationQuota(this.rsServices.getConfiguration(), rsLimiter, - userLimiter, tableLimiter, nsLimiter); + return new ExceedOperationQuota(this.rsServices.getConfiguration(), blockSizeBytes, + rsLimiter, userLimiter, tableLimiter, nsLimiter); } else { - return new DefaultOperationQuota(this.rsServices.getConfiguration(), userLimiter, - tableLimiter, nsLimiter, rsLimiter); + return new DefaultOperationQuota(this.rsServices.getConfiguration(), blockSizeBytes, + userLimiter, tableLimiter, nsLimiter, rsLimiter); } } } @@ -168,6 +171,8 @@ public OperationQuota checkQuota(final Region region, final OperationQuota.Opera return checkQuota(region, 0, 1, 0); case MUTATE: return checkQuota(region, 1, 0, 0); + case CHECK_AND_MUTATE: + return checkQuota(region, 1, 1, 0); } throw new RuntimeException("Invalid operation type: " + type); } @@ -175,18 +180,24 @@ public OperationQuota checkQuota(final Region region, final OperationQuota.Opera /** * Check the quota for the current (rpc-context) user. Returns the OperationQuota used to get the * available quota and to report the data/usage of the operation. - * @param region the region where the operation will be performed - * @param actions the "multi" actions to perform + * @param region the region where the operation will be performed + * @param actions the "multi" actions to perform + * @param hasCondition whether the RegionAction has a condition * @return the OperationQuota * @throws RpcThrottlingException if the operation cannot be executed due to quota exceeded. */ - public OperationQuota checkQuota(final Region region, final List actions) - throws IOException, RpcThrottlingException { + public OperationQuota checkQuota(final Region region, final List actions, + boolean hasCondition) throws IOException, RpcThrottlingException { int numWrites = 0; int numReads = 0; for (final ClientProtos.Action action : actions) { if (action.hasMutation()) { numWrites++; + OperationQuota.OperationType operationType = + QuotaUtil.getQuotaOperationType(action, hasCondition); + if (operationType == OperationQuota.OperationType.CHECK_AND_MUTATE) { + numReads++; + } } else if (action.hasGet()) { numReads++; } @@ -213,9 +224,10 @@ private OperationQuota checkQuota(final Region region, final int numWrites, fina } else { ugi = User.getCurrent().getUGI(); } - TableName table = region.getTableDescriptor().getTableName(); + TableDescriptor tableDescriptor = region.getTableDescriptor(); + TableName table = tableDescriptor.getTableName(); - OperationQuota quota = getQuota(ugi, table); + OperationQuota quota = getQuota(ugi, table, region.getMinBlockSizeBytes()); try { quota.checkQuota(numWrites, numReads, numScans); } catch (RpcThrottlingException e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index 5cd3a92e5b69..62ff6f9a92fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -157,7 +157,7 @@ protected void doAdd(MutableSegment currentActive, Cell cell, MemStoreSizing mem Cell toAdd = maybeCloneWithAllocator(currentActive, cell, false); boolean mslabUsed = (toAdd != cell); // This cell data is backed by the same byte[] where we read request in RPC(See - // HBASE-15180). By default MSLAB is ON and we might have copied cell to MSLAB area. If + // HBASE-15180). By default, MSLAB is ON and we might have copied cell to MSLAB area. If // not we must do below deep copy. Or else we will keep referring to the bigger chunk of // memory and prevent it from getting GCed. // Copy to MSLAB would not have happened if diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMultiFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMultiFileWriter.java index a02b05f66ba3..6370d6a79ccb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMultiFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMultiFileWriter.java @@ -64,7 +64,7 @@ public void init(StoreScanner sourceScanner, WriterFactory factory) { * Commit all writers. *

* Notice that here we use the same maxSeqId for all output files since we haven't - * find an easy to find enough sequence ids for different output files in some corner cases. See + * found an easy to find enough sequence ids for different output files in some corner cases. See * comments in HBASE-15400 for more details. */ public List commitWriters(long maxSeqId, boolean majorCompaction) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java index 984b9e09a5af..20c0d93b4708 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.errorprone.annotations.RestrictedApi; import java.util.List; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Server; @@ -70,7 +71,9 @@ public CompactedHFilesDischarger(final int period, final Stoppable stopper, * no-executor before you call run. * @return The old setting for useExecutor */ - boolean setUseExecutor(final boolean useExecutor) { + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public boolean setUseExecutor(final boolean useExecutor) { boolean oldSetting = this.useExecutor; this.useExecutor = useExecutor; return oldSetting; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 0dc96747dd36..02dc5e71b185 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -458,6 +458,8 @@ public MetricsTableRequests getMetricsTableRequests() { private final CellComparator cellComparator; + private final int minBlockSizeBytes; + /** * @return The smallest mvcc readPoint across all the scanners in this region. Writes older than * this readPoint, are included in every read operation. @@ -916,6 +918,9 @@ public HRegion(final HRegionFileSystem fs, final WAL wal, final Configuration co .remove(getRegionInfo().getEncodedName()); } } + + minBlockSizeBytes = Arrays.stream(this.htableDescriptor.getColumnFamilies()) + .mapToInt(ColumnFamilyDescriptor::getBlocksize).min().orElse(HConstants.DEFAULT_BLOCKSIZE); } private void setHTableSpecificConf() { @@ -2047,6 +2052,11 @@ public Configuration getReadOnlyConfiguration() { return new ReadOnlyConfiguration(this.conf); } + @Override + public int getMinBlockSizeBytes() { + return minBlockSizeBytes; + } + private ThreadPoolExecutor getStoreOpenAndCloseThreadPool(final String threadNamePrefix) { int numStores = Math.max(1, this.htableDescriptor.getColumnFamilyCount()); int maxThreads = Math.min(numStores, conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX, @@ -2512,6 +2522,20 @@ public FlushResultImpl flushcache(List families, boolean writeFlushReque LOG.debug(msg); return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false); } + // cannot flush non-existing column families, and fail-fast + if (families != null) { + List noSuchFamilies = + families.stream().filter(cf -> !getTableDescriptor().hasColumnFamily(cf)) + .map(cf -> Bytes.toString(cf)).collect(Collectors.toList()); + if (noSuchFamilies.size() > 0) { + String noSuchFamiliesMsg = String.format( + "There are non-existing families %s, we cannot flush the region %s, in table %s.", + noSuchFamilies, getRegionInfo().getRegionNameAsString(), + getTableDescriptor().getTableName().getNameAsString()); + LOG.warn(noSuchFamiliesMsg); + return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, noSuchFamiliesMsg, false); + } + } MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this); status.setStatus("Acquiring readlock on region"); // block waiting for the lock for flushing cache diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 3042a2eae451..dfb8e2a204fe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -587,6 +587,11 @@ protected String getProcessName() { return REGIONSERVER; } + @Override + protected RegionServerCoprocessorHost getCoprocessorHost() { + return getRegionServerCoprocessorHost(); + } + @Override protected boolean canCreateBaseZNode() { return !clusterMode(); @@ -1218,11 +1223,15 @@ private ClusterStatusProtos.ServerLoad buildServerLoad(long reportStartTime, lon serverLoad.addCoprocessors(coprocessorBuilder.setName(coprocessor).build()); } } - computeIfPersistentBucketCache(bc -> { - bc.getRegionCachedInfo().forEach((regionName, prefetchSize) -> { - serverLoad.putRegionCachedInfo(regionName, roundSize(prefetchSize, unitMB)); + + getBlockCache().ifPresent(cache -> { + cache.getRegionCachedInfo().ifPresent(regionCachedInfo -> { + regionCachedInfo.forEach((regionName, prefetchSize) -> { + serverLoad.putRegionCachedInfo(regionName, roundSize(prefetchSize, unitMB)); + }); }); }); + serverLoad.setReportStartTime(reportStartTime); serverLoad.setReportEndTime(reportEndTime); if (this.infoServer != null) { @@ -1587,13 +1596,14 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, int totalStaticBloomSizeKB = roundSize(totalStaticBloomSize, unitKB); int regionSizeMB = roundSize(totalRegionSize, unitMB); final MutableFloat currentRegionCachedRatio = new MutableFloat(0.0f); - computeIfPersistentBucketCache(bc -> { - if (bc.getRegionCachedInfo().containsKey(regionEncodedName)) { - currentRegionCachedRatio.setValue(regionSizeMB == 0 - ? 0.0f - : (float) roundSize(bc.getRegionCachedInfo().get(regionEncodedName), unitMB) - / regionSizeMB); - } + getBlockCache().ifPresent(bc -> { + bc.getRegionCachedInfo().ifPresent(regionCachedInfo -> { + if (regionCachedInfo.containsKey(regionEncodedName)) { + currentRegionCachedRatio.setValue(regionSizeMB == 0 + ? 0.0f + : (float) roundSize(regionCachedInfo.get(regionEncodedName), unitMB) / regionSizeMB); + } + }); }); HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution(); @@ -1674,14 +1684,13 @@ private static class CompactionChecker extends ScheduledChore { @Override protected void chore() { - for (Region r : this.instance.onlineRegions.values()) { + for (HRegion hr : this.instance.onlineRegions.values()) { // If region is read only or compaction is disabled at table level, there's no need to // iterate through region's stores - if (r == null || r.isReadOnly() || !r.getTableDescriptor().isCompactionEnabled()) { + if (hr == null || hr.isReadOnly() || !hr.getTableDescriptor().isCompactionEnabled()) { continue; } - HRegion hr = (HRegion) r; for (HStore s : hr.stores.values()) { try { long multiplier = s.getCompactionCheckMultiplier(); @@ -1709,7 +1718,7 @@ protected void chore() { } } } catch (IOException e) { - LOG.warn("Failed major compaction check on " + r, e); + LOG.warn("Failed major compaction check on " + hr, e); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 5e2bf00f85be..dccfd0c0af7b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -1616,7 +1616,7 @@ protected void refreshStoreSizeAndTotalBytes() throws IOException { for (HStoreFile hsf : this.storeEngine.getStoreFileManager().getStorefiles()) { StoreFileReader r = hsf.getReader(); if (r == null) { - LOG.warn("StoreFile {} has a null Reader", hsf); + LOG.debug("StoreFile {} has a null Reader", hsf); continue; } this.storeSize.addAndGet(r.length()); @@ -1785,7 +1785,7 @@ public int getCompactedFilesCount() { private LongStream getStoreFileAgeStream() { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { - LOG.warn("StoreFile {} has a null Reader", sf); + LOG.debug("StoreFile {} has a null Reader", sf); return false; } else { return true; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 6c7fc504b5fd..ef0ee71f7fdc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.OptionalDouble; -import java.util.OptionalLong; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -72,62 +71,16 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { private CacheStats cacheStats; private CacheStats l1Stats = null; private CacheStats l2Stats = null; - - private volatile long numStores = 0; private volatile long numWALFiles = 0; private volatile long walFileSize = 0; - private volatile long numStoreFiles = 0; - private volatile long memstoreSize = 0; - private volatile long onHeapMemstoreSize = 0; - private volatile long offHeapMemstoreSize = 0; - private volatile long storeFileSize = 0; - private volatile double storeFileSizeGrowthRate = 0; - private volatile long maxStoreFileCount = 0; - private volatile long maxStoreFileAge = 0; - private volatile long minStoreFileAge = 0; - private volatile long avgStoreFileAge = 0; - private volatile long numReferenceFiles = 0; - private volatile double requestsPerSecond = 0.0; - private volatile long readRequestsCount = 0; - private volatile double readRequestsRatePerSecond = 0; - private volatile long cpRequestsCount = 0; - private volatile long filteredReadRequestsCount = 0; - private volatile long writeRequestsCount = 0; - private volatile double writeRequestsRatePerSecond = 0; - private volatile long checkAndMutateChecksFailed = 0; - private volatile long checkAndMutateChecksPassed = 0; - private volatile long storefileIndexSize = 0; - private volatile long totalStaticIndexSize = 0; - private volatile long totalStaticBloomSize = 0; - private volatile long bloomFilterRequestsCount = 0; - private volatile long bloomFilterNegativeResultsCount = 0; - private volatile long bloomFilterEligibleRequestsCount = 0; - private volatile long numMutationsWithoutWAL = 0; - private volatile long dataInMemoryWithoutWAL = 0; - private volatile double percentFileLocal = 0; - private volatile double percentFileLocalSecondaryRegions = 0; - private volatile long flushedCellsCount = 0; - private volatile long compactedCellsCount = 0; - private volatile long majorCompactedCellsCount = 0; - private volatile long flushedCellsSize = 0; - private volatile long compactedCellsSize = 0; - private volatile long majorCompactedCellsSize = 0; - private volatile long cellsCountCompactedToMob = 0; - private volatile long cellsCountCompactedFromMob = 0; - private volatile long cellsSizeCompactedToMob = 0; - private volatile long cellsSizeCompactedFromMob = 0; - private volatile long mobFlushCount = 0; - private volatile long mobFlushedCellsCount = 0; - private volatile long mobFlushedCellsSize = 0; - private volatile long mobScanCellsCount = 0; - private volatile long mobScanCellsSize = 0; private volatile long mobFileCacheAccessCount = 0; private volatile long mobFileCacheMissCount = 0; private volatile double mobFileCacheHitRatio = 0; private volatile long mobFileCacheEvictedCount = 0; private volatile long mobFileCacheCount = 0; - private volatile long blockedRequestsCount = 0L; - private volatile long averageRegionSize = 0L; + + private volatile RegionMetricAggregate aggregate = new RegionMetricAggregate(null); + protected final Map> requestsCountCache = new ConcurrentHashMap>(); @@ -249,7 +202,7 @@ public long getTotalRequestCount() { @Override public long getTotalRowActionRequestCount() { - return readRequestsCount + writeRequestsCount; + return aggregate.readRequestsCount + aggregate.writeRequestsCount; } @Override @@ -462,7 +415,7 @@ public void forceRecompute() { @Override public long getNumStores() { - return numStores; + return aggregate.numStores; } @Override @@ -491,92 +444,92 @@ public long getNumWALSlowAppend() { @Override public long getNumStoreFiles() { - return numStoreFiles; + return aggregate.numStoreFiles; } @Override public long getMaxStoreFiles() { - return maxStoreFileCount; + return aggregate.maxStoreFileCount; } @Override public long getMaxStoreFileAge() { - return maxStoreFileAge; + return aggregate.maxStoreFileAge; } @Override public long getMinStoreFileAge() { - return minStoreFileAge; + return aggregate.minStoreFileAge; } @Override public long getAvgStoreFileAge() { - return avgStoreFileAge; + return aggregate.avgStoreFileAge; } @Override public long getNumReferenceFiles() { - return numReferenceFiles; + return aggregate.numReferenceFiles; } @Override public long getMemStoreSize() { - return memstoreSize; + return aggregate.memstoreSize; } @Override public long getOnHeapMemStoreSize() { - return onHeapMemstoreSize; + return aggregate.onHeapMemstoreSize; } @Override public long getOffHeapMemStoreSize() { - return offHeapMemstoreSize; + return aggregate.offHeapMemstoreSize; } @Override public long getStoreFileSize() { - return storeFileSize; + return aggregate.storeFileSize; } @Override public double getStoreFileSizeGrowthRate() { - return storeFileSizeGrowthRate; + return aggregate.storeFileSizeGrowthRate; } @Override public double getRequestsPerSecond() { - return requestsPerSecond; + return aggregate.requestsPerSecond; } @Override public long getReadRequestsCount() { - return readRequestsCount; + return aggregate.readRequestsCount; } @Override public long getCpRequestsCount() { - return cpRequestsCount; + return aggregate.cpRequestsCount; } @Override public double getReadRequestsRatePerSecond() { - return readRequestsRatePerSecond; + return aggregate.readRequestsRatePerSecond; } @Override public long getFilteredReadRequestsCount() { - return filteredReadRequestsCount; + return aggregate.filteredReadRequestsCount; } @Override public long getWriteRequestsCount() { - return writeRequestsCount; + return aggregate.writeRequestsCount; } @Override public double getWriteRequestsRatePerSecond() { - return writeRequestsRatePerSecond; + return aggregate.writeRequestsRatePerSecond; } @Override @@ -606,62 +559,62 @@ public long getRpcMutateRequestsCount() { @Override public long getCheckAndMutateChecksFailed() { - return checkAndMutateChecksFailed; + return aggregate.checkAndMutateChecksFailed; } @Override public long getCheckAndMutateChecksPassed() { - return checkAndMutateChecksPassed; + return aggregate.checkAndMutateChecksPassed; } @Override public long getStoreFileIndexSize() { - return storefileIndexSize; + return aggregate.storefileIndexSize; } @Override public long getTotalStaticIndexSize() { - return totalStaticIndexSize; + return aggregate.totalStaticIndexSize; } @Override public long getTotalStaticBloomSize() { - return totalStaticBloomSize; + return aggregate.totalStaticBloomSize; } @Override public long getBloomFilterRequestsCount() { - return bloomFilterRequestsCount; + return aggregate.bloomFilterRequestsCount; } @Override public long getBloomFilterNegativeResultsCount() { - return bloomFilterNegativeResultsCount; + return aggregate.bloomFilterNegativeResultsCount; } @Override public long getBloomFilterEligibleRequestsCount() { - return bloomFilterEligibleRequestsCount; + return aggregate.bloomFilterEligibleRequestsCount; } @Override public long getNumMutationsWithoutWAL() { - return numMutationsWithoutWAL; + return aggregate.numMutationsWithoutWAL; } @Override public long getDataInMemoryWithoutWAL() { - return dataInMemoryWithoutWAL; + return aggregate.dataInMemoryWithoutWAL; } @Override public double getPercentFileLocal() { - return percentFileLocal; + return aggregate.percentFileLocal; } @Override public double getPercentFileLocalSecondaryRegions() { - return percentFileLocalSecondaryRegions; + return aggregate.percentFileLocalSecondaryRegions; } @Override @@ -674,77 +627,77 @@ public long getUpdatesBlockedTime() { @Override public long getFlushedCellsCount() { - return flushedCellsCount; + return aggregate.flushedCellsCount; } @Override public long getCompactedCellsCount() { - return compactedCellsCount; + return aggregate.compactedCellsCount; } @Override public long getMajorCompactedCellsCount() { - return majorCompactedCellsCount; + return aggregate.majorCompactedCellsCount; } @Override public long getFlushedCellsSize() { - return flushedCellsSize; + return aggregate.flushedCellsSize; } @Override public long getCompactedCellsSize() { - return compactedCellsSize; + return aggregate.compactedCellsSize; } @Override public long getMajorCompactedCellsSize() { - return majorCompactedCellsSize; + return aggregate.majorCompactedCellsSize; } @Override public long getCellsCountCompactedFromMob() { - return cellsCountCompactedFromMob; + return aggregate.cellsCountCompactedFromMob; } @Override public long getCellsCountCompactedToMob() { - return cellsCountCompactedToMob; + return aggregate.cellsCountCompactedToMob; } @Override public long getCellsSizeCompactedFromMob() { - return cellsSizeCompactedFromMob; + return aggregate.cellsSizeCompactedFromMob; } @Override public long getCellsSizeCompactedToMob() { - return cellsSizeCompactedToMob; + return aggregate.cellsSizeCompactedToMob; } @Override public long getMobFlushCount() { - return mobFlushCount; + return aggregate.mobFlushCount; } @Override public long getMobFlushedCellsCount() { - return mobFlushedCellsCount; + return aggregate.mobFlushedCellsCount; } @Override public long getMobFlushedCellsSize() { - return mobFlushedCellsSize; + return aggregate.mobFlushedCellsSize; } @Override public long getMobScanCellsCount() { - return mobScanCellsCount; + return aggregate.mobScanCellsCount; } @Override public long getMobScanCellsSize() { - return mobScanCellsSize; + return aggregate.mobScanCellsSize; } @Override @@ -777,6 +730,247 @@ public int getActiveScanners() { return regionServer.getRpcServices().getScannersCount(); } + private static final class RegionMetricAggregate { + private long numStores = 0; + private long numStoreFiles = 0; + private long memstoreSize = 0; + private long onHeapMemstoreSize = 0; + private long offHeapMemstoreSize = 0; + private long storeFileSize = 0; + private double storeFileSizeGrowthRate = 0; + private long maxStoreFileCount = 0; + private long maxStoreFileAge = 0; + private long minStoreFileAge = Long.MAX_VALUE; + private long avgStoreFileAge = 0; + private long numReferenceFiles = 0; + + private long cpRequestsCount = 0; + private double requestsPerSecond = 0.0; + private long readRequestsCount = 0; + private double readRequestsRatePerSecond = 0; + private long filteredReadRequestsCount = 0; + private long writeRequestsCount = 0; + private double writeRequestsRatePerSecond = 0; + private long checkAndMutateChecksFailed = 0; + private long checkAndMutateChecksPassed = 0; + private long storefileIndexSize = 0; + private long totalStaticIndexSize = 0; + private long totalStaticBloomSize = 0; + private long bloomFilterRequestsCount = 0; + private long bloomFilterNegativeResultsCount = 0; + private long bloomFilterEligibleRequestsCount = 0; + private long numMutationsWithoutWAL = 0; + private long dataInMemoryWithoutWAL = 0; + private double percentFileLocal = 0; + private double percentFileLocalSecondaryRegions = 0; + private long flushedCellsCount = 0; + private long compactedCellsCount = 0; + private long majorCompactedCellsCount = 0; + private long flushedCellsSize = 0; + private long compactedCellsSize = 0; + private long majorCompactedCellsSize = 0; + private long cellsCountCompactedToMob = 0; + private long cellsCountCompactedFromMob = 0; + private long cellsSizeCompactedToMob = 0; + private long cellsSizeCompactedFromMob = 0; + private long mobFlushCount = 0; + private long mobFlushedCellsCount = 0; + private long mobFlushedCellsSize = 0; + private long mobScanCellsCount = 0; + private long mobScanCellsSize = 0; + private long blockedRequestsCount = 0L; + private long averageRegionSize = 0L; + private long totalReadRequestsDelta = 0; + private long totalWriteRequestsDelta = 0; + + private RegionMetricAggregate(RegionMetricAggregate other) { + if (other != null) { + requestsPerSecond = other.requestsPerSecond; + readRequestsRatePerSecond = other.readRequestsRatePerSecond; + writeRequestsRatePerSecond = other.writeRequestsRatePerSecond; + } + } + + private void aggregate(HRegionServer regionServer, + Map> requestsCountCache) { + HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); + HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = new HDFSBlocksDistribution(); + + long avgAgeNumerator = 0; + long numHFiles = 0; + int regionCount = 0; + + for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { + Deltas deltas = calculateReadWriteDeltas(r, requestsCountCache); + totalReadRequestsDelta += deltas.readRequestsCountDelta; + totalWriteRequestsDelta += deltas.writeRequestsCountDelta; + + numMutationsWithoutWAL += r.getNumMutationsWithoutWAL(); + dataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL(); + cpRequestsCount += r.getCpRequestsCount(); + readRequestsCount += r.getReadRequestsCount(); + filteredReadRequestsCount += r.getFilteredReadRequestsCount(); + writeRequestsCount += r.getWriteRequestsCount(); + checkAndMutateChecksFailed += r.getCheckAndMutateChecksFailed(); + checkAndMutateChecksPassed += r.getCheckAndMutateChecksPassed(); + blockedRequestsCount += r.getBlockedRequestsCount(); + + StoreFileStats storeFileStats = aggregateStores(r.getStores()); + numHFiles += storeFileStats.numHFiles; + avgAgeNumerator += storeFileStats.avgAgeNumerator; + + HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution(); + hdfsBlocksDistribution.add(distro); + if (r.getRegionInfo().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { + hdfsBlocksDistributionSecondaryRegions.add(distro); + } + + regionCount++; + } + + float localityIndex = + hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname()); + percentFileLocal = Double.isNaN(localityIndex) ? 0 : (localityIndex * 100); + + float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions + .getBlockLocalityIndex(regionServer.getServerName().getHostname()); + percentFileLocalSecondaryRegions = + Double.isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100); + + if (regionCount > 0) { + averageRegionSize = (memstoreSize + storeFileSize) / regionCount; + } + + // if there were no store files, we'll never have updated this with Math.min + // so set it to 0, which is a better value to display in case of no storefiles + if (minStoreFileAge == Long.MAX_VALUE) { + this.minStoreFileAge = 0; + } + + if (numHFiles != 0) { + avgStoreFileAge = avgAgeNumerator / numHFiles; + } + } + + private static final class Deltas { + private final long readRequestsCountDelta; + private final long writeRequestsCountDelta; + + private Deltas(long readRequestsCountDelta, long writeRequestsCountDelta) { + this.readRequestsCountDelta = readRequestsCountDelta; + this.writeRequestsCountDelta = writeRequestsCountDelta; + } + } + + private Deltas calculateReadWriteDeltas(HRegion r, + Map> requestsCountCache) { + String encodedRegionName = r.getRegionInfo().getEncodedName(); + long currentReadRequestsCount = r.getReadRequestsCount(); + long currentWriteRequestsCount = r.getWriteRequestsCount(); + if (requestsCountCache.containsKey(encodedRegionName)) { + long lastReadRequestsCount = requestsCountCache.get(encodedRegionName).get(0); + long lastWriteRequestsCount = requestsCountCache.get(encodedRegionName).get(1); + + // Update cache for our next comparison + requestsCountCache.get(encodedRegionName).set(0, currentReadRequestsCount); + requestsCountCache.get(encodedRegionName).set(1, currentWriteRequestsCount); + + long readRequestsDelta = currentReadRequestsCount - lastReadRequestsCount; + long writeRequestsDelta = currentWriteRequestsCount - lastWriteRequestsCount; + return new Deltas(readRequestsDelta, writeRequestsDelta); + } else { + // List[0] -> readRequestCount + // List[1] -> writeRequestCount + ArrayList requests = new ArrayList(2); + requests.add(currentReadRequestsCount); + requests.add(currentWriteRequestsCount); + requestsCountCache.put(encodedRegionName, requests); + return new Deltas(currentReadRequestsCount, currentWriteRequestsCount); + } + } + + public void updateRates(long timeSinceLastRun, long expectedPeriod, long lastStoreFileSize) { + requestsPerSecond = + (totalReadRequestsDelta + totalWriteRequestsDelta) / (timeSinceLastRun / 1000.0); + + double readRequestsRatePerMilliSecond = (double) totalReadRequestsDelta / expectedPeriod; + double writeRequestsRatePerMilliSecond = (double) totalWriteRequestsDelta / expectedPeriod; + + readRequestsRatePerSecond = readRequestsRatePerMilliSecond * 1000.0; + writeRequestsRatePerSecond = writeRequestsRatePerMilliSecond * 1000.0; + + long intervalStoreFileSize = storeFileSize - lastStoreFileSize; + storeFileSizeGrowthRate = (double) intervalStoreFileSize * 1000.0 / expectedPeriod; + } + + private static final class StoreFileStats { + private final long numHFiles; + private final long avgAgeNumerator; + + private StoreFileStats(long numHFiles, long avgAgeNumerator) { + this.numHFiles = numHFiles; + this.avgAgeNumerator = avgAgeNumerator; + } + } + + private StoreFileStats aggregateStores(List stores) { + numStores += stores.size(); + long numHFiles = 0; + long avgAgeNumerator = 0; + for (Store store : stores) { + numStoreFiles += store.getStorefilesCount(); + memstoreSize += store.getMemStoreSize().getDataSize(); + onHeapMemstoreSize += store.getMemStoreSize().getHeapSize(); + offHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize(); + storeFileSize += store.getStorefilesSize(); + maxStoreFileCount = Math.max(maxStoreFileCount, store.getStorefilesCount()); + + maxStoreFileAge = + Math.max(store.getMaxStoreFileAge().orElse(maxStoreFileAge), maxStoreFileAge); + minStoreFileAge = + Math.min(store.getMinStoreFileAge().orElse(minStoreFileAge), minStoreFileAge); + + long storeHFiles = store.getNumHFiles(); + numHFiles += storeHFiles; + numReferenceFiles += store.getNumReferenceFiles(); + + OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge(); + if (storeAvgStoreFileAge.isPresent()) { + avgAgeNumerator = + (long) (avgAgeNumerator + storeAvgStoreFileAge.getAsDouble() * storeHFiles); + } + + storefileIndexSize += store.getStorefilesRootLevelIndexSize(); + totalStaticBloomSize += store.getTotalStaticBloomSize(); + totalStaticIndexSize += store.getTotalStaticIndexSize(); + bloomFilterRequestsCount += store.getBloomFilterRequestsCount(); + bloomFilterNegativeResultsCount += store.getBloomFilterNegativeResultsCount(); + bloomFilterEligibleRequestsCount += store.getBloomFilterEligibleRequestsCount(); + flushedCellsCount += store.getFlushedCellsCount(); + compactedCellsCount += store.getCompactedCellsCount(); + majorCompactedCellsCount += store.getMajorCompactedCellsCount(); + flushedCellsSize += store.getFlushedCellsSize(); + compactedCellsSize += store.getCompactedCellsSize(); + majorCompactedCellsSize += store.getMajorCompactedCellsSize(); + if (store instanceof HMobStore) { + HMobStore mobStore = (HMobStore) store; + cellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob(); + cellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob(); + cellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob(); + cellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob(); + mobFlushCount += mobStore.getMobFlushCount(); + mobFlushedCellsCount += mobStore.getMobFlushedCellsCount(); + mobFlushedCellsSize += mobStore.getMobFlushedCellsSize(); + mobScanCellsCount += mobStore.getMobScanCellsCount(); + mobScanCellsSize += mobStore.getMobScanCellsSize(); + } + } + + return new StoreFileStats(numHFiles, avgAgeNumerator); + } + + } + /** * This is the runnable that will be executed on the executor every PERIOD number of seconds It * will take metrics/numbers from all of the regions and use them to compute point in time @@ -790,170 +984,8 @@ public class RegionServerMetricsWrapperRunnable implements Runnable { @Override synchronized public void run() { try { - HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); - HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = - new HDFSBlocksDistribution(); - - long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0; - long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0, tempOffHeapMemstoreSize = 0; - long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0; - long tempMaxStoreFileCount = 0; - long avgAgeNumerator = 0, numHFiles = 0; - long tempMinStoreFileAge = Long.MAX_VALUE; - long tempFilteredReadRequestsCount = 0, tempCpRequestsCount = 0; - long tempCheckAndMutateChecksFailed = 0; - long tempCheckAndMutateChecksPassed = 0; - long tempStorefileIndexSize = 0; - long tempTotalStaticIndexSize = 0; - long tempTotalStaticBloomSize = 0; - long tempBloomFilterRequestsCount = 0; - long tempBloomFilterNegativeResultsCount = 0; - long tempBloomFilterEligibleRequestsCount = 0; - long tempNumMutationsWithoutWAL = 0; - long tempDataInMemoryWithoutWAL = 0; - double tempPercentFileLocal = 0; - double tempPercentFileLocalSecondaryRegions = 0; - long tempFlushedCellsCount = 0; - long tempCompactedCellsCount = 0; - long tempMajorCompactedCellsCount = 0; - long tempFlushedCellsSize = 0; - long tempCompactedCellsSize = 0; - long tempMajorCompactedCellsSize = 0; - long tempCellsCountCompactedToMob = 0; - long tempCellsCountCompactedFromMob = 0; - long tempCellsSizeCompactedToMob = 0; - long tempCellsSizeCompactedFromMob = 0; - long tempMobFlushCount = 0; - long tempMobFlushedCellsCount = 0; - long tempMobFlushedCellsSize = 0; - long tempMobScanCellsCount = 0; - long tempMobScanCellsSize = 0; - long tempBlockedRequestsCount = 0; - int regionCount = 0; - - long tempReadRequestsCount = 0; - long tempWriteRequestsCount = 0; - long currentReadRequestsCount = 0; - long currentWriteRequestsCount = 0; - long lastReadRequestsCount = 0; - long lastWriteRequestsCount = 0; - long readRequestsDelta = 0; - long writeRequestsDelta = 0; - long totalReadRequestsDelta = 0; - long totalWriteRequestsDelta = 0; - String encodedRegionName; - for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { - encodedRegionName = r.getRegionInfo().getEncodedName(); - currentReadRequestsCount = r.getReadRequestsCount(); - currentWriteRequestsCount = r.getWriteRequestsCount(); - if (requestsCountCache.containsKey(encodedRegionName)) { - lastReadRequestsCount = requestsCountCache.get(encodedRegionName).get(0); - lastWriteRequestsCount = requestsCountCache.get(encodedRegionName).get(1); - readRequestsDelta = currentReadRequestsCount - lastReadRequestsCount; - writeRequestsDelta = currentWriteRequestsCount - lastWriteRequestsCount; - totalReadRequestsDelta += readRequestsDelta; - totalWriteRequestsDelta += writeRequestsDelta; - // Update cache for our next comparision - requestsCountCache.get(encodedRegionName).set(0, currentReadRequestsCount); - requestsCountCache.get(encodedRegionName).set(1, currentWriteRequestsCount); - } else { - // List[0] -> readRequestCount - // List[1] -> writeRequestCount - ArrayList requests = new ArrayList(2); - requests.add(currentReadRequestsCount); - requests.add(currentWriteRequestsCount); - requestsCountCache.put(encodedRegionName, requests); - totalReadRequestsDelta += currentReadRequestsCount; - totalWriteRequestsDelta += currentWriteRequestsCount; - } - tempReadRequestsCount += r.getReadRequestsCount(); - tempWriteRequestsCount += r.getWriteRequestsCount(); - tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL(); - tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL(); - tempCpRequestsCount += r.getCpRequestsCount(); - tempFilteredReadRequestsCount += r.getFilteredReadRequestsCount(); - tempCheckAndMutateChecksFailed += r.getCheckAndMutateChecksFailed(); - tempCheckAndMutateChecksPassed += r.getCheckAndMutateChecksPassed(); - tempBlockedRequestsCount += r.getBlockedRequestsCount(); - List storeList = r.getStores(); - tempNumStores += storeList.size(); - for (Store store : storeList) { - tempNumStoreFiles += store.getStorefilesCount(); - tempMemstoreSize += store.getMemStoreSize().getDataSize(); - tempOnHeapMemstoreSize += store.getMemStoreSize().getHeapSize(); - tempOffHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize(); - tempStoreFileSize += store.getStorefilesSize(); - - tempMaxStoreFileCount = Math.max(tempMaxStoreFileCount, store.getStorefilesCount()); - - OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge(); - if ( - storeMaxStoreFileAge.isPresent() - && storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge - ) { - tempMaxStoreFileAge = storeMaxStoreFileAge.getAsLong(); - } - - OptionalLong storeMinStoreFileAge = store.getMinStoreFileAge(); - if ( - storeMinStoreFileAge.isPresent() - && storeMinStoreFileAge.getAsLong() < tempMinStoreFileAge - ) { - tempMinStoreFileAge = storeMinStoreFileAge.getAsLong(); - } - - long storeHFiles = store.getNumHFiles(); - numHFiles += storeHFiles; - tempNumReferenceFiles += store.getNumReferenceFiles(); - - OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge(); - if (storeAvgStoreFileAge.isPresent()) { - avgAgeNumerator = - (long) (avgAgeNumerator + storeAvgStoreFileAge.getAsDouble() * storeHFiles); - } - - tempStorefileIndexSize += store.getStorefilesRootLevelIndexSize(); - tempTotalStaticBloomSize += store.getTotalStaticBloomSize(); - tempTotalStaticIndexSize += store.getTotalStaticIndexSize(); - tempBloomFilterRequestsCount += store.getBloomFilterRequestsCount(); - tempBloomFilterNegativeResultsCount += store.getBloomFilterNegativeResultsCount(); - tempBloomFilterEligibleRequestsCount += store.getBloomFilterEligibleRequestsCount(); - tempFlushedCellsCount += store.getFlushedCellsCount(); - tempCompactedCellsCount += store.getCompactedCellsCount(); - tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount(); - tempFlushedCellsSize += store.getFlushedCellsSize(); - tempCompactedCellsSize += store.getCompactedCellsSize(); - tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize(); - if (store instanceof HMobStore) { - HMobStore mobStore = (HMobStore) store; - tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob(); - tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob(); - tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob(); - tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob(); - tempMobFlushCount += mobStore.getMobFlushCount(); - tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount(); - tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize(); - tempMobScanCellsCount += mobStore.getMobScanCellsCount(); - tempMobScanCellsSize += mobStore.getMobScanCellsSize(); - } - } - - HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution(); - hdfsBlocksDistribution.add(distro); - if (r.getRegionInfo().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { - hdfsBlocksDistributionSecondaryRegions.add(distro); - } - regionCount++; - } - - float localityIndex = - hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname()); - tempPercentFileLocal = Double.isNaN(tempBlockedRequestsCount) ? 0 : (localityIndex * 100); - - float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions - .getBlockLocalityIndex(regionServer.getServerName().getHostname()); - tempPercentFileLocalSecondaryRegions = - Double.isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100); + RegionMetricAggregate newVal = new RegionMetricAggregate(aggregate); + newVal.aggregate(regionServer, requestsCountCache); // Compute the number of requests per second long currentTime = EnvironmentEdgeManager.currentTime(); @@ -963,24 +995,14 @@ synchronized public void run() { if (lastRan == 0) { lastRan = currentTime - period; } - // If we've time traveled keep the last requests per second. - if ((currentTime - lastRan) > 0) { - requestsPerSecond = - (totalReadRequestsDelta + totalWriteRequestsDelta) / ((currentTime - lastRan) / 1000.0); - - double readRequestsRatePerMilliSecond = (double) totalReadRequestsDelta / period; - double writeRequestsRatePerMilliSecond = (double) totalWriteRequestsDelta / period; - - readRequestsRatePerSecond = readRequestsRatePerMilliSecond * 1000.0; - writeRequestsRatePerSecond = writeRequestsRatePerMilliSecond * 1000.0; - - long intervalStoreFileSize = tempStoreFileSize - lastStoreFileSize; - storeFileSizeGrowthRate = (double) intervalStoreFileSize * 1000.0 / period; - lastStoreFileSize = tempStoreFileSize; + long timeSinceLastRun = currentTime - lastRan; + // If we've time traveled keep the last requests per second. + if (timeSinceLastRun > 0) { + newVal.updateRates(timeSinceLastRun, period, lastStoreFileSize); } - lastRan = currentTime; + aggregate = newVal; List providers = regionServer.getWalFactory().getAllWALProviders(); for (WALProvider provider : providers) { @@ -988,58 +1010,6 @@ synchronized public void run() { walFileSize += provider.getLogFileSize(); } - // Copy over computed values so that no thread sees half computed values. - numStores = tempNumStores; - numStoreFiles = tempNumStoreFiles; - memstoreSize = tempMemstoreSize; - onHeapMemstoreSize = tempOnHeapMemstoreSize; - offHeapMemstoreSize = tempOffHeapMemstoreSize; - storeFileSize = tempStoreFileSize; - maxStoreFileCount = tempMaxStoreFileCount; - maxStoreFileAge = tempMaxStoreFileAge; - if (regionCount > 0) { - averageRegionSize = (memstoreSize + storeFileSize) / regionCount; - } - if (tempMinStoreFileAge != Long.MAX_VALUE) { - minStoreFileAge = tempMinStoreFileAge; - } - - if (numHFiles != 0) { - avgStoreFileAge = avgAgeNumerator / numHFiles; - } - - numReferenceFiles = tempNumReferenceFiles; - readRequestsCount = tempReadRequestsCount; - cpRequestsCount = tempCpRequestsCount; - filteredReadRequestsCount = tempFilteredReadRequestsCount; - writeRequestsCount = tempWriteRequestsCount; - checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed; - checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed; - storefileIndexSize = tempStorefileIndexSize; - totalStaticIndexSize = tempTotalStaticIndexSize; - totalStaticBloomSize = tempTotalStaticBloomSize; - bloomFilterRequestsCount = tempBloomFilterRequestsCount; - bloomFilterNegativeResultsCount = tempBloomFilterNegativeResultsCount; - bloomFilterEligibleRequestsCount = tempBloomFilterEligibleRequestsCount; - numMutationsWithoutWAL = tempNumMutationsWithoutWAL; - dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL; - percentFileLocal = tempPercentFileLocal; - percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions; - flushedCellsCount = tempFlushedCellsCount; - compactedCellsCount = tempCompactedCellsCount; - majorCompactedCellsCount = tempMajorCompactedCellsCount; - flushedCellsSize = tempFlushedCellsSize; - compactedCellsSize = tempCompactedCellsSize; - majorCompactedCellsSize = tempMajorCompactedCellsSize; - cellsCountCompactedToMob = tempCellsCountCompactedToMob; - cellsCountCompactedFromMob = tempCellsCountCompactedFromMob; - cellsSizeCompactedToMob = tempCellsSizeCompactedToMob; - cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob; - mobFlushCount = tempMobFlushCount; - mobFlushedCellsCount = tempMobFlushedCellsCount; - mobFlushedCellsSize = tempMobFlushedCellsSize; - mobScanCellsCount = tempMobScanCellsCount; - mobScanCellsSize = tempMobScanCellsSize; mobFileCacheAccessCount = mobFileCache != null ? mobFileCache.getAccessCount() : 0L; mobFileCacheMissCount = mobFileCache != null ? mobFileCache.getMissCount() : 0L; mobFileCacheHitRatio = mobFileCache != null ? mobFileCache.getHitRatio() : 0.0; @@ -1048,7 +1018,9 @@ synchronized public void run() { } mobFileCacheEvictedCount = mobFileCache != null ? mobFileCache.getEvictedFileCount() : 0L; mobFileCacheCount = mobFileCache != null ? mobFileCache.getCacheSize() : 0; - blockedRequestsCount = tempBlockedRequestsCount; + + lastStoreFileSize = aggregate.storeFileSize; + lastRan = currentTime; } catch (Throwable e) { LOG.warn("Caught exception! Will suppress and retry.", e); } @@ -1094,12 +1066,12 @@ public long getZeroCopyBytesRead() { @Override public long getBlockedRequestsCount() { - return blockedRequestsCount; + return aggregate.blockedRequestsCount; } @Override public long getAverageRegionSize() { - return averageRegionSize; + return aggregate.averageRegionSize; } @Override @@ -1226,4 +1198,9 @@ public long getByteBuffAllocatorTotalBufferCount() { public long getByteBuffAllocatorUsedBufferCount() { return this.allocator.getUsedBufferCount(); } + + // Visible for testing + long getPeriod() { + return period; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index c9595c12b5e9..b3e2e93fc9e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -26,6 +26,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.mutable.MutableLong; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -62,6 +63,8 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable private ScheduledFuture regionMetricsUpdateTask; + private float currentRegionCacheRatio; + public MetricsRegionWrapperImpl(HRegion region) { this.region = region; this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor(); @@ -121,6 +124,10 @@ public long getStoreFileSize() { return storeFileSize; } + public float getCurrentRegionCacheRatio() { + return currentRegionCacheRatio; + } + @Override public long getStoreRefCount() { return storeRefCount; @@ -315,7 +322,15 @@ public void run() { readsOnlyFromMemstore.put(store.getColumnFamilyName(), tempVal); } } - + MutableLong regionCachedAmount = new MutableLong(0); + region.getBlockCache().getRegionCachedInfo().ifPresent(regionCacheRatio -> regionCachedAmount + .addAndGet(regionCacheRatio.getOrDefault(region.getRegionInfo().getEncodedName(), 0L))); + if (tempStoreFileSize > 0) { + LOG.debug("Region {}, had cached {} bytes from a total of {}", + region.getRegionInfo().getEncodedName(), regionCachedAmount.getValue(), + tempStoreFileSize); + currentRegionCacheRatio = regionCachedAmount.floatValue() / tempStoreFileSize; + } numStoreFiles = tempNumStoreFiles; storeRefCount = tempStoreRefCount; maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 4f04457e91b6..4926aa30c8a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -336,8 +336,8 @@ public class RSRpcServices extends HBaseRpcServicesBase /** * Services launched in RSRpcServices. By default they are on but you can use the below booleans - * to selectively enable/disable either Admin or Client Service (Rare is the case where you would - * ever turn off one or the other). + * to selectively enable/disable these services (Rare is the case where you would ever turn off + * one or the other). */ public static final String REGIONSERVER_ADMIN_SERVICE_CONFIG = "hbase.regionserver.admin.executorService"; @@ -345,6 +345,8 @@ public class RSRpcServices extends HBaseRpcServicesBase "hbase.regionserver.client.executorService"; public static final String REGIONSERVER_CLIENT_META_SERVICE_CONFIG = "hbase.regionserver.client.meta.executorService"; + public static final String REGIONSERVER_BOOTSTRAP_NODES_SERVICE_CONFIG = + "hbase.regionserver.bootstrap.nodes.executorService"; /** * An Rpc callback for closing a RegionScanner. @@ -1449,6 +1451,8 @@ protected List getServices() { boolean client = getConfiguration().getBoolean(REGIONSERVER_CLIENT_SERVICE_CONFIG, true); boolean clientMeta = getConfiguration().getBoolean(REGIONSERVER_CLIENT_META_SERVICE_CONFIG, true); + boolean bootstrapNodes = + getConfiguration().getBoolean(REGIONSERVER_BOOTSTRAP_NODES_SERVICE_CONFIG, true); List bssi = new ArrayList<>(); if (client) { bssi.add(new BlockingServiceAndInterface(ClientService.newReflectiveBlockingService(this), @@ -1462,6 +1466,11 @@ protected List getServices() { bssi.add(new BlockingServiceAndInterface(ClientMetaService.newReflectiveBlockingService(this), ClientMetaService.BlockingInterface.class)); } + if (bootstrapNodes) { + bssi.add( + new BlockingServiceAndInterface(BootstrapNodeService.newReflectiveBlockingService(this), + BootstrapNodeService.BlockingInterface.class)); + } return new ImmutableList.Builder().addAll(bssi).build(); } @@ -1545,6 +1554,7 @@ public CompactRegionResponse compactRegion(final RpcController controller, } @Override + @QosPriority(priority = HConstants.ADMIN_QOS) public CompactionSwitchResponse compactionSwitch(RpcController controller, CompactionSwitchRequest request) throws ServiceException { rpcPreCheck("compactionSwitch"); @@ -2223,6 +2233,7 @@ public ReplicateWALEntryResponse replicateWALEntry(final RpcController controlle * @param request the request */ @Override + @QosPriority(priority = HConstants.ADMIN_QOS) public RollWALWriterResponse rollWALWriter(final RpcController controller, final RollWALWriterRequest request) throws ServiceException { try { @@ -2670,7 +2681,8 @@ public MultiResponse multi(final RpcController rpcc, final MultiRequest request) try { region = getRegion(regionSpecifier); - quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList()); + quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList(), + regionAction.hasCondition()); } catch (IOException e) { failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, e); return responseBuilder.build(); @@ -2732,7 +2744,8 @@ public MultiResponse multi(final RpcController rpcc, final MultiRequest request) try { region = getRegion(regionSpecifier); - quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList()); + quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList(), + regionAction.hasCondition()); } catch (IOException e) { failRegionAction(responseBuilder, regionActionResultBuilder, regionAction, cellScanner, e); continue; // For this region it's a failure. @@ -2915,7 +2928,8 @@ public MutateResponse mutate(final RpcController rpcc, final MutateRequest reque server.getMemStoreFlusher().reclaimMemStoreMemory(); } long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE; - quota = getRpcQuotaManager().checkQuota(region, OperationQuota.OperationType.MUTATE); + OperationQuota.OperationType operationType = QuotaUtil.getQuotaOperationType(request); + quota = getRpcQuotaManager().checkQuota(region, operationType); ActivePolicyEnforcement spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements(); @@ -3445,6 +3459,9 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan // from block size progress before writing into the response scannerContext.getMetrics().countOfBlockBytesScanned .set(scannerContext.getBlockSizeProgress()); + if (rpcCall != null) { + scannerContext.getMetrics().fsReadTime.set(rpcCall.getFsReadTime()); + } Map metrics = scannerContext.getMetrics().getMetricsMap(); ScanMetrics.Builder metricBuilder = ScanMetrics.newBuilder(); NameInt64Pair.Builder pairBuilder = NameInt64Pair.newBuilder(); @@ -3802,19 +3819,25 @@ public GetSpaceQuotaSnapshotsResponse getSpaceQuotaSnapshots(RpcController contr @Override public ClearRegionBlockCacheResponse clearRegionBlockCache(RpcController controller, ClearRegionBlockCacheRequest request) throws ServiceException { - rpcPreCheck("clearRegionBlockCache"); - ClearRegionBlockCacheResponse.Builder builder = ClearRegionBlockCacheResponse.newBuilder(); - CacheEvictionStatsBuilder stats = CacheEvictionStats.builder(); - List regions = getRegions(request.getRegionList(), stats); - for (HRegion region : regions) { - try { - stats = stats.append(this.server.clearRegionBlockCache(region)); - } catch (Exception e) { - stats.addException(region.getRegionInfo().getRegionName(), e); + try { + rpcPreCheck("clearRegionBlockCache"); + ClearRegionBlockCacheResponse.Builder builder = ClearRegionBlockCacheResponse.newBuilder(); + CacheEvictionStatsBuilder stats = CacheEvictionStats.builder(); + server.getRegionServerCoprocessorHost().preClearRegionBlockCache(); + List regions = getRegions(request.getRegionList(), stats); + for (HRegion region : regions) { + try { + stats = stats.append(this.server.clearRegionBlockCache(region)); + } catch (Exception e) { + stats.addException(region.getRegionInfo().getRegionName(), e); + } } + stats.withMaxCacheSize(server.getBlockCache().map(BlockCache::getMaxSize).orElse(0L)); + server.getRegionServerCoprocessorHost().postClearRegionBlockCache(stats.build()); + return builder.setStats(ProtobufUtil.toCacheEvictionStats(stats.build())).build(); + } catch (IOException e) { + throw new ServiceException(e); } - stats.withMaxCacheSize(server.getBlockCache().map(BlockCache::getMaxSize).orElse(0L)); - return builder.setStats(ProtobufUtil.toCacheEvictionStats(stats.build())).build(); } private void executeOpenRegionProcedures(OpenRegionRequest request, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java index 6a897a5b9f36..42069e58092e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java @@ -571,4 +571,10 @@ void requestCompaction(byte[] family, String why, int priority, boolean major, * if you try to set a configuration. */ Configuration getReadOnlyConfiguration(); + + /** + * The minimum block size configuration from all relevant column families. This is used when + * estimating quota consumption. + */ + int getMinBlockSizeBytes(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java index af1e923760d9..06eabdad67d4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CacheEvictionStats; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Mutation; @@ -240,6 +241,42 @@ public void call(RegionServerObserver observer) throws IOException { }); } + public void preUpdateConfiguration(Configuration preReloadConf) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() { + @Override + public void call(RegionServerObserver observer) throws IOException { + observer.preUpdateRegionServerConfiguration(this, preReloadConf); + } + }); + } + + public void postUpdateConfiguration(Configuration postReloadConf) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() { + @Override + public void call(RegionServerObserver observer) throws IOException { + observer.postUpdateRegionServerConfiguration(this, postReloadConf); + } + }); + } + + public void preClearRegionBlockCache() throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() { + @Override + public void call(RegionServerObserver observer) throws IOException { + observer.preClearRegionBlockCache(this); + } + }); + } + + public void postClearRegionBlockCache(CacheEvictionStats stats) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() { + @Override + public void call(RegionServerObserver observer) throws IOException { + observer.postClearRegionBlockCache(this, stats); + } + }); + } + /** * Coprocessor environment extension providing access to region server related services. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java index acf3231d4e90..ef25068512f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java @@ -259,8 +259,8 @@ public abstract class AbstractFSWAL implements WAL { protected final long blocksize; /* - * If more than this many logs, force flush of oldest region to oldest edit goes to disk. If too - * many and we crash, then will take forever replaying. Keep the number of logs tidy. + * If more than this many logs, force flush of oldest region to the oldest edit goes to disk. If + * too many and we crash, then will take forever replaying. Keep the number of logs tidy. */ protected final int maxLogs; @@ -336,7 +336,7 @@ private static final class WALProps { /** * The log file size. Notice that the size may not be accurate if we do asynchronous close in - * sub classes. + * subclasses. */ private final long logSize; @@ -346,7 +346,7 @@ private static final class WALProps { private final long rollTimeNs; /** - * If we do asynchronous close in sub classes, it is possible that when adding WALProps to the + * If we do asynchronous close in subclasses, it is possible that when adding WALProps to the * rolled map, the file is not closed yet, so in cleanOldLogs we should not archive this file, * for safety. */ @@ -404,9 +404,9 @@ private static final class WALProps { protected Supplier hasConsumerTask; private static final int MAX_EPOCH = 0x3FFFFFFF; - // the lowest bit is waitingRoll, which means new writer is created and we are waiting for old + // the lowest bit is waitingRoll, which means new writer is created, and we are waiting for old // writer to be closed. - // the second lowest bit is writerBroken which means the current writer is broken and rollWriter + // the second-lowest bit is writerBroken which means the current writer is broken and rollWriter // is needed. // all other bits are the epoch number of the current writer, this is used to detect whether the // writer is still the one when you issue the sync. @@ -807,7 +807,7 @@ public int getNumLogFiles() { * If the number of un-archived WAL files ('live' WALs) is greater than maximum allowed, check the * first (oldest) WAL, and return those regions which should be flushed so that it can be * let-go/'archived'. - * @return stores of regions (encodedRegionNames) to flush in order to archive oldest WAL file. + * @return stores of regions (encodedRegionNames) to flush in order to archive the oldest WAL file */ Map> findRegionsToForceFlush() throws IOException { Map> regions = null; @@ -861,7 +861,7 @@ private synchronized void cleanOldLogs() { long now = System.nanoTime(); boolean mayLogTooOld = nextLogTooOldNs <= now; ArrayList regionsBlockingWal = null; - // For each log file, look at its Map of regions to highest sequence id; if all sequence ids + // For each log file, look at its Map of regions to the highest sequence id; if all sequence ids // are older than what is currently in memory, the WAL can be GC'd. for (Map.Entry e : this.walFile2Props.entrySet()) { if (!e.getValue().closed) { @@ -1084,7 +1084,7 @@ private Map> rollWriterInternal(boolean force) throws IOExc try { Path oldPath = getOldPath(); Path newPath = getNewPath(); - // Any exception from here on is catastrophic, non-recoverable so we currently abort. + // Any exception from here on is catastrophic, non-recoverable, so we currently abort. W nextWriter = this.createWriterInstance(fs, newPath); if (remoteFs != null) { // create a remote wal if necessary @@ -1107,7 +1107,7 @@ private Map> rollWriterInternal(boolean force) throws IOExc regionsToFlush = findRegionsToForceFlush(); } } catch (CommonFSUtils.StreamLacksCapabilityException exception) { - // If the underlying FileSystem can't do what we ask, treat as IO failure so + // If the underlying FileSystem can't do what we ask, treat as IO failure, so // we'll abort. throw new IOException( "Underlying FileSystem can't meet stream requirements. See RS log " + "for details.", @@ -1195,9 +1195,9 @@ public Void call() throws Exception { throw new IOException(e.getCause()); } } finally { - // in shutdown we may call cleanOldLogs so shutdown this executor in the end. - // In sync replication implementation, we may shutdown a WAL without shutting down the whole - // region server, if we shutdown this executor earlier we may get reject execution exception + // in shutdown, we may call cleanOldLogs so shutdown this executor in the end. + // In sync replication implementation, we may shut down a WAL without shutting down the whole + // region server, if we shut down this executor earlier we may get reject execution exception // and abort the region server logArchiveExecutor.shutdown(); } @@ -1467,8 +1467,8 @@ private static int epoch(int epochAndState) { // return whether we have successfully set readyForRolling to true. private boolean trySetReadyForRolling() { // Check without holding lock first. Usually we will just return here. - // waitingRoll is volatile and unacedEntries is only accessed inside event loop so it is safe to - // check them outside the consumeLock. + // waitingRoll is volatile and unacedEntries is only accessed inside event loop, so it is safe + // to check them outside the consumeLock. if (!waitingRoll(epochAndState) || !unackedAppends.isEmpty()) { return false; } @@ -1532,13 +1532,13 @@ private void syncCompleted(long epochWhenSync, W writer, long processedTxid, lon // changed, i.e, we have already rolled the writer, or the writer is already broken, we should // just skip here, to avoid mess up the state or accidentally release some WAL entries and // cause data corruption. - // The syncCompleted call is on the critical write path so we should try our best to make it + // The syncCompleted call is on the critical write path, so we should try our best to make it // fast. So here we do not hold consumeLock, for increasing performance. It is safe because // there are only 3 possible situations: // 1. For normal case, the only place where we change epochAndState is when rolling the writer. // Before rolling actually happen, we will only change the state to waitingRoll which is another // bit than writerBroken, and when we actually change the epoch, we can make sure that there is - // no out going sync request. So we will always pass the check here and there is no problem. + // no outgoing sync request. So we will always pass the check here and there is no problem. // 2. The writer is broken, but we have not called syncFailed yet. In this case, since // syncFailed and syncCompleted are executed in the same thread, we will just face the same // situation with #1. @@ -1706,7 +1706,7 @@ private void appendAndSync() throws IOException { for (Iterator iter = toWriteAppends.iterator(); iter.hasNext();) { FSWALEntry entry = iter.next(); /** - * For {@link FSHog},here may throws IOException,but for {@link AsyncFSWAL}, here would not + * For {@link FSHog},here may throw IOException,but for {@link AsyncFSWAL}, here would not * throw any IOException. */ boolean appended = appendEntry(writer, entry); @@ -1753,7 +1753,7 @@ private void appendAndSync() throws IOException { } if (writer.getLength() == fileLengthAtLastSync) { // we haven't written anything out, just advance the highestSyncedSequence since we may only - // stamped some region sequence id. + // stamp some region sequence id. if (unackedAppends.isEmpty()) { highestSyncedTxid.set(highestProcessedAppendTxid); finishSync(); @@ -1761,7 +1761,7 @@ private void appendAndSync() throws IOException { } return; } - // reach here means that we have some unsynced data but haven't reached the batch size yet + // reach here means that we have some unsynced data but haven't reached the batch size yet, // but we will not issue a sync directly here even if there are sync requests because we may // have some new data in the ringbuffer, so let's just return here and delay the decision of // whether to issue a sync in the caller method. @@ -1876,12 +1876,12 @@ private boolean shouldScheduleConsumer() { * have its region edit/sequence id assigned else it messes up our unification of mvcc and * sequenceid. On return key will have the region edit/sequence id filled in. *

- * NOTE: This append, at a time that is usually after this call returns, starts an mvcc + * NOTE: This appends, at a time that is usually after this call returns, starts a mvcc * transaction by calling 'begin' wherein which we assign this update a sequenceid. At assignment * time, we stamp all the passed in Cells inside WALEdit with their sequenceId. You must * 'complete' the transaction this mvcc transaction by calling * MultiVersionConcurrencyControl#complete(...) or a variant otherwise mvcc will get stuck. Do it - * in the finally of a try/finally block within which this append lives and any subsequent + * in the finally of a try/finally block within which this appends lives and any subsequent * operations like sync or update of memstore, etc. Get the WriteEntry to pass mvcc out of the * passed in WALKey walKey parameter. Be warned that the WriteEntry is not * immediately available on return from this method. It WILL be available subsequent to a sync of @@ -2034,14 +2034,14 @@ protected final void closeWriter(W writer, Path path) { * Notice that you need to clear the {@link #rollRequested} flag in this method, as the new writer * will begin to work before returning from this method. If we clear the flag after returning from * this call, we may miss a roll request. The implementation class should choose a proper place to - * clear the {@link #rollRequested} flag so we do not miss a roll request, typically before you + * clear the {@link #rollRequested} flag, so we do not miss a roll request, typically before you * start writing to the new writer. */ protected void doReplaceWriter(Path oldPath, Path newPath, W nextWriter) throws IOException { Preconditions.checkNotNull(nextWriter); waitForSafePoint(); /** - * For {@link FSHLog},here would shutdown {@link FSHLog.SyncRunner}. + * For {@link FSHLog},here would shut down {@link FSHLog.SyncRunner}. */ doCleanUpResources(); // we will call rollWriter in init method, where we want to create the first writer and @@ -2084,7 +2084,7 @@ protected void doReplaceWriter(Path oldPath, Path newPath, W nextWriter) throws protected void doShutdown() throws IOException { waitForSafePoint(); /** - * For {@link FSHLog},here would shutdown {@link FSHLog.SyncRunner}. + * For {@link FSHLog},here would shut down {@link FSHLog.SyncRunner}. */ doCleanUpResources(); if (this.writer != null) { @@ -2214,7 +2214,7 @@ public void checkLogLowReplication(long checkInterval) { // So here we need to skip the creation of remote writer and make it possible to write the region // close marker. // Setting markerEdit only to true is for transiting from A to S, where we need to give up writing - // any pending wal entries as they will be discarded. The remote cluster will replicated the + // any pending wal entries as they will be discarded. The remote cluster will replicate the // correct data back later. We still need to allow writing marker edits such as close region event // to allow closing a region. @Override @@ -2245,6 +2245,10 @@ private static void split(final Configuration conf, final Path p) throws IOExcep WALSplitter.split(baseDir, p, archiveDir, fs, conf, WALFactory.getInstance(conf)); } + W getWriter() { + return this.writer; + } + private static void usage() { System.err.println("Usage: AbstractFSWAL "); System.err.println("Arguments:"); @@ -2257,7 +2261,7 @@ private static void usage() { } /** - * Pass one or more log file names and it will either dump out a text version on + * Pass one or more log file names, and it will either dump out a text version on * stdout or split the specified log files. */ public static void main(String[] args) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufWALReader.java index f5e65e08c84c..5d51750ba5f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufWALReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufWALReader.java @@ -121,7 +121,7 @@ public abstract class AbstractProtobufWALReader * Get or create the input stream used by cell decoder. *

* For implementing replication, we may need to limit the bytes we can read, so here we provide a - * method so sub classes can wrap the original input stream. + * method so subclasses can wrap the original input stream. */ protected abstract InputStream getCellCodecInputStream(FSDataInputStream stream); @@ -366,7 +366,7 @@ protected final void readTrailer(FSDataInputStream stream, FileStatus stat) thro this.fileLength = stat.getLen(); this.walEditsStopOffset = this.fileLength; long currentPos = stream.getPos(); - // we will reset walEditsStopOffset if trailer if available + // we will reset walEditsStopOffset if trailer is available trailerPresent = setTrailerIfPresent(stream); if (currentPos != stream.getPos()) { // seek back @@ -509,18 +509,18 @@ protected final IOException extractHiddenEof(Exception ex) { * This is used to determine whether we have already reached the WALTrailer. As the size and magic * are at the end of the WAL file, it is possible that these two options are missing while * writing, so we will consider there is no trailer. And when we actually reach the WALTrailer, we - * will try to decode it as WALKey and we will fail but the error could be vary as it is parsing + * will try to decode it as WALKey and we will fail but the error could be varied as it is parsing * WALTrailer actually. * @return whether this is a WALTrailer and we should throw EOF to upper layer the file is done */ protected final boolean isWALTrailer(long startPosition) throws IOException { - // We have nothing in the WALTrailer PB message now so its size is just a int length size and a + // We have nothing in the WALTrailer PB message now so its size is just an int length size and a // magic at the end int trailerSize = PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT; if (fileLength - startPosition >= trailerSize) { // We still have more than trailerSize bytes before reaching the EOF so this is not a trailer. // We also test for == here because if this is a valid trailer, we can read it while opening - // the reader so we should not reach here + // the reader, so we should not reach here return false; } inputStream.seek(startPosition); @@ -548,7 +548,7 @@ protected final boolean isWALTrailer(long startPosition) throws IOException { return false; } } - // in fact we should not reach here, as this means the trailer bytes are all matched and + // in fact, we should not reach here, as this means the trailer bytes are all matched and // complete, then we should not call this method... return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index d0d5ce5f2e17..131f284557af 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -603,14 +603,6 @@ DatanodeInfo[] getPipeline() { return new DatanodeInfo[0]; } - Writer getWriter() { - return this.writer; - } - - void setWriter(Writer writer) { - this.writer = writer; - } - @Override protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) { // put remote writer first as usually it will cost more time to finish, so we write to it first diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java index 91323a722150..30578a91909b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/HBasePolicyProvider.java @@ -25,7 +25,9 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.BootstrapNodeProtos.BootstrapNodeService; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos.LockService; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService; @@ -44,8 +46,11 @@ public class HBasePolicyProvider extends PolicyProvider { new Service("security.client.protocol.acl", RegistryProtos.ClientMetaService.BlockingInterface.class), new Service("security.admin.protocol.acl", MasterService.BlockingInterface.class), + new Service("security.admin.protocol.acl", LockService.BlockingInterface.class), new Service("security.masterregion.protocol.acl", - RegionServerStatusService.BlockingInterface.class) }; + RegionServerStatusService.BlockingInterface.class), + new Service("security.regionserver.protocol.acl", + BootstrapNodeService.BlockingInterface.class) }; @Override public Service[] getServices() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index 90a51e2cb03e..66a7b3a27032 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -2576,4 +2576,27 @@ public void preUpdateRSGroupConfig(final ObserverContext ctx) + throws IOException { + accessChecker.requirePermission(getActiveUser(ctx), "clearRegionBlockCache", null, + Permission.Action.ADMIN); + } + + @Override + public void preUpdateRegionServerConfiguration( + ObserverContext ctx, Configuration preReloadConf) + throws IOException { + accessChecker.requirePermission(getActiveUser(ctx), "updateConfiguration", null, + Permission.Action.ADMIN); + } + + @Override + public void preUpdateMasterConfiguration(ObserverContext ctx, + Configuration preReloadConf) throws IOException { + accessChecker.requirePermission(getActiveUser(ctx), "updateConfiguration", null, + Permission.Action.ADMIN); + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java index d0cd199ecdc9..92dca7c24c92 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java @@ -510,32 +510,53 @@ public Void call() { private Void readColumnFamily(Table table, ColumnFamilyDescriptor column) { byte[] startKey = null; - Scan scan = new Scan(); + Scan scan = null; ResultScanner rs = null; StopWatch stopWatch = new StopWatch(); startKey = region.getStartKey(); // Can't do a get on empty start row so do a Scan of first element if any instead. if (startKey.length > 0) { - // There are 4 types of region for any table. - // 1. Start and End key are empty. (Table with Single region) - // 2. Start key is empty. (First region of the table) - // 3. End key is empty. (Last region of the table) - // 4. Region with Start & End key. (All the regions between first & last region of the - // table.) + Get get = new Get(startKey); + get.setCacheBlocks(false); + get.setFilter(new FirstKeyOnlyFilter()); + get.addFamily(column.getName()); + // Converting get object to scan to enable RAW SCAN. + // This will work for all the regions of the HBase tables except first region of the table. + scan = new Scan(get); + scan.setRaw(rawScanEnabled); + } else { + scan = new Scan(); + // In case of first region of the HBase Table, we do not have start-key for the region. + // For Region Canary, we only need to scan a single row/cell in the region to make sure that + // region is accessible. + // + // When HBase table has more than 1 empty regions at start of the row-key space, Canary will + // create multiple scan object to find first available row in the table by scanning all the + // regions in sequence until it can find first available row. // - // Since Scan only takes Start and/or End Row and doesn't accept the region ID, - // we set the start row when Regions are of type 3 OR 4 as mentioned above. - // For type 1 and 2, We don't need to set this option. - scan.withStartRow(startKey); + // This could result in multiple millions of scans based on the size of table and number of + // empty regions in sequence. In test environment, A table with no data and 1100 empty + // regions, Single canary run was creating close to half million to 1 million scans to + // successfully do canary run for the table. + // + // Since First region of the table doesn't have any start key, We should set End Key as + // stop row and set inclusive=false to limit scan to single region only. + // + // TODO : In future, we can streamline Canary behaviour for all the regions by doing scan + // with startRow inclusive and stopRow exclusive instead of different behaviour for First + // Region of the table and rest of the region of the table. This way implementation is + // simplified. As of now this change has been kept minimal to avoid any unnecessary + // perf impact. + scan.withStopRow(region.getEndKey(), false); + LOG.debug("rawScan {} for {}", rawScanEnabled, region.getTable()); + scan.setRaw(rawScanEnabled); + scan.setCaching(1); + scan.setCacheBlocks(false); + scan.setFilter(new FirstKeyOnlyFilter()); + scan.addFamily(column.getName()); + scan.setMaxResultSize(1L); + scan.setOneRowLimit(); } - LOG.debug("rawScan {} for {}", rawScanEnabled, region.getTable()); - scan.setRaw(rawScanEnabled); - scan.setCaching(1); - scan.setCacheBlocks(false); - scan.setFilter(new FirstKeyOnlyFilter()); - scan.addFamily(column.getName()); - scan.setMaxResultSize(1L); - scan.setOneRowLimit(); LOG.debug("Reading from {} {} {} {}", region.getTable(), region.getRegionNameAsString(), column.getNameAsString(), Bytes.toStringBinary(startKey)); try { @@ -649,14 +670,16 @@ static class RegionServerTask implements Callable { private String serverName; private RegionInfo region; private RegionServerStdOutSink sink; + private Boolean rawScanEnabled; private AtomicLong successes; RegionServerTask(Connection connection, String serverName, RegionInfo region, - RegionServerStdOutSink sink, AtomicLong successes) { + RegionServerStdOutSink sink, Boolean rawScanEnabled, AtomicLong successes) { this.connection = connection; this.serverName = serverName; this.region = region; this.sink = sink; + this.rawScanEnabled = rawScanEnabled; this.successes = successes; } @@ -681,22 +704,35 @@ public Void call() { get = new Get(startKey); get.setCacheBlocks(false); get.setFilter(new FirstKeyOnlyFilter()); - stopWatch.start(); - table.get(get); - stopWatch.stop(); + // Converting get object to scan to enable RAW SCAN. + // This will work for all the regions of the HBase tables except first region. + scan = new Scan(get); + } else { scan = new Scan(); + // In case of first region of the HBase Table, we do not have start-key for the region. + // For Region Canary, we only need scan a single row/cell in the region to make sure that + // region is accessible. + // + // When HBase table has more than 1 empty regions at start of the row-key space, Canary + // will create multiple scan object to find first available row in the table by scanning + // all the regions in sequence until it can find first available row. + // + // Since First region of the table doesn't have any start key, We should set End Key as + // stop row and set inclusive=false to limit scan to first region only. + scan.withStopRow(region.getEndKey(), false); scan.setCacheBlocks(false); scan.setFilter(new FirstKeyOnlyFilter()); scan.setCaching(1); scan.setMaxResultSize(1L); scan.setOneRowLimit(); - stopWatch.start(); - ResultScanner s = table.getScanner(scan); - s.next(); - s.close(); - stopWatch.stop(); } + scan.setRaw(rawScanEnabled); + stopWatch.start(); + ResultScanner s = table.getScanner(scan); + s.next(); + s.close(); + stopWatch.stop(); successes.incrementAndGet(); sink.publishReadTiming(tableName.getNameAsString(), serverName, stopWatch.getTime()); } catch (TableNotFoundException tnfe) { @@ -1757,6 +1793,7 @@ private ZookeeperStdOutSink getSink() { * A monitor for regionserver mode */ private static class RegionServerMonitor extends Monitor { + private boolean rawScanEnabled; private boolean allRegions; public RegionServerMonitor(Connection connection, String[] monitorTargets, boolean useRegExp, @@ -1764,6 +1801,8 @@ public RegionServerMonitor(Connection connection, String[] monitorTargets, boole long allowedFailures) { super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError, allowedFailures); + Configuration conf = connection.getConfiguration(); + this.rawScanEnabled = conf.getBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, false); this.allRegions = allRegions; } @@ -1836,14 +1875,14 @@ private void monitorRegionServers(Map> rsAndRMap, } else if (this.allRegions) { for (RegionInfo region : entry.getValue()) { tasks.add(new RegionServerTask(this.connection, serverName, region, regionServerSink, - successes)); + this.rawScanEnabled, successes)); } } else { // random select a region if flag not set RegionInfo region = entry.getValue().get(ThreadLocalRandom.current().nextInt(entry.getValue().size())); - tasks.add( - new RegionServerTask(this.connection, serverName, region, regionServerSink, successes)); + tasks.add(new RegionServerTask(this.connection, serverName, region, regionServerSink, + this.rawScanEnabled, successes)); } } try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java index ede1f8b71508..7143598679be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MoveWithAck.java @@ -110,8 +110,9 @@ private static String getTimeDiffInSec(long startTime) { * Tries to scan a row from passed region */ private void isSuccessfulScan(RegionInfo region) throws IOException { - Scan scan = new Scan().withStartRow(region.getStartKey()).setRaw(true).setOneRowLimit() - .setMaxResultSize(1L).setCaching(1).setFilter(new FirstKeyOnlyFilter()).setCacheBlocks(false); + Scan scan = new Scan().withStartRow(region.getStartKey()).withStopRow(region.getEndKey(), false) + .setRaw(true).setOneRowLimit().setMaxResultSize(1L).setCaching(1) + .setFilter(new FirstKeyOnlyFilter()).setCacheBlocks(false); try (Table table = conn.getTable(region.getTable()); ResultScanner scanner = table.getScanner(scan)) { scanner.next(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java index e7080fc9323f..94e2e4bbfa08 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; @@ -145,6 +146,11 @@ public static void sanityCheck(final Configuration c, final TableDescriptor td) // check bloom filter type checkBloomFilterType(conf, td); + if (td.getErasureCodingPolicy() != null) { + warnOrThrowExceptionForFailure(logWarn, + () -> ErasureCodingUtils.verifySupport(conf, td.getErasureCodingPolicy())); + } + for (ColumnFamilyDescriptor hcd : td.getColumnFamilies()) { if (hcd.getTimeToLive() <= 0) { String message = "TTL for column family " + hcd.getNameAsString() + " must be positive."; @@ -184,19 +190,13 @@ public static void sanityCheck(final Configuration c, final TableDescriptor td) } // check in-memory compaction - try { - hcd.getInMemoryCompaction(); - } catch (IllegalArgumentException e) { - warnOrThrowExceptionForFailure(logWarn, e.getMessage(), e); - } + warnOrThrowExceptionForFailure(logWarn, hcd::getInMemoryCompaction); } } private static void checkReplicationScope(final Configuration conf, final TableDescriptor td) throws IOException { - // Setting logs to warning instead of throwing exception if sanityChecks are disabled - boolean logWarn = !shouldSanityCheck(conf); - try { + warnOrThrowExceptionForFailure(conf, () -> { for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) { // check replication scope WALProtos.ScopeType scop = WALProtos.ScopeType.valueOf(cfd.getScope()); @@ -207,15 +207,12 @@ private static void checkReplicationScope(final Configuration conf, final TableD throw new DoNotRetryIOException(message); } } - } catch (IOException e) { - warnOrThrowExceptionForFailure(logWarn, e.getMessage(), e); - } - + }); } private static void checkCompactionPolicy(final Configuration conf, final TableDescriptor td) throws IOException { - try { + warnOrThrowExceptionForFailure(false, () -> { // FIFO compaction has some requirements // Actually FCP ignores periodic major compactions String className = td.getValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY); @@ -268,16 +265,12 @@ private static void checkCompactionPolicy(final Configuration conf, final TableD throw new IOException(message); } } - } catch (IOException e) { - warnOrThrowExceptionForFailure(false, e.getMessage(), e); - } + }); } private static void checkBloomFilterType(final Configuration conf, final TableDescriptor td) throws IOException { - // Setting logs to warning instead of throwing exception if sanityChecks are disabled - boolean logWarn = !shouldSanityCheck(conf); - try { + warnOrThrowExceptionForFailure(conf, () -> { for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) { Configuration cfdConf = new CompoundConfiguration().addStringMap(cfd.getConfiguration()); try { @@ -286,50 +279,36 @@ private static void checkBloomFilterType(final Configuration conf, final TableDe throw new DoNotRetryIOException("Failed to get bloom filter param", e); } } - } catch (IOException e) { - warnOrThrowExceptionForFailure(logWarn, e.getMessage(), e); - } + }); } public static void checkCompression(final Configuration conf, final TableDescriptor td) throws IOException { - // Setting logs to warning instead of throwing exception if sanityChecks are disabled - boolean logWarn = !shouldSanityCheck(conf); - try { + warnOrThrowExceptionForFailure(conf, () -> { for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) { CompressionTest.testCompression(cfd.getCompressionType()); CompressionTest.testCompression(cfd.getCompactionCompressionType()); CompressionTest.testCompression(cfd.getMajorCompactionCompressionType()); CompressionTest.testCompression(cfd.getMinorCompactionCompressionType()); } - } catch (IOException e) { - warnOrThrowExceptionForFailure(logWarn, e.getMessage(), e); - } + }); } public static void checkEncryption(final Configuration conf, final TableDescriptor td) throws IOException { - // Setting logs to warning instead of throwing exception if sanityChecks are disabled - boolean logWarn = !shouldSanityCheck(conf); - try { + warnOrThrowExceptionForFailure(conf, () -> { for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) { EncryptionTest.testEncryption(conf, cfd.getEncryptionType(), cfd.getEncryptionKey()); } - } catch (IOException e) { - warnOrThrowExceptionForFailure(logWarn, e.getMessage(), e); - } + }); } public static void checkClassLoading(final Configuration conf, final TableDescriptor td) throws IOException { - // Setting logs to warning instead of throwing exception if sanityChecks are disabled - boolean logWarn = !shouldSanityCheck(conf); - try { + warnOrThrowExceptionForFailure(conf, () -> { RegionSplitPolicy.getSplitPolicyClass(td, conf); RegionCoprocessorHost.testTableCoprocessorAttrs(conf, td); - } catch (Exception e) { - warnOrThrowExceptionForFailure(logWarn, e.getMessage(), e); - } + }); } // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled. @@ -341,4 +320,24 @@ private static void warnOrThrowExceptionForFailure(boolean logWarn, String messa } LOG.warn(message); } + + private static void warnOrThrowExceptionForFailure(Configuration conf, ThrowingRunnable runnable) + throws IOException { + boolean logWarn = !shouldSanityCheck(conf); + warnOrThrowExceptionForFailure(logWarn, runnable); + } + + private static void warnOrThrowExceptionForFailure(boolean logWarn, ThrowingRunnable runnable) + throws IOException { + try { + runnable.run(); + } catch (Exception e) { + warnOrThrowExceptionForFailure(logWarn, e.getMessage(), e); + } + } + + @FunctionalInterface + interface ThrowingRunnable { + void run() throws Exception; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java index d001ea755b74..cbfde9c7e172 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java @@ -50,9 +50,9 @@ import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; -import org.apache.hadoop.hbase.util.ConcurrentMapUtils.IOExceptionSupplier; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.IOExceptionSupplier; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; import org.apache.yetus.audience.InterfaceAudience; diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 1d48a7561e1b..f0599b7aa64a 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -315,6 +315,7 @@ Region Server ReadRequests WriteRequests + Uncompressed StoreFileSize StorefileSize Num.Storefiles MemSize @@ -338,6 +339,7 @@ String hostAndPort = ""; String readReq = "N/A"; String writeReq = "N/A"; + String fileSizeUncompressed = ZEROMB; String fileSize = ZEROMB; String fileCount = "N/A"; String memSize = ZEROMB; @@ -356,6 +358,10 @@ if (rSize > 0) { fileSize = StringUtils.byteDesc((long) rSize); } + double rSizeUncompressed = load.getUncompressedStoreFileSize().get(Size.Unit.BYTE); + if (rSizeUncompressed > 0) { + fileSizeUncompressed = StringUtils.byteDesc((long) rSizeUncompressed); + } fileCount = String.format("%,1d", load.getStoreFileCount()); double mSize = load.getMemStoreSize().get(Size.Unit.BYTE); if (mSize > 0) { @@ -370,6 +376,7 @@ <%= StringEscapeUtils.escapeHtml4(hostAndPort) %> <%= readReq%> <%= writeReq%> + <%= fileSizeUncompressed%> <%= fileSize%> <%= fileCount%> <%= memSize%> @@ -834,6 +841,7 @@ <% long totalReadReq = 0; long totalWriteReq = 0; + long totalSizeUncompressed = 0; long totalSize = 0; long totalStoreFileCount = 0; long totalMemSize = 0; @@ -844,6 +852,7 @@ long totalBlocksLocalWithSsdWeight = 0; String totalCompactionProgress = ""; String totalMemSizeStr = ZEROMB; + String totalSizeUncompressedStr = ZEROMB; String totalSizeStr = ZEROMB; String totalLocality = ""; String totalLocalityForSsd = ""; @@ -865,6 +874,7 @@ if (regionMetrics != null) { totalReadReq += regionMetrics.getReadRequestCount(); totalWriteReq += regionMetrics.getWriteRequestCount(); + totalSizeUncompressed += regionMetrics.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); totalSize += regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE); totalStoreFileCount += regionMetrics.getStoreFileCount(); totalMemSize += regionMetrics.getMemStoreSize().get(Size.Unit.MEGABYTE); @@ -890,6 +900,9 @@ if (totalSize > 0) { totalSizeStr = StringUtils.byteDesc(totalSize*1024l*1024); } + if (totalSizeUncompressed > 0){ + totalSizeUncompressedStr = StringUtils.byteDesc(totalSizeUncompressed*1024l*1024); + } if (totalMemSize > 0) { totalMemSizeStr = StringUtils.byteDesc(totalMemSize*1024l*1024); } @@ -920,6 +933,7 @@ Region Server ReadRequests
(<%= String.format("%,1d", totalReadReq)%>) WriteRequests
(<%= String.format("%,1d", totalWriteReq)%>) + Uncompressed StoreFileSize
(<%= totalSizeUncompressedStr %>) StorefileSize
(<%= totalSizeStr %>) Num.Storefiles
(<%= String.format("%,1d", totalStoreFileCount)%>) MemSize
(<%= totalMemSizeStr %>) @@ -944,6 +958,7 @@ RegionMetrics load = hriEntry.getValue(); String readReq = "N/A"; String writeReq = "N/A"; + String regionSizeUncompressed = ZEROMB; String regionSize = ZEROMB; String fileCount = "N/A"; String memSize = ZEROMB; @@ -951,6 +966,10 @@ if (load != null) { readReq = String.format("%,1d", load.getReadRequestCount()); writeReq = String.format("%,1d", load.getWriteRequestCount()); + double rSizeUncompressed = load.getUncompressedStoreFileSize().get(Size.Unit.BYTE); + if (rSizeUncompressed > 0) { + regionSizeUncompressed = StringUtils.byteDesc((long)rSizeUncompressed); + } double rSize = load.getStoreFileSize().get(Size.Unit.BYTE); if (rSize > 0) { regionSize = StringUtils.byteDesc((long)rSize); @@ -987,6 +1006,7 @@ <%= buildRegionDeployedServerTag(regionInfo, master, regionsToServer) %> <%= readReq%> <%= writeReq%> + <%= regionSizeUncompressed%> <%= regionSize%> <%= fileCount%> <%= memSize%> diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java index b80cd207683c..6ac87ce03026 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -195,6 +196,18 @@ public void testRowsSeenMetric() throws Exception { } } + @Test + public void testFsReadTimeMetric() throws Exception { + // write some new puts and flush, as an easy way to ensure the read blocks are not cached + // so that we go into the fs write code path + List puts = createPuts(ROWS, FAMILIES, QUALIFIERS, VALUE); + TABLE.put(puts); + TEST_UTIL.flush(TABLE_NAME); + Scan scan = new Scan(); + scan.setScanMetricsEnabled(true); + testMetric(scan, ServerSideScanMetrics.FS_READ_TIME_METRIC_NAME, 0, CompareOperator.GREATER); + } + private void testRowsSeenMetric(Scan baseScan) throws Exception { Scan scan; scan = new Scan(baseScan); @@ -333,6 +346,11 @@ private void testRowsFilteredMetric(Scan baseScan, Filter filter, int expectedNu * @throws Exception on unexpected failure */ private void testMetric(Scan scan, String metricKey, long expectedValue) throws Exception { + testMetric(scan, metricKey, expectedValue, CompareOperator.EQUAL); + } + + private void testMetric(Scan scan, String metricKey, long expectedValue, + CompareOperator compareOperator) throws Exception { assertTrue("Scan should be configured to record metrics", scan.isScanMetricsEnabled()); ResultScanner scanner = TABLE.getScanner(scan); // Iterate through all the results @@ -341,11 +359,17 @@ private void testMetric(Scan scan, String metricKey, long expectedValue) throws } scanner.close(); ScanMetrics metrics = scanner.getScanMetrics(); - assertTrue("Metrics are null", metrics != null); + assertNotNull("Metrics are null", metrics); assertTrue("Metric : " + metricKey + " does not exist", metrics.hasCounter(metricKey)); final long actualMetricValue = metrics.getCounter(metricKey).get(); - assertEquals( - "Metric: " + metricKey + " Expected: " + expectedValue + " Actual: " + actualMetricValue, - expectedValue, actualMetricValue); + if (compareOperator == CompareOperator.EQUAL) { + assertEquals( + "Metric: " + metricKey + " Expected: " + expectedValue + " Actual: " + actualMetricValue, + expectedValue, actualMetricValue); + } else { + assertTrue( + "Metric: " + metricKey + " Expected: > " + expectedValue + " Actual: " + actualMetricValue, + actualMetricValue > expectedValue); + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java index a94c214a3250..509d74e0335c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java @@ -40,7 +40,8 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.DummyConnectionRegistry; +import org.apache.hadoop.hbase.client.ConnectionRegistry; +import org.apache.hadoop.hbase.client.DoNothingConnectionRegistry; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; import org.apache.hadoop.hbase.master.cleaner.DirScanPool; @@ -49,6 +50,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; @@ -92,9 +94,10 @@ public class TestZooKeeperTableArchiveClient { private static RegionServerServices rss; private static DirScanPool POOL; - public static final class MockRegistry extends DummyConnectionRegistry { + public static final class MockRegistry extends DoNothingConnectionRegistry { - public MockRegistry(Configuration conf) { + public MockRegistry(Configuration conf, User user) { + super(conf, user); } @Override @@ -110,8 +113,8 @@ public CompletableFuture getClusterId() { public static void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); UTIL.startMiniZKCluster(); - UTIL.getConfiguration().setClass(MockRegistry.REGISTRY_IMPL_CONF_KEY, MockRegistry.class, - DummyConnectionRegistry.class); + UTIL.getConfiguration().setClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, + MockRegistry.class, ConnectionRegistry.class); CONNECTION = ConnectionFactory.createConnection(UTIL.getConfiguration()); archivingClient = new ZKTableArchiveClient(UTIL.getConfiguration(), CONNECTION); // make hfile archiving node so we can archive files diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java index 8b38db974d7c..0ff105743e0c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/AbstractTestRegionLocator.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.junit.After; @@ -59,7 +60,7 @@ protected static void startClusterAndCreateTable() throws Exception { UTIL.getAdmin().createTable(td, SPLIT_KEYS); UTIL.waitTableAvailable(TABLE_NAME); try (ConnectionRegistry registry = - ConnectionRegistryFactory.getRegistry(UTIL.getConfiguration())) { + ConnectionRegistryFactory.getRegistry(UTIL.getConfiguration(), User.getCurrent())) { RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(UTIL, registry); } UTIL.getAdmin().balancerSwitch(false, true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/DummyConnectionRegistry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/DummyConnectionRegistry.java deleted file mode 100644 index cc2e9493d039..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/DummyConnectionRegistry.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import java.util.concurrent.CompletableFuture; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.RegionLocations; -import org.apache.hadoop.hbase.ServerName; - -/** - * Can be overridden in UT if you only want to implement part of the methods in - * {@link ConnectionRegistry}. - */ -public class DummyConnectionRegistry implements ConnectionRegistry { - - public static final String REGISTRY_IMPL_CONF_KEY = - HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY; - - @Override - public CompletableFuture getMetaRegionLocations() { - return null; - } - - @Override - public CompletableFuture getClusterId() { - return null; - } - - @Override - public CompletableFuture getActiveMaster() { - return null; - } - - @Override - public String getConnectionString() { - return null; - } - - @Override - public void close() { - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminWithRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminWithRegionReplicas.java index 4dd4d4550777..da400f29c0c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminWithRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminWithRegionReplicas.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -55,7 +56,7 @@ public static void setUpBeforeClass() throws Exception { TestAsyncAdminBase.setUpBeforeClass(); HBaseTestingUtil.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3); try (ConnectionRegistry registry = - ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration())) { + ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent())) { RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, registry); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java index ad9bf551c033..90d2cb51e8cf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.RegionReplicaTestHelper.Locator; import org.apache.hadoop.hbase.client.trace.StringTraceRenderer; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.trace.OpenTelemetryClassRule; @@ -106,7 +107,8 @@ protected void before() throws Throwable { testUtil = miniClusterRule.getTestingUtility(); HBaseTestingUtil.setReplicas(admin, TableName.META_TABLE_NAME, 3); testUtil.waitUntilNoRegionsInTransition(); - registry = ConnectionRegistryFactory.getRegistry(testUtil.getConfiguration()); + registry = + ConnectionRegistryFactory.getRegistry(testUtil.getConfiguration(), User.getCurrent()); RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(testUtil, registry); admin.balancerSwitch(false).get(); locator = new AsyncMetaRegionLocator(registry); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java index f3b231d5bde8..a6d0ab81f912 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java @@ -128,7 +128,7 @@ public void setUpBeforeTest() throws InterruptedException, ExecutionException, I // Enable meta replica LoadBalance mode for this connection. c.set(RegionLocator.LOCATOR_META_REPLICAS_MODE, metaReplicaMode.toString()); ConnectionRegistry registry = - ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); + ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent()); conn = new AsyncConnectionImpl(c, registry, registry.getClusterId().get(), null, User.getCurrent()); locator = new AsyncNonMetaRegionLocator(conn); @@ -147,7 +147,7 @@ public void tearDownAfterTest() throws IOException { } @Parameterized.Parameters - public static Collection parameters() { + public static Collection paramAbstractTestRegionLocatoreters() { return Arrays .asList(new Object[][] { { CatalogReplicaMode.NONE }, { CatalogReplicaMode.LOAD_BALANCE } }); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java index 2dd08b36b30d..50c9ab9f5657 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java @@ -125,7 +125,7 @@ public static void setUp() throws Exception { TEST_UTIL.startMiniCluster(3); TEST_UTIL.getAdmin().balancerSwitch(false, true); ConnectionRegistry registry = - ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); + ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent()); CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry, registry.getClusterId().get(), null, User.getCurrent()); LOCATOR = new AsyncNonMetaRegionLocator(CONN); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocator.java index ee0963e1f8b3..bacd7bb32d70 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocator.java @@ -100,7 +100,7 @@ public static void setUp() throws Exception { TEST_UTIL.createTable(TABLE_NAME, FAMILY); TEST_UTIL.waitTableAvailable(TABLE_NAME); ConnectionRegistry registry = - ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); + ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent()); CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry, registry.getClusterId().get(), null, User.getCurrent()); LOCATOR = CONN.getLocator(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java index ab43ec545d93..3c8327145f32 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java @@ -73,7 +73,7 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.createTable(TABLE_NAME, FAMILY); TEST_UTIL.waitTableAvailable(TABLE_NAME); ConnectionRegistry registry = - ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); + ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent()); CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry, registry.getClusterId().get(), null, User.getCurrent()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableUseMetaReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableUseMetaReplicas.java index 1a7ac8819e49..0de59a4c32bf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableUseMetaReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableUseMetaReplicas.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -94,7 +95,8 @@ public static void setUp() throws Exception { FailPrimaryMetaScanCp.class.getName()); UTIL.startMiniCluster(3); HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 3); - try (ConnectionRegistry registry = ConnectionRegistryFactory.getRegistry(conf)) { + try (ConnectionRegistry registry = + ConnectionRegistryFactory.getRegistry(conf, User.getCurrent())) { RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(UTIL, registry); } try (Table table = UTIL.createTable(TABLE_NAME, FAMILY)) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBootstrapNodeUpdate.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBootstrapNodeUpdate.java new file mode 100644 index 000000000000..549575f4f404 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBootstrapNodeUpdate.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseRpcServicesBase; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.regionserver.BootstrapNodeManager; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.io.Closeables; + +/** + * Make sure that we can update the bootstrap server from master to region server, and region server + * could also contact each other to update the bootstrap nodes. + */ +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestBootstrapNodeUpdate { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBootstrapNodeUpdate.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static RpcConnectionRegistry REGISTRY; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setLong(BootstrapNodeManager.REQUEST_MASTER_INTERVAL_SECS, 5); + conf.setLong(BootstrapNodeManager.REQUEST_MASTER_MIN_INTERVAL_SECS, 1); + conf.setLong(BootstrapNodeManager.REQUEST_REGIONSERVER_INTERVAL_SECS, 1); + conf.setInt(HBaseRpcServicesBase.CLIENT_BOOTSTRAP_NODE_LIMIT, 2); + conf.setLong(RpcConnectionRegistry.INITIAL_REFRESH_DELAY_SECS, 5); + conf.setLong(RpcConnectionRegistry.PERIODIC_REFRESH_INTERVAL_SECS, 1); + conf.setLong(RpcConnectionRegistry.MIN_SECS_BETWEEN_REFRESHES, 1); + UTIL.startMiniCluster(3); + REGISTRY = new RpcConnectionRegistry(conf, UserProvider.instantiate(conf).getCurrent()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + Closeables.close(REGISTRY, true); + UTIL.shutdownMiniCluster(); + } + + @Test + public void testUpdate() throws Exception { + ServerName activeMasterServerName = REGISTRY.getActiveMaster().get(); + ServerName masterInConf = ServerName.valueOf(activeMasterServerName.getHostname(), + activeMasterServerName.getPort(), -1); + // we should have master in the beginning + assertThat(REGISTRY.getParsedServers(), hasItem(masterInConf)); + // and after refreshing, we will switch to use region servers + UTIL.waitFor(15000, () -> !REGISTRY.getParsedServers().contains(masterInConf) + && !REGISTRY.getParsedServers().contains(activeMasterServerName)); + Set parsedServers = REGISTRY.getParsedServers(); + assertEquals(2, parsedServers.size()); + // now kill one region server + ServerName serverToKill = parsedServers.iterator().next(); + UTIL.getMiniHBaseCluster().killRegionServer(serverToKill); + // wait until the region server disappears + // since the min node limit is 2, this means region server will still contact each other for + // getting bootstrap nodes, instead of requesting master directly, so this assert can make sure + // that the getAllBootstrapNodes works fine, and also the client can communicate with region + // server to update bootstrap nodes + UTIL.waitFor(30000, () -> !REGISTRY.getParsedServers().contains(serverToKill)); + // should still have 2 servers, the remaining 2 live region servers + assertEquals(2, parsedServers.size()); + // make sure the registry still works fine + assertNotNull(REGISTRY.getClusterId().get()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCatalogReplicaLoadBalanceSimpleSelector.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCatalogReplicaLoadBalanceSimpleSelector.java index 0e2feae841cf..5c78e53f7e60 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCatalogReplicaLoadBalanceSimpleSelector.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCatalogReplicaLoadBalanceSimpleSelector.java @@ -77,7 +77,8 @@ public static void setUp() throws Exception { () -> TEST_UTIL.getMiniHBaseCluster().getRegions(TableName.META_TABLE_NAME).size() >= numOfMetaReplica); - registry = ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); + registry = + ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent()); CONN = new AsyncConnectionImpl(conf, registry, registry.getClusterId().get(), null, User.getCurrent()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java index d358695c5f9b..9a92f4b1aa55 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java @@ -17,9 +17,10 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.net.SocketAddress; import java.net.SocketTimeoutException; import java.util.Map; @@ -31,7 +32,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.exceptions.MasterRegistryFetchException; import org.apache.hadoop.hbase.ipc.AbstractRpcClient; import org.apache.hadoop.hbase.ipc.BlockingRpcClient; import org.apache.hadoop.hbase.ipc.HBaseRpcController; @@ -67,9 +67,6 @@ public class TestClientTimeouts { @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(SLAVES); - // Set the custom RPC client with random timeouts as the client - TEST_UTIL.getConfiguration().set(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY, - RandomTimeoutRpcClient.class.getName()); } @AfterClass @@ -77,51 +74,39 @@ public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); } + private Connection createConnection() { + // Ensure the HBaseAdmin uses a new connection by changing Configuration. + Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); + // Set the custom RPC client with random timeouts as the client + conf.set(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY, + RandomTimeoutRpcClient.class.getName()); + conf.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1)); + for (;;) { + try { + return ConnectionFactory.createConnection(conf); + } catch (IOException e) { + // since we randomly throw SocketTimeoutException, it is possible that we fail when creating + // the Connection, but this is not what we want to test here, so just ignore it and try + // again + } + } + } + /** * Test that a client that fails an RPC to the master retries properly and doesn't throw any * unexpected exceptions. */ @Test public void testAdminTimeout() throws Exception { - boolean lastFailed = false; - int initialInvocations = invokations.get(); - RandomTimeoutRpcClient rpcClient = (RandomTimeoutRpcClient) RpcClientFactory - .createClient(TEST_UTIL.getConfiguration(), TEST_UTIL.getClusterKey()); - - try { - for (int i = 0; i < 5 || (lastFailed && i < 100); ++i) { - lastFailed = false; - // Ensure the HBaseAdmin uses a new connection by changing Configuration. - Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); - conf.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1)); - Admin admin = null; - Connection connection = null; - try { - connection = ConnectionFactory.createConnection(conf); - admin = connection.getAdmin(); - admin.balancerSwitch(false, false); - } catch (MasterRegistryFetchException ex) { - // Since we are randomly throwing SocketTimeoutExceptions, it is possible to get - // a MasterRegistryFetchException. It's a bug if we get other exceptions. - lastFailed = true; - } finally { - if (admin != null) { - admin.close(); - if (admin.getConnection().isClosed()) { - rpcClient = (RandomTimeoutRpcClient) RpcClientFactory - .createClient(TEST_UTIL.getConfiguration(), TEST_UTIL.getClusterKey()); - } - } - if (connection != null) { - connection.close(); - } - } + try (Connection conn = createConnection(); Admin admin = conn.getAdmin()) { + int initialInvocations = invokations.get(); + boolean balanceEnabled = admin.isBalancerEnabled(); + for (int i = 0; i < 5; i++) { + assertEquals(balanceEnabled, admin.balancerSwitch(!balanceEnabled, false)); + balanceEnabled = !balanceEnabled; } // Ensure the RandomTimeoutRpcEngine is actually being used. - assertFalse(lastFailed); assertTrue(invokations.get() > initialInvocations); - } finally { - rpcClient.close(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java index 4756a5b22304..6659b6164f01 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java @@ -19,10 +19,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -30,9 +34,11 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.io.IOUtils; import org.junit.After; @@ -43,6 +49,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.ExpectedException; import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +73,9 @@ public class TestFlushFromClient { @Rule public TestName name = new TestName(); + @Rule + public ExpectedException exception = ExpectedException.none(); + public TableName tableName; @BeforeClass @@ -184,6 +194,32 @@ public void testAsyncFlushRegionFamily() throws Exception { } } + @Test + public void testAsyncFlushTableWithNonExistingFamilies() throws IOException { + AsyncAdmin admin = asyncConn.getAdmin(); + List families = new ArrayList<>(); + families.add(FAMILY_1); + families.add(FAMILY_2); + families.add(Bytes.toBytes("non_family01")); + families.add(Bytes.toBytes("non_family02")); + CompletableFuture future = CompletableFuture.allOf(admin.flush(tableName, families)); + exception.expect(NoSuchColumnFamilyException.class); + FutureUtils.get(future); + } + + @Test + public void testAsyncFlushRegionWithNonExistingFamily() throws IOException { + AsyncAdmin admin = asyncConn.getAdmin(); + List regions = getRegionInfo(); + assertNotNull(regions); + assertTrue(regions.size() > 0); + HRegion region = regions.get(0); + CompletableFuture future = CompletableFuture.allOf( + admin.flushRegion(region.getRegionInfo().getRegionName(), Bytes.toBytes("non_family"))); + exception.expect(NoSuchColumnFamilyException.class); + FutureUtils.get(future); + } + @Test public void testFlushRegionServer() throws Exception { try (Admin admin = TEST_UTIL.getAdmin()) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java index 406b25fed4e0..360641e64b7d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHbck.java @@ -250,11 +250,20 @@ public void testAssigns() throws Exception { for (long pid : pids) { assertEquals(Procedure.NO_PROC_ID, pid); } - // If we pass override, then we should be able to unassign EVEN THOUGH Regions already + // Rerun the unassign with override. Should fail for all Regions since they already + // unassigned; failed + // unassign will manifest as all pids being -1 (ever since HBASE-24885). + pids = hbck.unassigns( + regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.toList()), true, false); + waitOnPids(pids); + for (long pid : pids) { + assertEquals(Procedure.NO_PROC_ID, pid); + } + // If we pass force, then we should be able to unassign EVEN THOUGH Regions already // unassigned.... makes for a mess but operator might want to do this at an extreme when // doing fixup of broke cluster. pids = hbck.unassigns( - regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.toList()), true); + regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.toList()), true, true); waitOnPids(pids); for (long pid : pids) { assertNotEquals(Procedure.NO_PROC_ID, pid); @@ -283,6 +292,12 @@ public void testAssigns() throws Exception { LOG.info("RS: {}", rs.toString()); assertTrue(rs.toString(), rs.isOpened()); } + // Rerun the assign with override. Should fail for all Regions since they already assigned + pids = hbck.assigns( + regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.toList()), true, false); + for (long pid : pids) { + assertEquals(Procedure.NO_PROC_ID, pid); + } // What happens if crappy region list passed? pids = hbck.assigns( Arrays.stream(new String[] { "a", "some rubbish name" }).collect(Collectors.toList())); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMasterRegistry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMasterRegistry.java index c9238bc99978..d79603cea3cc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMasterRegistry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMasterRegistry.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -38,6 +37,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.AfterClass; @@ -85,45 +85,39 @@ private static String generateDummyMastersList(int size) { * Makes sure the master registry parses the master end points in the configuration correctly. */ @Test - public void testMasterAddressParsing() throws IOException { + public void testMasterAddressParsing() throws Exception { Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); int numMasters = 10; conf.set(HConstants.MASTER_ADDRS_KEY, generateDummyMastersList(numMasters)); - try (MasterRegistry registry = new MasterRegistry(conf)) { - List parsedMasters = new ArrayList<>(registry.getParsedServers()); - // Half of them would be without a port, duplicates are removed. - assertEquals(numMasters / 2 + 1, parsedMasters.size()); - // Sort in the increasing order of port numbers. - Collections.sort(parsedMasters, Comparator.comparingInt(ServerName::getPort)); - for (int i = 0; i < parsedMasters.size(); i++) { - ServerName sn = parsedMasters.get(i); - assertEquals("localhost", sn.getHostname()); - if (i == parsedMasters.size() - 1) { - // Last entry should be the one with default port. - assertEquals(HConstants.DEFAULT_MASTER_PORT, sn.getPort()); - } else { - assertEquals(1000 + (2 * i), sn.getPort()); - } + List parsedMasters = new ArrayList<>(MasterRegistry.parseMasterAddrs(conf)); + // Half of them would be without a port, duplicates are removed. + assertEquals(numMasters / 2 + 1, parsedMasters.size()); + // Sort in the increasing order of port numbers. + Collections.sort(parsedMasters, Comparator.comparingInt(ServerName::getPort)); + for (int i = 0; i < parsedMasters.size(); i++) { + ServerName sn = parsedMasters.get(i); + assertEquals("localhost", sn.getHostname()); + if (i == parsedMasters.size() - 1) { + // Last entry should be the one with default port. + assertEquals(HConstants.DEFAULT_MASTER_PORT, sn.getPort()); + } else { + assertEquals(1000 + (2 * i), sn.getPort()); } } } @Test - public void testMasterPortDefaults() throws IOException { + public void testMasterPortDefaults() throws Exception { Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.set(HConstants.MASTER_ADDRS_KEY, "localhost"); - try (MasterRegistry registry = new MasterRegistry(conf)) { - List parsedMasters = new ArrayList<>(registry.getParsedServers()); - ServerName sn = parsedMasters.get(0); - assertEquals(HConstants.DEFAULT_MASTER_PORT, sn.getPort()); - } + List parsedMasters = new ArrayList<>(MasterRegistry.parseMasterAddrs(conf)); + ServerName sn = parsedMasters.get(0); + assertEquals(HConstants.DEFAULT_MASTER_PORT, sn.getPort()); final int CUSTOM_MASTER_PORT = 9999; conf.setInt(HConstants.MASTER_PORT, CUSTOM_MASTER_PORT); - try (MasterRegistry registry = new MasterRegistry(conf)) { - List parsedMasters = new ArrayList<>(registry.getParsedServers()); - ServerName sn = parsedMasters.get(0); - assertEquals(CUSTOM_MASTER_PORT, sn.getPort()); - } + parsedMasters = new ArrayList<>(MasterRegistry.parseMasterAddrs(conf)); + sn = parsedMasters.get(0); + assertEquals(CUSTOM_MASTER_PORT, sn.getPort()); } @Test @@ -133,7 +127,7 @@ public void testRegistryRPCs() throws Exception { final int size = activeMaster.getMetaLocations().size(); for (int numHedgedReqs = 1; numHedgedReqs <= size; numHedgedReqs++) { conf.setInt(MasterRegistry.MASTER_REGISTRY_HEDGED_REQS_FANOUT_KEY, numHedgedReqs); - try (MasterRegistry registry = new MasterRegistry(conf)) { + try (MasterRegistry registry = new MasterRegistry(conf, User.getCurrent())) { // Add wait on all replicas being assigned before proceeding w/ test. Failed on occasion // because not all replicas had made it up before test started. RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, registry); @@ -166,7 +160,7 @@ public void testDynamicMasterConfigurationRefresh() throws Exception { conf.setInt(MasterRegistry.MASTER_REGISTRY_HEDGED_REQS_FANOUT_KEY, 4); // Do not limit the number of refreshes during the test run. conf.setLong(MasterRegistry.MASTER_REGISTRY_MIN_SECS_BETWEEN_REFRESHES, 0); - try (MasterRegistry registry = new MasterRegistry(conf)) { + try (MasterRegistry registry = new MasterRegistry(conf, User.getCurrent())) { final Set masters = registry.getParsedServers(); assertTrue(masters.contains(badServer)); // Make a registry RPC, this should trigger a refresh since one of the hedged RPC fails. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaRegionLocationCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaRegionLocationCache.java index d78832d9a8a0..beb054eaf366 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaRegionLocationCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaRegionLocationCache.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.JVMClusterUtil; @@ -63,7 +64,8 @@ public class TestMetaRegionLocationCache { public static void setUp() throws Exception { TEST_UTIL.startMiniCluster(3); HBaseTestingUtil.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3); - REGISTRY = ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); + REGISTRY = + ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent()); RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY); TEST_UTIL.getAdmin().balancerSwitch(false, true); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRpcConnectionRegistry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRpcConnectionRegistry.java index 9c26bccbbb31..d33cc943355c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRpcConnectionRegistry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRpcConnectionRegistry.java @@ -17,22 +17,29 @@ */ package org.apache.hadoop.hbase.client; -import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.BootstrapNodeManager; import org.apache.hadoop.hbase.regionserver.RSRpcServices; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.After; @@ -43,6 +50,7 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; @Category({ MediumTests.class, ClientTests.class }) @@ -74,7 +82,7 @@ public static void tearDownAfterClass() throws Exception { @Before public void setUp() throws IOException { - registry = new RpcConnectionRegistry(UTIL.getConfiguration()); + registry = new RpcConnectionRegistry(UTIL.getConfiguration(), User.getCurrent()); } @After @@ -94,9 +102,20 @@ private void setMaxNodeCount(int count) { @Test public void testRegistryRPCs() throws Exception { HMaster activeMaster = UTIL.getHBaseCluster().getMaster(); - // sleep 3 seconds, since our initial delay is 1 second, we should have refreshed the endpoints - Thread.sleep(3000); - assertThat(registry.getParsedServers(), + // should only contains the active master + Set initialParsedServers = registry.getParsedServers(); + assertThat(initialParsedServers, hasSize(1)); + // no start code in configuration + assertThat(initialParsedServers, + hasItem(ServerName.valueOf(activeMaster.getServerName().getHostname(), + activeMaster.getServerName().getPort(), -1))); + // Since our initial delay is 1 second, finally we should have refreshed the endpoints + UTIL.waitFor(5000, () -> registry.getParsedServers() + .contains(activeMaster.getServerManager().getOnlineServersList().get(0))); + Set parsedServers = registry.getParsedServers(); + assertThat(parsedServers, + hasSize(activeMaster.getServerManager().getOnlineServersList().size())); + assertThat(parsedServers, hasItems(activeMaster.getServerManager().getOnlineServersList().toArray(new ServerName[0]))); // Add wait on all replicas being assigned before proceeding w/ test. Failed on occasion @@ -116,4 +135,32 @@ public void testRegistryRPCs() throws Exception { setMaxNodeCount(1); UTIL.waitFor(10000, () -> registry.getParsedServers().size() == 1); } + + /** + * Make sure that we can create the RpcClient when there are broken servers in the bootstrap nodes + */ + @Test + public void testBrokenBootstrapNodes() throws Exception { + Configuration conf = new Configuration(UTIL.getConfiguration()); + String currentMasterAddrs = Preconditions.checkNotNull(conf.get(HConstants.MASTER_ADDRS_KEY)); + HMaster activeMaster = UTIL.getHBaseCluster().getMaster(); + String clusterId = activeMaster.getClusterId(); + // Add a non-working master + ServerName badServer = ServerName.valueOf("localhost", 1234, -1); + conf.set(RpcConnectionRegistry.BOOTSTRAP_NODES, badServer.toShortString()); + // only a bad server, the request should fail + try (RpcConnectionRegistry reg = new RpcConnectionRegistry(conf, User.getCurrent())) { + assertThrows(IOException.class, () -> reg.getParsedServers()); + } + + conf.set(RpcConnectionRegistry.BOOTSTRAP_NODES, + badServer.toShortString() + ", " + currentMasterAddrs); + // we will choose bootstrap node randomly so here we need to test it multiple times to make sure + // that we can skip the broken node + for (int i = 0; i < 10; i++) { + try (RpcConnectionRegistry reg = new RpcConnectionRegistry(conf, User.getCurrent())) { + assertEquals(clusterId, reg.getClusterId().get()); + } + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKConnectionRegistry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKConnectionRegistry.java index 1cbb36196684..6d585245e959 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKConnectionRegistry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKConnectionRegistry.java @@ -65,7 +65,7 @@ public class TestZKConnectionRegistry { public static void setUp() throws Exception { TEST_UTIL.startMiniCluster(3); HBaseTestingUtil.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3); - REGISTRY = new ZKConnectionRegistry(TEST_UTIL.getConfiguration()); + REGISTRY = new ZKConnectionRegistry(TEST_UTIL.getConfiguration(), null); } @AfterClass @@ -99,7 +99,7 @@ public void testIndependentZKConnections() throws IOException { try (ReadOnlyZKClient zk1 = REGISTRY.getZKClient()) { Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration()); otherConf.set(HConstants.ZOOKEEPER_QUORUM, MiniZooKeeperCluster.HOST); - try (ZKConnectionRegistry otherRegistry = new ZKConnectionRegistry(otherConf)) { + try (ZKConnectionRegistry otherRegistry = new ZKConnectionRegistry(otherConf, null)) { ReadOnlyZKClient zk2 = otherRegistry.getZKClient(); assertNotSame("Using a different configuration / quorum should result in different " + "backing zk connection.", zk1, zk2); @@ -116,7 +116,7 @@ public void testIndependentZKConnections() throws IOException { public void testNoMetaAvailable() throws InterruptedException { Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.set("zookeeper.znode.metaserver", "whatever"); - try (ZKConnectionRegistry registry = new ZKConnectionRegistry(conf)) { + try (ZKConnectionRegistry registry = new ZKConnectionRegistry(conf, null)) { try { registry.getMetaRegionLocations().get(); fail("Should have failed since we set an incorrect meta znode prefix"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java index 9e65e58a56d2..cf647b9fe8a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java @@ -191,6 +191,8 @@ public static class CPMasterObserver implements MasterCoprocessor, MasterObserve private boolean postLockHeartbeatCalled; private boolean preMasterStoreFlushCalled; private boolean postMasterStoreFlushCalled; + private boolean preUpdateMasterConfigurationCalled; + private boolean postUpdateMasterConfigurationCalled; public void resetStates() { preCreateTableRegionInfosCalled = false; @@ -284,6 +286,8 @@ public void resetStates() { postLockHeartbeatCalled = false; preMasterStoreFlushCalled = false; postMasterStoreFlushCalled = false; + preUpdateMasterConfigurationCalled = false; + postUpdateMasterConfigurationCalled = false; } @Override @@ -1264,6 +1268,17 @@ public void postRollBackMergeRegionsAction( throws IOException { } + @Override + public void preUpdateMasterConfiguration(ObserverContext ctx, + Configuration preReloadConf) throws IOException { + preUpdateMasterConfigurationCalled = true; + } + + @Override + public void postUpdateMasterConfiguration(ObserverContext ctx, + Configuration postReloadConf) throws IOException { + postUpdateMasterConfigurationCalled = true; + } } private static HBaseTestingUtil UTIL = new HBaseTestingUtil(); @@ -1715,4 +1730,23 @@ public void testMasterStoreOperations() throws Exception { assertTrue("Master store flush called", cp.postMasterStoreFlushCalled); } } + + @Test + public void testUpdateConfiguration() throws Exception { + SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + MasterCoprocessorHost host = master.getMasterCoprocessorHost(); + CPMasterObserver cp = host.findCoprocessor(CPMasterObserver.class); + cp.resetStates(); + assertFalse("No update configuration call", cp.preUpdateMasterConfigurationCalled); + assertFalse("No update configuration call", cp.postUpdateMasterConfigurationCalled); + + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.updateConfiguration(); + + assertTrue("Update configuration called", cp.preUpdateMasterConfigurationCalled); + assertTrue("Update configuration called", cp.postUpdateMasterConfigurationCalled); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java index bb476670f10f..77aa00ef91f9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -31,6 +32,7 @@ import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.HasEnhancedByteBufferAccess; import org.apache.hadoop.fs.ReadOption; +import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.io.ByteBufferPool; @@ -48,22 +50,22 @@ public class TestFSDataInputStreamWrapper { @Test public void testUnbuffer() throws Exception { InputStream pc = new ParentClass(); - FSDataInputStreamWrapper fsdisw1 = new FSDataInputStreamWrapper(new FSDataInputStream(pc)); + InputStream noChecksumPc = new ParentClass(); + FSDataInputStreamWrapper fsdisw1 = + new FSDataInputStreamWrapper(new FSDataInputStream(pc), new FSDataInputStream(noChecksumPc)); fsdisw1.unbuffer(); - // parent class should be true + // should have called main stream unbuffer, but not no-checksum assertTrue(((ParentClass) pc).getIsCallUnbuffer()); + assertFalse(((ParentClass) noChecksumPc).getIsCallUnbuffer()); + // switch to checksums and call unbuffer again. should unbuffer the nochecksum stream now + fsdisw1.setShouldUseHBaseChecksum(); + fsdisw1.unbuffer(); + assertTrue(((ParentClass) noChecksumPc).getIsCallUnbuffer()); fsdisw1.close(); - - InputStream cc1 = new ChildClass1(); - FSDataInputStreamWrapper fsdisw2 = new FSDataInputStreamWrapper(new FSDataInputStream(cc1)); - fsdisw2.unbuffer(); - // child1 class should be true - assertTrue(((ChildClass1) cc1).getIsCallUnbuffer()); - fsdisw2.close(); } private class ParentClass extends FSInputStream implements ByteBufferReadable, CanSetDropBehind, - CanSetReadahead, HasEnhancedByteBufferAccess, CanUnbuffer { + CanSetReadahead, HasEnhancedByteBufferAccess, CanUnbuffer, StreamCapabilities { public boolean isCallUnbuffer = false; @@ -122,12 +124,10 @@ public long getPos() throws IOException { public boolean seekToNewSource(long paramLong) throws IOException { return false; } - } - private class ChildClass1 extends ParentClass { @Override - public void unbuffer() { - isCallUnbuffer = true; + public boolean hasCapability(String s) { + return s.equals(StreamCapabilities.UNBUFFER); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index 0b45a930dceb..85b9199638c0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -42,6 +42,7 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -156,6 +157,43 @@ public void testPrefetchBlockCacheDisabled() throws Exception { poolExecutor.getCompletedTaskCount() + poolExecutor.getQueue().size()); } + @Test + public void testPrefetchHeapUsageAboveThreshold() throws Exception { + ColumnFamilyDescriptor columnFamilyDescriptor = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f")).setPrefetchBlocksOnOpen(true) + .setBlockCacheEnabled(true).build(); + HFileContext meta = new HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE).build(); + Configuration newConf = new Configuration(conf); + newConf.setDouble(CacheConfig.PREFETCH_HEAP_USAGE_THRESHOLD, 0.1); + CacheConfig cacheConfig = + new CacheConfig(newConf, columnFamilyDescriptor, blockCache, ByteBuffAllocator.HEAP); + Path storeFile = writeStoreFile("testPrefetchHeapUsageAboveThreshold", meta, cacheConfig); + MutableInt cachedCount = new MutableInt(0); + MutableInt unCachedCount = new MutableInt(0); + readStoreFile(storeFile, (r, o) -> { + HFileBlock block = null; + try { + block = r.readBlock(o, -1, false, true, false, true, null, null); + } catch (IOException e) { + fail(e.getMessage()); + } + return block; + }, (key, block) -> { + boolean isCached = blockCache.getBlock(key, true, false, true) != null; + if ( + block.getBlockType() == BlockType.DATA || block.getBlockType() == BlockType.ROOT_INDEX + || block.getBlockType() == BlockType.INTERMEDIATE_INDEX + ) { + if (isCached) { + cachedCount.increment(); + } else { + unCachedCount.increment(); + } + } + }, cacheConfig); + assertTrue(unCachedCount.compareTo(cachedCount) > 0); + } + @Test public void testPrefetch() throws Exception { TraceUtil.trace(() -> { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java index 446298235471..2d0a85962ef9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java @@ -32,6 +32,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.BiConsumer; import java.util.function.BiFunction; +import org.apache.commons.lang3.mutable.MutableLong; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -185,6 +186,30 @@ public void testPrefetchDoesntInterruptInMemoryOnCapacity() throws Exception { assertTrue(bc.getStats().getEvictedCount() > 200); } + @Test + public void testPrefetchMetricProgress() throws Exception { + conf.setLong(BUCKET_CACHE_SIZE_KEY, 200); + blockCache = BlockCacheFactory.createBlockCache(conf); + cacheConf = new CacheConfig(conf, blockCache); + Path storeFile = writeStoreFile("testPrefetchMetricsProgress", 100); + // Prefetches the file blocks + LOG.debug("First read should prefetch the blocks."); + readStoreFile(storeFile); + String regionName = storeFile.getParent().getParent().getName(); + BucketCache bc = BucketCache.getBucketCacheFromCacheConfig(cacheConf).get(); + MutableLong regionCachedSize = new MutableLong(0); + // Our file should have 6 DATA blocks. We should wait for all of them to be cached + long waitedTime = Waiter.waitFor(conf, 300, () -> { + if (bc.getBackingMap().size() > 0) { + long currentSize = bc.getRegionCachedInfo().get().get(regionName); + assertTrue(regionCachedSize.getValue() <= currentSize); + LOG.debug("Logging progress of region caching: {}", currentSize); + regionCachedSize.setValue(currentSize); + } + return bc.getBackingMap().size() == 6; + }); + } + private void readStoreFile(Path storeFilePath) throws Exception { readStoreFile(storeFilePath, (r, o) -> { HFileBlock block = null; @@ -216,6 +241,7 @@ private void readStoreFile(Path storeFilePath, Thread.sleep(1000); } long offset = 0; + long sizeForDataBlocks = 0; while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { HFileBlock block = readFunction.apply(reader, offset); BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset); @@ -276,5 +302,4 @@ public static KeyValue.Type generateKeyType(Random rand) { return keyType; } } - } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java index f6d3efa9015d..a39df7e14715 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCachePersister.java @@ -86,9 +86,10 @@ public Configuration setupBucketCacheConfig(long bucketCachePersistInterval) thr return conf; } - public BucketCache setupBucketCache(Configuration conf) throws IOException { - BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, - constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, + public BucketCache setupBucketCache(Configuration conf, String persistentCacheFile) + throws IOException { + BucketCache bucketCache = new BucketCache("file:" + testDir + "/" + persistentCacheFile, + capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, testDir + "/bucket.persistence", 60 * 1000, conf); return bucketCache; } @@ -103,7 +104,7 @@ public void cleanupBucketCache(BucketCache bucketCache) throws IOException { public void testPrefetchPersistenceCrash() throws Exception { long bucketCachePersistInterval = 3000; Configuration conf = setupBucketCacheConfig(bucketCachePersistInterval); - BucketCache bucketCache = setupBucketCache(conf); + BucketCache bucketCache = setupBucketCache(conf, "testPrefetchPersistenceCrash"); CacheConfig cacheConf = new CacheConfig(conf, bucketCache); FileSystem fs = HFileSystem.get(conf); // Load Cache @@ -121,7 +122,7 @@ public void testPrefetchPersistenceCrash() throws Exception { public void testPrefetchPersistenceCrashNegative() throws Exception { long bucketCachePersistInterval = Long.MAX_VALUE; Configuration conf = setupBucketCacheConfig(bucketCachePersistInterval); - BucketCache bucketCache = setupBucketCache(conf); + BucketCache bucketCache = setupBucketCache(conf, "testPrefetchPersistenceCrashNegative"); CacheConfig cacheConf = new CacheConfig(conf, bucketCache); FileSystem fs = HFileSystem.get(conf); // Load Cache @@ -134,7 +135,7 @@ public void testPrefetchPersistenceCrashNegative() throws Exception { @Test public void testPrefetchListUponBlockEviction() throws Exception { Configuration conf = setupBucketCacheConfig(200); - BucketCache bucketCache = setupBucketCache(conf); + BucketCache bucketCache = setupBucketCache(conf, "testPrefetchListUponBlockEviction"); CacheConfig cacheConf = new CacheConfig(conf, bucketCache); FileSystem fs = HFileSystem.get(conf); // Load Blocks in cache @@ -156,7 +157,8 @@ public void testPrefetchListUponBlockEviction() throws Exception { @Test public void testPrefetchBlockEvictionWhilePrefetchRunning() throws Exception { Configuration conf = setupBucketCacheConfig(200); - BucketCache bucketCache = setupBucketCache(conf); + BucketCache bucketCache = + setupBucketCache(conf, "testPrefetchBlockEvictionWhilePrefetchRunning"); CacheConfig cacheConf = new CacheConfig(conf, bucketCache); FileSystem fs = HFileSystem.get(conf); // Load Blocks in cache diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java index fe947d33110d..e4427c1690c3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java @@ -29,9 +29,12 @@ import static org.apache.hadoop.hbase.ipc.TestProtobufRpcServiceImpl.newStub; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -40,8 +43,10 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.mockito.internal.verification.VerificationModeFactory.times; import io.opentelemetry.api.common.AttributeKey; @@ -53,6 +58,7 @@ import java.net.InetSocketAddress; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; @@ -60,12 +66,18 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseServerBase; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MatcherPredicate; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.StringUtils; import org.hamcrest.Matcher; import org.junit.Rule; @@ -75,6 +87,8 @@ import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hbase.thirdparty.com.google.protobuf.BlockingRpcChannel; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcChannel; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; import org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProtos.EchoRequestProto; @@ -85,6 +99,9 @@ import org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface; import org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos.TestProtobufRpcProto.Interface; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.ConnectionRegistryService; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetConnectionRegistryRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetConnectionRegistryResponse; /** * Some basic ipc tests. @@ -102,9 +119,14 @@ public abstract class AbstractTestIPC { CONF.set(RpcServerFactory.CUSTOM_RPC_SERVER_IMPL_CONF_KEY, SimpleRpcServer.class.getName()); } - protected abstract RpcServer createRpcServer(final String name, - final List services, final InetSocketAddress bindAddress, - Configuration conf, RpcScheduler scheduler) throws IOException; + protected abstract RpcServer createRpcServer(Server server, String name, + List services, InetSocketAddress bindAddress, Configuration conf, + RpcScheduler scheduler) throws IOException; + + private RpcServer createRpcServer(String name, List services, + InetSocketAddress bindAddress, Configuration conf, RpcScheduler scheduler) throws IOException { + return createRpcServer(null, name, services, bindAddress, conf, scheduler); + } protected abstract AbstractRpcClient createRpcClientNoCodec(Configuration conf); @@ -545,4 +567,82 @@ public void testTracingErrorIpc() throws IOException { hasTraceId(traceRule.getSpans().iterator().next().getTraceId())))); } } + + protected abstract AbstractRpcClient createBadAuthRpcClient(Configuration conf); + + @Test + public void testBadPreambleHeader() throws IOException, ServiceException { + Configuration clientConf = new Configuration(CONF); + RpcServer rpcServer = createRpcServer("testRpcServer", Collections.emptyList(), + new InetSocketAddress("localhost", 0), CONF, new FifoRpcScheduler(CONF, 1)); + try (AbstractRpcClient client = createBadAuthRpcClient(clientConf)) { + rpcServer.start(); + BlockingInterface stub = newBlockingStub(client, rpcServer.getListenerAddress()); + ServiceException se = assertThrows(ServiceException.class, + () -> stub.echo(null, EchoRequestProto.newBuilder().setMessage("hello").build())); + IOException ioe = ProtobufUtil.handleRemoteException(se); + assertThat(ioe, instanceOf(BadAuthException.class)); + assertThat(ioe.getMessage(), containsString("authName=unknown")); + } finally { + rpcServer.stop(); + } + } + + /** + * Testcase for getting connection registry information through connection preamble header, see + * HBASE-25051 for more details. + */ + @Test + public void testGetConnectionRegistry() throws IOException, ServiceException { + Configuration clientConf = new Configuration(CONF); + String clusterId = "test_cluster_id"; + HBaseServerBase server = mock(HBaseServerBase.class); + when(server.getClusterId()).thenReturn(clusterId); + // do not need any services + RpcServer rpcServer = createRpcServer(server, "testRpcServer", Collections.emptyList(), + new InetSocketAddress("localhost", 0), CONF, new FifoRpcScheduler(CONF, 1)); + try (AbstractRpcClient client = createRpcClient(clientConf)) { + rpcServer.start(); + InetSocketAddress addr = rpcServer.getListenerAddress(); + BlockingRpcChannel channel = + client.createBlockingRpcChannel(ServerName.valueOf(addr.getHostName(), addr.getPort(), + EnvironmentEdgeManager.currentTime()), User.getCurrent(), 0); + ConnectionRegistryService.BlockingInterface stub = + ConnectionRegistryService.newBlockingStub(channel); + GetConnectionRegistryResponse resp = + stub.getConnectionRegistry(null, GetConnectionRegistryRequest.getDefaultInstance()); + assertEquals(clusterId, resp.getClusterId()); + } + } + + /** + * Test server does not support getting connection registry information through connection + * preamble header, i.e, a new client connecting to an old server. We simulate this by using a + * Server without implementing the ConnectionRegistryEndpoint interface. + */ + @Test + public void testGetConnectionRegistryError() throws IOException, ServiceException { + Configuration clientConf = new Configuration(CONF); + // do not need any services + RpcServer rpcServer = createRpcServer("testRpcServer", Collections.emptyList(), + new InetSocketAddress("localhost", 0), CONF, new FifoRpcScheduler(CONF, 1)); + try (AbstractRpcClient client = createRpcClient(clientConf)) { + rpcServer.start(); + InetSocketAddress addr = rpcServer.getListenerAddress(); + RpcChannel channel = client.createRpcChannel(ServerName.valueOf(addr.getHostName(), + addr.getPort(), EnvironmentEdgeManager.currentTime()), User.getCurrent(), 0); + ConnectionRegistryService.Interface stub = ConnectionRegistryService.newStub(channel); + HBaseRpcController pcrc = new HBaseRpcControllerImpl(); + BlockingRpcCallback done = new BlockingRpcCallback<>(); + stub.getConnectionRegistry(pcrc, GetConnectionRegistryRequest.getDefaultInstance(), done); + // should have failed so no response + assertNull(done.get()); + assertTrue(pcrc.failed()); + // should be a FatalConnectionException + assertThat(pcrc.getFailed(), instanceOf(RemoteException.class)); + assertEquals(FatalConnectionException.class.getName(), + ((RemoteException) pcrc.getFailed()).getClassName()); + assertThat(pcrc.getFailed().getMessage(), startsWith("Expected HEADER=")); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/BadAuthNettyRpcConnection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/BadAuthNettyRpcConnection.java new file mode 100644 index 000000000000..63554421dfb6 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/BadAuthNettyRpcConnection.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.ipc; + +import java.io.IOException; + +public class BadAuthNettyRpcConnection extends NettyRpcConnection { + + public BadAuthNettyRpcConnection(NettyRpcClient rpcClient, ConnectionId remoteId) + throws IOException { + super(rpcClient, remoteId); + } + + @Override + protected byte[] getConnectionHeaderPreamble() { + byte[] header = super.getConnectionHeaderPreamble(); + // set an invalid auth code + header[header.length - 1] = -10; + return header; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestBlockingIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestBlockingIPC.java index 4d7d0996fabd..e60cc879fd4f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestBlockingIPC.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestBlockingIPC.java @@ -40,10 +40,10 @@ public class TestBlockingIPC extends AbstractTestIPC { HBaseClassTestRule.forClass(TestBlockingIPC.class); @Override - protected RpcServer createRpcServer(String name, + protected RpcServer createRpcServer(Server server, String name, List services, InetSocketAddress bindAddress, Configuration conf, RpcScheduler scheduler) throws IOException { - return RpcServerFactory.createRpcServer(null, name, services, bindAddress, conf, scheduler); + return RpcServerFactory.createRpcServer(server, name, services, bindAddress, conf, scheduler); } @Override @@ -107,4 +107,24 @@ protected RpcServer createTestFailingRpcServer(String name, Configuration conf, RpcScheduler scheduler) throws IOException { return new TestFailingRpcServer(null, name, services, bindAddress, conf, scheduler); } + + @Override + protected AbstractRpcClient createBadAuthRpcClient(Configuration conf) { + return new BlockingRpcClient(conf) { + + @Override + protected BlockingRpcConnection createConnection(ConnectionId remoteId) throws IOException { + return new BlockingRpcConnection(this, remoteId) { + @Override + protected byte[] getConnectionHeaderPreamble() { + byte[] header = super.getConnectionHeaderPreamble(); + // set an invalid auth code + header[header.length - 1] = -10; + return header; + } + }; + } + + }; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java index 265ae7852f02..a1b60e2cfa45 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RPCTests; @@ -103,10 +104,10 @@ private void setConf(Configuration conf) { } @Override - protected RpcServer createRpcServer(String name, + protected RpcServer createRpcServer(Server server, String name, List services, InetSocketAddress bindAddress, Configuration conf, RpcScheduler scheduler) throws IOException { - return new NettyRpcServer(null, name, services, bindAddress, conf, scheduler, true); + return new NettyRpcServer(server, name, services, bindAddress, conf, scheduler, true); } @Override @@ -146,4 +147,15 @@ protected RpcServer createTestFailingRpcServer(String name, Configuration conf, RpcScheduler scheduler) throws IOException { return new FailingNettyRpcServer(null, name, services, bindAddress, conf, scheduler); } + + @Override + protected AbstractRpcClient createBadAuthRpcClient(Configuration conf) { + return new NettyRpcClient(conf) { + + @Override + protected NettyRpcConnection createConnection(ConnectionId remoteId) throws IOException { + return new BadAuthNettyRpcConnection(this, remoteId); + } + }; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcServer.java index 6af67e2190d3..2d5b95028f6f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcServer.java @@ -17,11 +17,25 @@ */ package org.apache.hadoop.hbase.ipc; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.io.ByteArrayInputStream; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.Collection; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSession; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; @@ -47,6 +61,8 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler; + @Category({ RPCTests.class, MediumTests.class }) @RunWith(Parameterized.class) public class TestNettyRpcServer { @@ -122,4 +138,51 @@ protected void doTest(TableName tableName) throws Exception { } } + private static final String CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" + + "MIIEITCCAwmgAwIBAgIUaLL8vLOhWLCLXVHEJqXJhfmsTB8wDQYJKoZIhvcNAQEL\n" + + "BQAwgawxCzAJBgNVBAYTAlVTMRYwFAYDVQQIDA1NYXNzYWNodXNldHRzMRIwEAYD\n" + + "VQQHDAlDYW1icmlkZ2UxGDAWBgNVBAoMD25ldHR5IHRlc3QgY2FzZTEYMBYGA1UE\n" + + "CwwPbmV0dHkgdGVzdCBjYXNlMRgwFgYDVQQDDA9uZXR0eSB0ZXN0IGNhc2UxIzAh\n" + + "BgkqhkiG9w0BCQEWFGNjb25uZWxsQGh1YnNwb3QuY29tMB4XDTI0MDEyMTE5MzMy\n" + + "MFoXDTI1MDEyMDE5MzMyMFowgawxCzAJBgNVBAYTAlVTMRYwFAYDVQQIDA1NYXNz\n" + + "YWNodXNldHRzMRIwEAYDVQQHDAlDYW1icmlkZ2UxGDAWBgNVBAoMD25ldHR5IHRl\n" + + "c3QgY2FzZTEYMBYGA1UECwwPbmV0dHkgdGVzdCBjYXNlMRgwFgYDVQQDDA9uZXR0\n" + + "eSB0ZXN0IGNhc2UxIzAhBgkqhkiG9w0BCQEWFGNjb25uZWxsQGh1YnNwb3QuY29t\n" + + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy+qzEZpQMjVdLj0siUcG\n" + + "y8LIHOW4S+tgHIKFkF865qWq6FVGbROe2Z0f5W6yIamZkdxzptT0iv+8S5okNNeW\n" + + "2NbsN/HNJIRtWfxku1Jh1gBqSkAYIjXyq7+20hIaJTzzxqike9M/Lc14EGb33Ja/\n" + + "kDPRV3UtiM3Ntf3eALXKbrWptkbgQngCaTgtfg8IkMAEpP270wZ9fW0lDHv3NPPt\n" + + "Zt0QSJzWSqWfu+l4ayvcUQYyNJesx9YmTHSJu69lvT4QApoX8FEiHfNCJ28R50CS\n" + + "aIgOpCWUvkH7rqx0p9q393uJRS/S6RlLbU30xUN1fNrVmP/XAapfy+R0PSgiUi8o\n" + + "EQIDAQABozkwNzAWBgNVHRIEDzANggt3d3cuZm9vLmNvbTAdBgNVHQ4EFgQUl4FD\n" + + "Y8jJ/JHJR68YqPsGUjUJuwgwDQYJKoZIhvcNAQELBQADggEBADVzivYz2M0qsWUc\n" + + "jXjCHymwTIr+7ud10um53FbYEAfKWsIY8Pp35fKpFzUwc5wVdCnLU86K/YMKRzNB\n" + + "zL2Auow3PJFRvXecOv7dWxNlNneLDcwbVrdNRu6nQXmZUgyz0oUKuJbF+JGtI+7W\n" + + "kRw7yhBfki+UCSQWeDqvaWzgmA4Us0N8NFq3euAs4xFbMMPMQWrT9Z7DGchCeRiB\n" + + "dkQBvh88vbR3v2Saq14W4Wt5rj2++vXWGQSeAQL6nGbOwc3ohW6isNNV0eGQQTmS\n" + + "khS2d/JDZq2XL5RGexf3CA6YYzWiTr9YZHNjuobvLH7mVnA2c8n6Zty/UhfnuK1x\n" + "JbkleFk=\n" + + "-----END CERTIFICATE-----"; + + @Test + public void testHandshakeCompleteHandler() + throws SSLPeerUnverifiedException, CertificateException { + NettyServerRpcConnection conn = mock(NettyServerRpcConnection.class); + SslHandler sslHandler = mock(SslHandler.class); + SocketAddress remoteAddress = new InetSocketAddress("localhost", 5555); + SSLEngine engine = mock(SSLEngine.class); + SSLSession session = mock(SSLSession.class); + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + X509Certificate x509Certificate = (X509Certificate) certificateFactory + .generateCertificate(new ByteArrayInputStream(CERTIFICATE.getBytes(StandardCharsets.UTF_8))); + Certificate[] certificates = new Certificate[] { x509Certificate }; + + when(sslHandler.engine()).thenReturn(engine); + when(engine.getSession()).thenReturn(session); + when(session.getPeerCertificates()).thenReturn(certificates); + + NettyRpcServer.sslHandshakeCompleteHandler(conn, sslHandler, remoteAddress); + + assertArrayEquals(certificates, conn.clientCertificateChain); + } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTlsIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyTlsIPC.java similarity index 82% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTlsIPC.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyTlsIPC.java index f21e3b93bef5..1cbf6be26c65 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTlsIPC.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyTlsIPC.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hbase.security; +package org.apache.hadoop.hbase.ipc; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -31,19 +31,14 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtil; import org.apache.hadoop.hbase.HBaseServerBase; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.client.ConnectionRegistryEndpoint; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.io.crypto.tls.KeyStoreFileType; import org.apache.hadoop.hbase.io.crypto.tls.X509KeyType; import org.apache.hadoop.hbase.io.crypto.tls.X509TestContext; import org.apache.hadoop.hbase.io.crypto.tls.X509TestContextProvider; import org.apache.hadoop.hbase.io.crypto.tls.X509Util; -import org.apache.hadoop.hbase.ipc.AbstractRpcClient; -import org.apache.hadoop.hbase.ipc.AbstractTestIPC; -import org.apache.hadoop.hbase.ipc.FailingNettyRpcServer; -import org.apache.hadoop.hbase.ipc.NettyRpcClient; -import org.apache.hadoop.hbase.ipc.NettyRpcServer; -import org.apache.hadoop.hbase.ipc.RpcScheduler; -import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RPCTests; @@ -72,8 +67,6 @@ public class TestNettyTlsIPC extends AbstractTestIPC { private static NettyEventLoopGroupConfig EVENT_LOOP_GROUP_CONFIG; - private static HBaseServerBase SERVER; - @Parameterized.Parameter(0) public X509KeyType caKeyType; @@ -122,8 +115,6 @@ public static void setUpBeforeClass() throws IOException { PROVIDER = new X509TestContextProvider(CONF, dir); EVENT_LOOP_GROUP_CONFIG = NettyEventLoopGroupConfig.setup(CONF, TestNettyTlsIPC.class.getSimpleName()); - SERVER = mock(HBaseServerBase.class); - when(SERVER.getEventLoopGroupConfig()).thenReturn(EVENT_LOOP_GROUP_CONFIG); } @AfterClass @@ -154,9 +145,16 @@ public void tearDown() { } @Override - protected RpcServer createRpcServer(String name, List services, - InetSocketAddress bindAddress, Configuration conf, RpcScheduler scheduler) throws IOException { - return new NettyRpcServer(SERVER, name, services, bindAddress, conf, scheduler, true); + protected RpcServer createRpcServer(Server server, String name, + List services, InetSocketAddress bindAddress, Configuration conf, + RpcScheduler scheduler) throws IOException { + HBaseServerBase mockServer = mock(HBaseServerBase.class); + when(mockServer.getEventLoopGroupConfig()).thenReturn(EVENT_LOOP_GROUP_CONFIG); + if (server instanceof ConnectionRegistryEndpoint) { + String clusterId = ((ConnectionRegistryEndpoint) server).getClusterId(); + when(mockServer.getClusterId()).thenReturn(clusterId); + } + return new NettyRpcServer(mockServer, name, services, bindAddress, conf, scheduler, true); } @Override @@ -191,6 +189,19 @@ protected boolean isTcpNoDelay() { protected RpcServer createTestFailingRpcServer(String name, List services, InetSocketAddress bindAddress, Configuration conf, RpcScheduler scheduler) throws IOException { - return new FailingNettyRpcServer(SERVER, name, services, bindAddress, conf, scheduler); + HBaseServerBase mockServer = mock(HBaseServerBase.class); + when(mockServer.getEventLoopGroupConfig()).thenReturn(EVENT_LOOP_GROUP_CONFIG); + return new FailingNettyRpcServer(mockServer, name, services, bindAddress, conf, scheduler); + } + + @Override + protected AbstractRpcClient createBadAuthRpcClient(Configuration conf) { + return new NettyRpcClient(conf) { + + @Override + protected NettyRpcConnection createConnection(ConnectionId remoteId) throws IOException { + return new BadAuthNettyRpcConnection(this, remoteId); + } + }; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestBalancer.java index 8870c0f6dc36..d5ff71000a8c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestBalancer.java @@ -78,7 +78,7 @@ public void testAssignmentsForBalancer() throws Exception { AssignmentManager assignmentManager = master.getAssignmentManager(); RegionStates regionStates = assignmentManager.getRegionStates(); ServerName sn1 = ServerName.parseServerName("asf903.gq1.ygridcore.net,52690,1517835491385"); - regionStates.getOrCreateServer(sn1); + regionStates.createServer(sn1); TableStateManager tableStateManager = master.getTableStateManager(); ServerManager serverManager = master.getServerManager(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java index 13b1f5fea61e..2d5d9a4bc18f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java @@ -18,12 +18,16 @@ package org.apache.hadoop.hbase.master; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.net.InetAddress; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -44,11 +48,28 @@ public class TestClockSkewDetection { private static final Logger LOG = LoggerFactory.getLogger(TestClockSkewDetection.class); + private static final class DummyMasterServices extends MockNoopMasterServices { + + private final AssignmentManager am; + + public DummyMasterServices(Configuration conf) { + super(conf); + am = mock(AssignmentManager.class); + RegionStates rss = mock(RegionStates.class); + when(am.getRegionStates()).thenReturn(rss); + } + + @Override + public AssignmentManager getAssignmentManager() { + return am; + } + } + @Test public void testClockSkewDetection() throws Exception { final Configuration conf = HBaseConfiguration.create(); ServerManager sm = - new ServerManager(new MockNoopMasterServices(conf), new DummyRegionServerList()); + new ServerManager(new DummyMasterServices(conf), new DummyRegionServerList()); LOG.debug("regionServerStartup 1"); InetAddress ia1 = InetAddress.getLocalHost(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java index 69d81a986fe8..64ca75217928 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.master; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.StartTestingClusterOption; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; @@ -105,36 +106,44 @@ public void test() throws Exception { UTIL.waitFor(60000, () -> UTIL.getHBaseCluster().getMaster().isInitialized()); LOG.info("Started cluster master, waiting for {}", SERVER_FOR_TEST); UTIL.waitFor(60000, () -> getServerStateNode(SERVER_FOR_TEST) != null); - serverNode = getServerStateNode(SERVER_FOR_TEST); - assertFalse("serverNode should not be ONLINE during SCP processing", - serverNode.isInState(ServerState.ONLINE)); + UTIL.waitFor(30000, new ExplainingPredicate() { + + @Override + public boolean evaluate() throws Exception { + return !getServerStateNode(SERVER_FOR_TEST).isInState(ServerState.ONLINE); + } + + @Override + public String explainFailure() throws Exception { + return "serverNode should not be ONLINE during SCP processing"; + } + }); Optional> procedure = UTIL.getHBaseCluster().getMaster().getProcedures().stream() .filter(p -> (p instanceof ServerCrashProcedure) && ((ServerCrashProcedure) p).getServerName().equals(SERVER_FOR_TEST)) .findAny(); assertTrue("Should have one SCP for " + SERVER_FOR_TEST, procedure.isPresent()); - assertTrue("Submit the SCP for the same serverName " + SERVER_FOR_TEST + " which should fail", - UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(SERVER_FOR_TEST) - == Procedure.NO_PROC_ID); + assertEquals("Submit the SCP for the same serverName " + SERVER_FOR_TEST + " which should fail", + Procedure.NO_PROC_ID, + UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(SERVER_FOR_TEST)); // Wait the SCP to finish LOG.info("Waiting on latch"); SCP_LATCH.countDown(); UTIL.waitFor(60000, () -> procedure.get().isFinished()); + assertNull("serverNode should be deleted after SCP finished", + getServerStateNode(SERVER_FOR_TEST)); - assertFalse( + assertEquals( "Even when the SCP is finished, the duplicate SCP should not be scheduled for " + SERVER_FOR_TEST, - UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(SERVER_FOR_TEST) - == Procedure.NO_PROC_ID); - serverNode = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates() - .getServerNode(SERVER_FOR_TEST); - assertNull("serverNode should be deleted after SCP finished", serverNode); + Procedure.NO_PROC_ID, + UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(SERVER_FOR_TEST)); MetricsMasterSource masterSource = UTIL.getHBaseCluster().getMaster().getMasterMetrics().getMetricsSource(); metricsHelper.assertCounter(MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount", - 4, masterSource); + 3, masterSource); } private void setupCluster() throws Exception { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 3966b1f6bec9..09618b3d899e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -183,6 +183,8 @@ public void testDefaultMasterMetrics() throws Exception { metricsHelper.assertCounter(MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount", 0, masterSource); + metricsHelper.assertGauge("oldWALsDirSize", master.getMasterWalManager().getOldWALsDirSize(), + masterSource); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java index f73ebde89c11..d1389ee68e9f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java @@ -85,6 +85,7 @@ public void testInfo() throws IOException { assertEquals(master.getMasterCoprocessors().length, info.getCoprocessors().length); assertEquals(master.getServerManager().getOnlineServersList().size(), info.getNumRegionServers()); + assertEquals(master.getMasterWalManager().getOldWALsDirSize(), info.getOldWALsDirSize()); int regionServerCount = NUM_RS; assertEquals(regionServerCount, info.getNumRegionServers()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldWALsDirSizeChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldWALsDirSizeChore.java new file mode 100644 index 000000000000..7bd4ec5a1c24 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldWALsDirSizeChore.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.master.assignment.MockMasterServices; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests for OldWALsDirSizeChore Here we are using the {@link MockMasterServices} to mock the Hbase + * Master. Chore's won't be running automatically; we need to run every time. + */ +@Category({ MasterTests.class, SmallTests.class }) +public class TestOldWALsDirSizeChore { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestOldWALsDirSizeChore.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestOldWALsDirSizeChore.class); + + private MockMasterServices master; + + private static final HBaseTestingUtil HBASE_TESTING_UTILITY = new HBaseTestingUtil(); + + @Before + public void setUp() throws Exception { + master = new MockMasterServices(HBASE_TESTING_UTILITY.getConfiguration()); + master.start(10, null); + } + + @After + public void tearDown() throws Exception { + master.stop("tearDown"); + } + + @Test + public void testOldWALsDirSizeChore() throws IOException { + // Assume the OldWALs directory size is initially zero as the chore hasn't run yet + long currentOldWALsDirSize = master.getMasterWalManager().getOldWALsDirSize(); + assertEquals("Initial OldWALs directory size should be zero before running the chore", 0, + currentOldWALsDirSize); + + int dummyFileSize = 50 * 1024 * 1024; // 50MB + byte[] dummyData = new byte[dummyFileSize]; + + // Create a dummy file in the OldWALs directory + Path dummyFileInOldWALsDir = new Path(master.getMasterWalManager().getOldLogDir(), "dummy.txt"); + try (FSDataOutputStream outputStream = + master.getMasterWalManager().getFileSystem().create(dummyFileInOldWALsDir)) { + outputStream.write(dummyData); + } + + // Run the OldWALsDirSizeChore to update the directory size + OldWALsDirSizeChore oldWALsDirSizeChore = new OldWALsDirSizeChore(master); + oldWALsDirSizeChore.chore(); + + // Verify that the OldWALs directory size has increased by the file size + assertEquals("OldWALs directory size after chore should be as expected", dummyFileSize, + master.getMasterWalManager().getOldWALsDirSize()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java index c601425e5f0a..6bc4c9d14e6f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CoordinatedStateManager; @@ -305,7 +306,8 @@ public MockRegionStateStore(MasterServices master, MasterRegion masterRegion) { } @Override - public void updateRegionLocation(RegionStateNode regionNode) throws IOException { + public CompletableFuture updateRegionLocation(RegionStateNode regionNode) { + return CompletableFuture.completedFuture(null); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerUtil.java index eb6f069474a9..69381b37e38c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerUtil.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.List; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -80,7 +79,7 @@ public void tearDownAfterTest() throws IOException { for (RegionInfo region : UTIL.getAdmin().getRegions(TABLE_NAME)) { RegionStateNode regionNode = AM.getRegionStates().getRegionStateNode(region); // confirm that we have released the lock - assertFalse(((ReentrantLock) regionNode.lock).isLocked()); + assertFalse(regionNode.isLocked()); TransitRegionStateProcedure proc = regionNode.getProcedure(); if (proc != null) { regionNode.unsetProcedure(proc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java index abc6fc45ad30..c0c4e355f2bd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master.assignment; +import static org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcFailed; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -33,16 +34,20 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.SnapshotType; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; +import org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -59,6 +64,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; + @Category({ MasterTests.class, LargeTests.class }) public class TestMergeTableRegionsProcedure { @@ -347,6 +355,42 @@ public void testMergeWithoutPONR() throws Exception { assertRegionCount(tableName, initialRegionCount - 1); } + @Test + public void testMergingRegionWhileTakingSnapshot() throws Exception { + final TableName tableName = TableName.valueOf("testMergingRegionWhileTakingSnapshot"); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + List tableRegions = createTable(tableName); + + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + + SnapshotDescription snapshot = + new SnapshotDescription("SnapshotProcedureTest", tableName, SnapshotType.FLUSH); + SnapshotProtos.SnapshotDescription snapshotProto = + ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); + snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto, + UTIL.getHBaseCluster().getMaster().getConfiguration()); + long snapshotProcId = procExec.submitProcedure( + new TestSnapshotProcedure.DelaySnapshotProcedure(procExec.getEnvironment(), snapshotProto)); + UTIL.getHBaseCluster().getMaster().getSnapshotManager().registerSnapshotProcedure(snapshotProto, + snapshotProcId); + + RegionInfo[] regionsToMerge = new RegionInfo[2]; + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long mergeProcId = procExec.submitProcedure( + new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); + + ProcedureTestingUtility + .waitProcedure(UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(), mergeProcId); + ProcedureTestingUtility.waitProcedure( + UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(), snapshotProcId); + + assertProcFailed(procExec, mergeProcId); + assertEquals(initialRegionCount, UTIL.getAdmin().getRegions(tableName).size()); + } + private List createTable(final TableName tableName) throws Exception { TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java index 2757e0dd9f20..2f88f6087dd4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureBackoff.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FutureUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -61,11 +62,11 @@ public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion } @Override - void persistToMeta(RegionStateNode regionNode) throws IOException { + CompletableFuture persistToMeta(RegionStateNode regionNode) { if (FAIL) { - throw new IOException("Inject Error!"); + return FutureUtils.failedFuture(new IOException("Inject Error!")); } - super.persistToMeta(regionNode); + return super.persistToMeta(regionNode); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java index 1d13912fb72c..05179c5eadb7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRaceBetweenSCPAndTRSP.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; @@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -76,8 +78,14 @@ public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion } @Override - void regionOpening(RegionStateNode regionNode) throws IOException { - super.regionOpening(regionNode); + CompletableFuture regionOpening(RegionStateNode regionNode) { + CompletableFuture future = super.regionOpening(regionNode); + try { + // wait until the operation done, then trigger later processing, to make the test more + // stable + FutureUtils.get(future); + } catch (IOException e) { + } if (regionNode.getRegionInfo().getTable().equals(NAME) && ARRIVE_REGION_OPENING != null) { ARRIVE_REGION_OPENING.countDown(); ARRIVE_REGION_OPENING = null; @@ -86,6 +94,7 @@ void regionOpening(RegionStateNode regionNode) throws IOException { } catch (InterruptedException e) { } } + return future; } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java index 61520873240c..8295da82f49c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionBypass.java @@ -125,7 +125,7 @@ public void testBypass() throws IOException, InterruptedException { .getMasterProcedureExecutor().getActiveProcIds().isEmpty()); // Now assign with the override flag. for (RegionInfo ri : regions) { - TEST_UTIL.getHbck().assigns(Arrays. asList(ri.getEncodedName()), true); + TEST_UTIL.getHbck().assigns(Arrays. asList(ri.getEncodedName()), true, true); } TEST_UTIL.waitFor(60000, () -> TEST_UTIL.getHBaseCluster().getMaster() .getMasterProcedureExecutor().getActiveProcIds().isEmpty()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateNodeLock.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateNodeLock.java new file mode 100644 index 000000000000..d61fdcfce3bd --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateNodeLock.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.assignment; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.AtomicUtils; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, SmallTests.class }) +public class TestRegionStateNodeLock { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionStateNodeLock.class); + + private final RegionInfo regionInfo = + RegionInfoBuilder.newBuilder(TableName.valueOf("test")).build(); + + private RegionStateNodeLock lock; + + @Before + public void setUp() { + lock = new RegionStateNodeLock(regionInfo); + } + + @Test + public void testLockByThread() { + assertFalse(lock.isLocked()); + assertFalse(lock.isLockedBy(Thread.currentThread())); + assertThrows(IllegalMonitorStateException.class, () -> lock.unlock()); + lock.lock(); + assertTrue(lock.isLocked()); + assertTrue(lock.isLockedBy(Thread.currentThread())); + assertFalse(lock.isLockedBy(new Object())); + // reentrant + assertTrue(lock.tryLock()); + lock.unlock(); + assertTrue(lock.isLocked()); + lock.unlock(); + assertFalse(lock.isLocked()); + } + + @Test + public void testLockByProc() throws ProcedureSuspendedException { + NoopProcedure proc = new NoopProcedure(); + assertFalse(lock.isLocked()); + assertFalse(lock.isLockedBy(proc)); + assertThrows(IllegalMonitorStateException.class, () -> lock.unlock(proc)); + // here we do not need wake up + lock.lock(proc, null); + assertTrue(lock.isLocked()); + assertTrue(lock.isLockedBy(proc)); + // reentrant + assertTrue(lock.tryLock(proc)); + lock.unlock(proc); + assertTrue(lock.isLocked()); + assertTrue(lock.isLockedBy(proc)); + lock.unlock(proc); + assertFalse(lock.isLocked()); + assertFalse(lock.isLockedBy(proc)); + } + + @Test + public void testLockProcThenThread() throws ProcedureSuspendedException { + NoopProcedure proc = new NoopProcedure(); + assertFalse(lock.isLocked()); + lock.lock(proc, null); + assertFalse(lock.tryLock()); + assertThrows(IllegalMonitorStateException.class, () -> lock.unlock()); + long startNs = System.nanoTime(); + new Thread(() -> { + Threads.sleepWithoutInterrupt(2000); + lock.unlock(proc); + }).start(); + lock.lock(); + long costNs = System.nanoTime() - startNs; + assertThat(TimeUnit.NANOSECONDS.toMillis(costNs), greaterThanOrEqualTo(1800L)); + assertTrue(lock.isLocked()); + lock.unlock(); + assertFalse(lock.isLocked()); + } + + @Test + public void testLockThreadThenProc() throws ProcedureSuspendedException { + lock.lock(); + NoopProcedure proc = new NoopProcedure(); + Runnable wakeUp = mock(Runnable.class); + assertThrows(ProcedureSuspendedException.class, () -> lock.lock(proc, wakeUp)); + lock.unlock(); + // make sure that we have woken up the procedure, and the lock has been passed + verify(wakeUp).run(); + assertTrue(lock.isLockedBy(proc)); + } + + @Test + public void testLockMultiThread() throws InterruptedException { + int nThreads = 10; + AtomicLong concurrency = new AtomicLong(0); + AtomicLong maxConcurrency = new AtomicLong(0); + Thread[] threads = new Thread[nThreads]; + for (int i = 0; i < nThreads; i++) { + threads[i] = new Thread(() -> { + for (int j = 0; j < 1000; j++) { + lock.lock(); + try { + long c = concurrency.incrementAndGet(); + AtomicUtils.updateMax(maxConcurrency, c); + concurrency.decrementAndGet(); + } finally { + lock.unlock(); + } + Threads.sleepWithoutInterrupt(1); + } + }); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + assertEquals(0, concurrency.get()); + assertEquals(1, maxConcurrency.get()); + assertFalse(lock.isLocked()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRollbackSCP.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRollbackSCP.java index 3d1a2c4caa94..cd73e09af6db 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRollbackSCP.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRollbackSCP.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -46,6 +47,7 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FutureUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; @@ -85,7 +87,7 @@ public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion } @Override - void persistToMeta(RegionStateNode regionNode) throws IOException { + CompletableFuture persistToMeta(RegionStateNode regionNode) { TransitRegionStateProcedure proc = regionNode.getProcedure(); if (!regionNode.getRegionInfo().isMetaRegion() && proc.hasParent()) { Procedure p = @@ -96,10 +98,10 @@ void persistToMeta(RegionStateNode regionNode) throws IOException { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdateInRollback( getMaster().getMasterProcedureExecutor(), true); } - throw new RuntimeException("inject code bug"); + return FutureUtils.failedFuture(new RuntimeException("inject code bug")); } } - super.persistToMeta(regionNode); + return super.persistToMeta(regionNode); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java index 6e25dbab48ce..6ec36e75bea2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java @@ -41,6 +41,8 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.SnapshotType; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; @@ -50,10 +52,12 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; +import org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -69,6 +73,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; + @Category({ MasterTests.class, MediumTests.class }) public class TestSplitTableRegionProcedure { @@ -550,6 +557,52 @@ public void testSplitWithoutPONR() throws Exception { verify(tableName, splitRowNum); } + @Test + public void testSplitRegionWhileTakingSnapshot() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, + columnFamilyName1, columnFamilyName2); + int splitRowNum = startRowNum + rowCount / 2; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + + // task snapshot + SnapshotDescription snapshot = + new SnapshotDescription("SnapshotProcedureTest", tableName, SnapshotType.FLUSH); + SnapshotProtos.SnapshotDescription snapshotProto = + ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); + snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto, + UTIL.getHBaseCluster().getMaster().getConfiguration()); + long snapshotProcId = procExec.submitProcedure( + new TestSnapshotProcedure.DelaySnapshotProcedure(procExec.getEnvironment(), snapshotProto)); + UTIL.getHBaseCluster().getMaster().getSnapshotManager().registerSnapshotProcedure(snapshotProto, + snapshotProcId); + + // collect AM metrics before test + collectAssignmentManagerMetrics(); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.waitProcedure(procExec, snapshotProcId); + + ProcedureTestingUtility.assertProcFailed(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, snapshotProcId); + + assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 1); + assertTrue(UTIL.countRows(tableName) == 0); + + assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); + assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount()); + } + private void deleteData(final TableName tableName, final int startDeleteRowNum) throws IOException, InterruptedException { Table t = UTIL.getConnection().getTable(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerWorkQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerWorkQueue.java index c6d14c191145..088df7e7376e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerWorkQueue.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestRegionNormalizerWorkQueue.java @@ -22,7 +22,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.util.ArrayList; @@ -41,6 +40,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.ClassRule; @@ -186,6 +187,7 @@ public void testTake() throws Exception { final RegionNormalizerWorkQueue queue = new RegionNormalizerWorkQueue<>(); final ConcurrentLinkedQueue takeTimes = new ConcurrentLinkedQueue<>(); final AtomicBoolean finished = new AtomicBoolean(false); + final int count = 5; final Runnable consumer = () -> { try { while (!finished.get()) { @@ -199,11 +201,12 @@ public void testTake() throws Exception { CompletableFuture worker = CompletableFuture.runAsync(consumer); final long testStart = System.nanoTime(); - for (int i = 0; i < 5; i++) { + for (int i = 0; i < count; i++) { Thread.sleep(10); queue.put(i); } - + // should have timing information for 5 calls to take. + Waiter.waitFor(HBaseConfiguration.create(), 1000, () -> count == takeTimes.size()); // set finished = true and pipe one more value in case the thread needs an extra pass through // the loop. finished.set(true); @@ -211,9 +214,7 @@ public void testTake() throws Exception { worker.get(1, TimeUnit.SECONDS); final Iterator times = takeTimes.iterator(); - assertTrue("should have timing information for at least 2 calls to take.", - takeTimes.size() >= 5); - for (int i = 0; i < 5; i++) { + for (int i = 0; i < count; i++) { assertThat( "Observations collected in takeTimes should increase by roughly 10ms every interval", times.next(), greaterThan(testStart + TimeUnit.MILLISECONDS.toNanos(i * 10))); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index 8b7bd8d92e7f..9456849bd37b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -87,6 +87,7 @@ public static void restartMasterProcedureExecutor(ProcedureExecutor() { @Override public Void call() throws Exception { + master.setServiceStarted(false); AssignmentManager am = env.getAssignmentManager(); // try to simulate a master restart by removing the ServerManager states about seqIDs for (RegionState regionState : am.getRegionStates().getRegionStates()) { @@ -109,6 +110,10 @@ public Void call() throws Exception { am.setupRIT(procExec.getActiveProceduresNoCopy().stream().filter(p -> !p.isSuccess()) .filter(p -> p instanceof TransitRegionStateProcedure) .map(p -> (TransitRegionStateProcedure) p).collect(Collectors.toList())); + // create server state node, to simulate master start up + env.getMasterServices().getServerManager().getOnlineServersList() + .forEach(am.getRegionStates()::createServer); + master.setServiceStarted(true); return null; } }, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestManageTableErasureCodingPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestManageTableErasureCodingPolicy.java new file mode 100644 index 000000000000..34a77229dcad --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestManageTableErasureCodingPolicy.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThrows; + +import java.io.IOException; +import java.util.function.Function; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.util.TableDescriptorChecker; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestManageTableErasureCodingPolicy { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestManageTableErasureCodingPolicy.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestManageTableErasureCodingPolicy.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + private static final byte[] FAMILY = Bytes.toBytes("a"); + private static final TableName NON_EC_TABLE = TableName.valueOf("foo"); + private static final TableDescriptor NON_EC_TABLE_DESC = TableDescriptorBuilder + .newBuilder(NON_EC_TABLE).setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(); + private static final TableName EC_TABLE = TableName.valueOf("bar"); + private static final TableDescriptor EC_TABLE_DESC = + TableDescriptorBuilder.newBuilder(EC_TABLE).setErasureCodingPolicy("XOR-2-1-1024k") + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(); + + @BeforeClass + public static void beforeClass() throws Exception { + // enable because we are testing the checks below + UTIL.getConfiguration().setBoolean(TableDescriptorChecker.TABLE_SANITY_CHECKS, true); + UTIL.startMiniDFSCluster(3); // 3 necessary for XOR-2-1-1024k + UTIL.startMiniCluster(1); + DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration()); + fs.enableErasureCodingPolicy("XOR-2-1-1024k"); + fs.enableErasureCodingPolicy("RS-6-3-1024k"); + Table table = UTIL.createTable(NON_EC_TABLE_DESC, null); + UTIL.loadTable(table, FAMILY); + UTIL.flush(); + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + UTIL.shutdownMiniDFSCluster(); + } + + @Test + public void itValidatesPolicyNameForCreate() { + runValidatePolicyNameTest(unused -> EC_TABLE_DESC, Admin::createTable); + } + + @Test + public void itValidatesPolicyNameForAlter() { + runValidatePolicyNameTest(admin -> { + try { + return admin.getDescriptor(NON_EC_TABLE); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, Admin::modifyTable); + } + + @FunctionalInterface + interface ThrowingTableDescriptorConsumer { + void accept(Admin admin, TableDescriptor desc) throws IOException; + } + + private void runValidatePolicyNameTest(Function descriptorSupplier, + ThrowingTableDescriptorConsumer consumer) { + HBaseIOException thrown = assertThrows(HBaseIOException.class, () -> { + try (Admin admin = UTIL.getAdmin()) { + TableDescriptor desc = descriptorSupplier.apply(admin); + consumer.accept(admin, + TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("foo").build()); + } + }); + assertThat(thrown.getMessage(), + containsString("Cannot set Erasure Coding policy: foo. Policy not found")); + + thrown = assertThrows(HBaseIOException.class, () -> { + try (Admin admin = UTIL.getAdmin()) { + TableDescriptor desc = descriptorSupplier.apply(admin); + consumer.accept(admin, + TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("RS-10-4-1024k").build()); + } + }); + assertThat(thrown.getMessage(), containsString( + "Cannot set Erasure Coding policy: RS-10-4-1024k. The policy must be enabled")); + + // RS-6-3-1024k requires at least 6 datanodes, so should fail write test + thrown = assertThrows(HBaseIOException.class, () -> { + try (Admin admin = UTIL.getAdmin()) { + TableDescriptor desc = descriptorSupplier.apply(admin); + consumer.accept(admin, + TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("RS-6-3-1024k").build()); + } + }); + assertThat(thrown.getMessage(), containsString("Failed write test for EC policy")); + } + + @Test + public void testCreateTableErasureCodingSync() throws IOException { + try (Admin admin = UTIL.getAdmin()) { + recreateTable(admin, EC_TABLE_DESC); + UTIL.flush(EC_TABLE); + Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()); + DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration()); + checkRegionDirAndFilePolicies(dfs, rootDir, EC_TABLE, "XOR-2-1-1024k", "XOR-2-1-1024k"); + } + } + + private void recreateTable(Admin admin, TableDescriptor desc) throws IOException { + if (admin.tableExists(desc.getTableName())) { + admin.disableTable(desc.getTableName()); + admin.deleteTable(desc.getTableName()); + } + admin.createTable(desc); + try (Table table = UTIL.getConnection().getTable(desc.getTableName())) { + UTIL.loadTable(table, FAMILY); + } + } + + @Test + public void testModifyTableErasureCodingSync() throws IOException, InterruptedException { + try (Admin admin = UTIL.getAdmin()) { + Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()); + DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration()); + + // start off without EC + checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, null, null); + + // add EC + TableDescriptor desc = UTIL.getAdmin().getDescriptor(NON_EC_TABLE); + TableDescriptor newDesc = + TableDescriptorBuilder.newBuilder(desc).setErasureCodingPolicy("XOR-2-1-1024k").build(); + admin.modifyTable(newDesc); + + // check dirs, but files should not be changed yet + checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, "XOR-2-1-1024k", null); + + compactAwayOldFiles(NON_EC_TABLE); + + // expect both dirs and files to be EC now + checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, "XOR-2-1-1024k", "XOR-2-1-1024k"); + + newDesc = TableDescriptorBuilder.newBuilder(newDesc).setErasureCodingPolicy(null).build(); + // remove EC now + admin.modifyTable(newDesc); + + // dirs should no longer be EC, but old EC files remain + checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, null, "XOR-2-1-1024k"); + + // compact to rewrite EC files without EC, then run discharger to get rid of the old EC files + UTIL.compact(NON_EC_TABLE, true); + for (JVMClusterUtil.RegionServerThread regionserver : UTIL.getHBaseCluster() + .getLiveRegionServerThreads()) { + CompactedHFilesDischarger chore = + regionserver.getRegionServer().getCompactedHFilesDischarger(); + chore.setUseExecutor(false); + chore.chore(); + } + + checkRegionDirAndFilePolicies(dfs, rootDir, NON_EC_TABLE, null, null); + } + } + + private void compactAwayOldFiles(TableName tableName) throws IOException { + LOG.info("Compacting and discharging files for {}", tableName); + // compact to rewrit files, then run discharger to get rid of the old files + UTIL.compact(tableName, true); + for (JVMClusterUtil.RegionServerThread regionserver : UTIL.getHBaseCluster() + .getLiveRegionServerThreads()) { + CompactedHFilesDischarger chore = + regionserver.getRegionServer().getCompactedHFilesDischarger(); + chore.setUseExecutor(false); + chore.chore(); + } + } + + @Test + public void testRestoreSnapshot() throws IOException { + String snapshotName = "testRestoreSnapshot_snap"; + TableName tableName = TableName.valueOf("testRestoreSnapshot_tbl"); + try (Admin admin = UTIL.getAdmin()) { + Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()); + DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(UTIL.getConfiguration()); + + // recreate EC test table and load it + recreateTable(admin, EC_TABLE_DESC); + + // Take a snapshot, then clone it into a new table + admin.snapshot(snapshotName, EC_TABLE); + admin.cloneSnapshot(snapshotName, tableName); + compactAwayOldFiles(tableName); + + // Verify the new table has the right EC policy + checkRegionDirAndFilePolicies(dfs, rootDir, tableName, "XOR-2-1-1024k", "XOR-2-1-1024k"); + + // Remove the EC policy from the EC test table, and verify that worked + admin.modifyTable( + TableDescriptorBuilder.newBuilder(EC_TABLE_DESC).setErasureCodingPolicy(null).build()); + compactAwayOldFiles(EC_TABLE); + checkRegionDirAndFilePolicies(dfs, rootDir, EC_TABLE, null, null); + + // Restore snapshot, and then verify it has the policy again + admin.disableTable(EC_TABLE); + admin.restoreSnapshot(snapshotName); + admin.enableTable(EC_TABLE); + compactAwayOldFiles(EC_TABLE); + checkRegionDirAndFilePolicies(dfs, rootDir, EC_TABLE, "XOR-2-1-1024k", "XOR-2-1-1024k"); + } + } + + private void checkRegionDirAndFilePolicies(DistributedFileSystem dfs, Path rootDir, + TableName testTable, String expectedDirPolicy, String expectedFilePolicy) throws IOException { + Path tableDir = CommonFSUtils.getTableDir(rootDir, testTable); + checkPolicy(dfs, tableDir, expectedDirPolicy); + + int filesMatched = 0; + for (HRegion region : UTIL.getHBaseCluster().getRegions(testTable)) { + Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); + checkPolicy(dfs, regionDir, expectedDirPolicy); + RemoteIterator itr = dfs.listFiles(regionDir, true); + while (itr.hasNext()) { + LocatedFileStatus fileStatus = itr.next(); + Path path = fileStatus.getPath(); + if (!HFile.isHFileFormat(dfs, path)) { + LOG.info("{} is not an hfile", path); + continue; + } + filesMatched++; + checkPolicy(dfs, path, expectedFilePolicy); + } + } + assertThat(filesMatched, greaterThan(0)); + } + + private void checkPolicy(DistributedFileSystem dfs, Path path, String expectedPolicy) + throws IOException { + ErasureCodingPolicy policy = dfs.getErasureCodingPolicy(path); + if (expectedPolicy == null) { + assertThat("policy for " + path, policy, nullValue()); + } else { + assertThat("policy for " + path, policy, notNullValue()); + assertThat("policy for " + path, policy.getName(), equalTo(expectedPolicy)); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java index 3319a761eb4c..d1e7dc147615 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedurePriority.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -55,10 +56,16 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; + /** * Test to ensure that the priority for procedures and stuck checker can partially solve the problem * describe in HBASE-19976, that is, RecoverMetaProcedure can finally be executed within a certain * period of time. + *

+ * As of HBASE-28199, we no longer block a worker when updating meta now, so this test can not test + * adding procedure worker now, but it could still be used to make sure that we could make progress + * when meta is gone and we have a lot of pending TRSPs. */ @Category({ MasterTests.class, LargeTests.class }) public class TestProcedurePriority { @@ -129,6 +136,7 @@ public static void setUp() throws Exception { } UTIL.getAdmin().balance(BalanceRequest.newBuilder().setIgnoreRegionsInTransition(true).build()); UTIL.waitUntilNoRegionsInTransition(); + UTIL.getAdmin().balancerSwitch(false, true); } @AfterClass @@ -144,22 +152,26 @@ public void test() throws Exception { HRegionServer rsNoMeta = UTIL.getOtherRegionServer(rsWithMetaThread.getRegionServer()); FAIL = true; UTIL.getMiniHBaseCluster().killRegionServer(rsNoMeta.getServerName()); - // wait until all the worker thread are stuck, which means that the stuck checker will start to - // add new worker thread. ProcedureExecutor executor = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); + // wait until we have way more TRSPs than the core pool size, and then make sure we can recover + // normally UTIL.waitFor(60000, new ExplainingPredicate() { @Override public boolean evaluate() throws Exception { - return executor.getWorkerThreadCount() > CORE_POOL_SIZE; + return executor.getProcedures().stream().filter(p -> !p.isFinished()) + .filter(p -> p.getState() != ProcedureState.INITIALIZING) + .filter(p -> p instanceof TransitRegionStateProcedure).count() > 5 * CORE_POOL_SIZE; } @Override public String explainFailure() throws Exception { - return "Stuck checker does not add new worker thread"; + return "Not enough TRSPs scheduled"; } }); + // sleep more time to make sure the TRSPs have been executed + Thread.sleep(10000); UTIL.getMiniHBaseCluster().killRegionServer(rsWithMetaThread.getRegionServer().getServerName()); rsWithMetaThread.join(); FAIL = false; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureBatchBackoff.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureBatchBackoff.java new file mode 100644 index 000000000000..fbabb1fa22cc --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureBatchBackoff.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Confirm that we will rate limit reopen batches when reopening all table regions. This can avoid + * the pain associated with reopening too many regions at once. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestReopenTableRegionsProcedureBatchBackoff { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestReopenTableRegionsProcedureBatchBackoff.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static TableName TABLE_NAME = TableName.valueOf("BatchBackoff"); + private static final int BACKOFF_MILLIS_PER_RS = 3_000; + private static final int REOPEN_BATCH_SIZE = 1; + + private static byte[] CF = Bytes.toBytes("cf"); + + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + UTIL.startMiniCluster(1); + UTIL.createMultiRegionTable(TABLE_NAME, CF, 10); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testRegionBatchBackoff() throws IOException { + ProcedureExecutor procExec = + UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); + List regions = UTIL.getAdmin().getRegions(TABLE_NAME); + assertTrue(10 <= regions.size()); + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(TABLE_NAME, BACKOFF_MILLIS_PER_RS, REOPEN_BATCH_SIZE); + procExec.submitProcedure(proc); + Instant startedAt = Instant.now(); + ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, 60_000); + Instant stoppedAt = Instant.now(); + assertTrue(Duration.between(startedAt, stoppedAt).toMillis() + > (long) regions.size() * BACKOFF_MILLIS_PER_RS); + } + + @Test + public void testRegionBatchNoBackoff() throws IOException { + ProcedureExecutor procExec = + UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); + List regions = UTIL.getAdmin().getRegions(TABLE_NAME); + assertTrue(10 <= regions.size()); + int noBackoffMillis = 0; + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(TABLE_NAME, noBackoffMillis, REOPEN_BATCH_SIZE); + procExec.submitProcedure(proc); + ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, + (long) regions.size() * BACKOFF_MILLIS_PER_RS); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureBatching.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureBatching.java new file mode 100644 index 000000000000..8ea9b3c6a309 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureBatching.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.RegionState.State; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.RegionStateNode; +import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; + +/** + * Confirm that we will batch region reopens when reopening all table regions. This can avoid the + * pain associated with reopening too many regions at once. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestReopenTableRegionsProcedureBatching { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestReopenTableRegionsProcedureBatching.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + private static final int BACKOFF_MILLIS_PER_RS = 0; + private static final int REOPEN_BATCH_SIZE_MAX = 1; + + private static TableName TABLE_NAME = TableName.valueOf("Batching"); + + private static byte[] CF = Bytes.toBytes("cf"); + + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + UTIL.startMiniCluster(1); + UTIL.createMultiRegionTable(TABLE_NAME, CF); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testSmallMaxBatchSize() throws IOException { + AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager(); + ProcedureExecutor procExec = + UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); + List regions = UTIL.getAdmin().getRegions(TABLE_NAME); + assertTrue(2 <= regions.size()); + Set stuckRegions = + regions.stream().map(r -> stickRegion(am, procExec, r)).collect(Collectors.toSet()); + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(TABLE_NAME, BACKOFF_MILLIS_PER_RS, REOPEN_BATCH_SIZE_MAX); + procExec.submitProcedure(proc); + UTIL.waitFor(10000, () -> proc.getState() == ProcedureState.WAITING_TIMEOUT); + + // the first batch should be small + confirmBatchSize(1, stuckRegions, proc); + ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, 60_000); + + // other batches should also be small + assertTrue(proc.getBatchesProcessed() >= regions.size()); + + // all regions should only be opened once + assertEquals(proc.getRegionsReopened(), regions.size()); + } + + @Test + public void testDefaultMaxBatchSize() throws IOException { + AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager(); + ProcedureExecutor procExec = + UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); + List regions = UTIL.getAdmin().getRegions(TABLE_NAME); + assertTrue(2 <= regions.size()); + Set stuckRegions = + regions.stream().map(r -> stickRegion(am, procExec, r)).collect(Collectors.toSet()); + ReopenTableRegionsProcedure proc = new ReopenTableRegionsProcedure(TABLE_NAME); + procExec.submitProcedure(proc); + UTIL.waitFor(10000, () -> proc.getState() == ProcedureState.WAITING_TIMEOUT); + + // the first batch should be large + confirmBatchSize(regions.size(), stuckRegions, proc); + ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, 60_000); + + // all regions should only be opened once + assertEquals(proc.getRegionsReopened(), regions.size()); + } + + @Test + public void testNegativeBatchSizeDoesNotBreak() throws IOException { + AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager(); + ProcedureExecutor procExec = + UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); + List regions = UTIL.getAdmin().getRegions(TABLE_NAME); + assertTrue(2 <= regions.size()); + Set stuckRegions = + regions.stream().map(r -> stickRegion(am, procExec, r)).collect(Collectors.toSet()); + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(TABLE_NAME, BACKOFF_MILLIS_PER_RS, -100); + procExec.submitProcedure(proc); + UTIL.waitFor(10000, () -> proc.getState() == ProcedureState.WAITING_TIMEOUT); + + // the first batch should be small + confirmBatchSize(1, stuckRegions, proc); + ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, 60_000); + + // other batches should also be small + assertTrue(proc.getBatchesProcessed() >= regions.size()); + + // all regions should only be opened once + assertEquals(proc.getRegionsReopened(), regions.size()); + } + + @Test + public void testBatchSizeDoesNotOverflow() { + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(TABLE_NAME, BACKOFF_MILLIS_PER_RS, Integer.MAX_VALUE); + int currentBatchSize = 1; + while (currentBatchSize < Integer.MAX_VALUE) { + currentBatchSize = proc.progressBatchSize(); + assertTrue(currentBatchSize > 0); + } + } + + private void confirmBatchSize(int expectedBatchSize, Set stuckRegions, + ReopenTableRegionsProcedure proc) { + while (true) { + if (proc.getBatchesProcessed() == 0) { + continue; + } + stuckRegions.forEach(this::unstickRegion); + UTIL.waitFor(5000, () -> expectedBatchSize == proc.getRegionsReopened()); + break; + } + } + + static class StuckRegion { + final TransitRegionStateProcedure trsp; + final RegionStateNode regionNode; + final long openSeqNum; + + public StuckRegion(TransitRegionStateProcedure trsp, RegionStateNode regionNode, + long openSeqNum) { + this.trsp = trsp; + this.regionNode = regionNode; + this.openSeqNum = openSeqNum; + } + } + + private StuckRegion stickRegion(AssignmentManager am, + ProcedureExecutor procExec, RegionInfo regionInfo) { + RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(regionInfo); + TransitRegionStateProcedure trsp = + TransitRegionStateProcedure.unassign(procExec.getEnvironment(), regionInfo); + regionNode.lock(); + long openSeqNum; + try { + openSeqNum = regionNode.getOpenSeqNum(); + regionNode.setState(State.OPENING); + regionNode.setOpenSeqNum(-1L); + regionNode.setProcedure(trsp); + } finally { + regionNode.unlock(); + } + return new StuckRegion(trsp, regionNode, openSeqNum); + } + + private void unstickRegion(StuckRegion stuckRegion) { + stuckRegion.regionNode.lock(); + try { + stuckRegion.regionNode.setState(State.OPEN); + stuckRegion.regionNode.setOpenSeqNum(stuckRegion.openSeqNum); + stuckRegion.regionNode.unsetProcedure(stuckRegion.trsp); + } finally { + stuckRegion.regionNode.unlock(); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java index f482c5441d6a..b7c2d5099879 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerRemoteProcedure.java @@ -88,7 +88,7 @@ public void setUp() throws Exception { master.start(2, rsDispatcher); am = master.getAssignmentManager(); master.getServerManager().getOnlineServersList().stream() - .forEach(serverName -> am.getRegionStates().getOrCreateServer(serverName)); + .forEach(serverName -> am.getRegionStates().createServer(serverName)); } @After diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java index 04442eb771d8..84a3e84763b6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.hbase.master.procedure; +import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SnapshotState.SNAPSHOT_SNAPSHOT_ONLINE_REGIONS; import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Optional; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; @@ -51,6 +53,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SnapshotState; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; @@ -70,6 +73,27 @@ public class TestSnapshotProcedure { protected SnapshotDescription snapshot; protected SnapshotProtos.SnapshotDescription snapshotProto; + public static final class DelaySnapshotProcedure extends SnapshotProcedure { + public DelaySnapshotProcedure() { + } + + public DelaySnapshotProcedure(final MasterProcedureEnv env, + final SnapshotProtos.SnapshotDescription snapshot) { + super(env, snapshot); + } + + @Override + protected Flow executeFromState(MasterProcedureEnv env, + MasterProcedureProtos.SnapshotState state) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + Flow flow = super.executeFromState(env, state); + if (state == SNAPSHOT_SNAPSHOT_ONLINE_REGIONS) { + TimeUnit.SECONDS.sleep(20); + } + return flow; + } + } + @Before public void setup() throws Exception { TEST_UTIL = new HBaseTestingUtil(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureWithLockTimeout.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureWithLockTimeout.java new file mode 100644 index 000000000000..f780d68acf49 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureWithLockTimeout.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.SnapshotType; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.snapshot.TakeSnapshotHandler; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.RegionSplitter; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; + +/** + * Snapshot creation with master lock timeout test. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestSnapshotProcedureWithLockTimeout { + + private static final Logger LOG = + LoggerFactory.getLogger(TestSnapshotProcedureWithLockTimeout.class); + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSnapshotProcedureWithLockTimeout.class); + + private static HBaseTestingUtil TEST_UTIL; + private HMaster master; + private TableName TABLE_NAME; + private byte[] CF; + private String SNAPSHOT_NAME; + + @Before + public void setup() throws Exception { + TEST_UTIL = new HBaseTestingUtil(); + Configuration config = TEST_UTIL.getConfiguration(); + config.setInt("hbase.snapshot.remote.verify.threshold", 1); + config.setLong(TakeSnapshotHandler.HBASE_SNAPSHOT_MASTER_LOCK_ACQUIRE_TIMEOUT, 1L); + TEST_UTIL.startMiniCluster(3); + master = TEST_UTIL.getHBaseCluster().getMaster(); + TABLE_NAME = TableName.valueOf(Bytes.toBytes("TestSnapshotProcedureWithLockTimeout")); + CF = Bytes.toBytes("cf"); + SNAPSHOT_NAME = "SnapshotProcLockTimeout"; + final byte[][] splitKeys = new RegionSplitter.HexStringSplit().split(10); + Table table = TEST_UTIL.createTable(TABLE_NAME, CF, splitKeys); + TEST_UTIL.loadTable(table, CF, false); + } + + @After + public void teardown() throws Exception { + if (this.master != null) { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(master.getMasterProcedureExecutor(), + false); + } + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testTakeZkCoordinatedSnapshot() { + for (int i = 0; i < 10; i++) { + try { + // Verify that snapshot creation is not possible because lock could not be + // acquired on time. This can be flaky behavior because even though we provide 1ms + // as lock timeout, it could still be fast enough and eventually lead to successful + // snapshot creation. If that happens, retry again. + testTakeZkCoordinatedSnapshot(i); + break; + } catch (Exception e) { + LOG.error("Error because of faster lock acquisition. retrying....", e); + } + assertNotEquals("Retries exhausted", 9, i); + } + } + + private void testTakeZkCoordinatedSnapshot(int i) throws Exception { + SnapshotDescription snapshotOnSameTable = + new SnapshotDescription(SNAPSHOT_NAME + i, TABLE_NAME, SnapshotType.SKIPFLUSH); + SnapshotProtos.SnapshotDescription snapshotOnSameTableProto = + ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotOnSameTable); + Thread second = new Thread("zk-snapshot") { + @Override + public void run() { + try { + master.getSnapshotManager().takeSnapshot(snapshotOnSameTableProto); + } catch (IOException e) { + LOG.error("zk snapshot failed", e); + fail("zk snapshot failed"); + } + } + }; + second.start(); + + Thread.sleep(5000); + boolean snapshotCreated = false; + try { + SnapshotTestingUtils.confirmSnapshotValid(TEST_UTIL, snapshotOnSameTableProto, TABLE_NAME, + CF); + snapshotCreated = true; + } catch (AssertionError e) { + LOG.error("Assertion error..", e); + if ( + e.getMessage() != null && e.getMessage().contains("target snapshot directory") + && e.getMessage().contains("doesn't exist.") + ) { + LOG.debug("Expected behaviour - snapshot could not be created"); + } else { + throw new IOException(e); + } + } + + if (snapshotCreated) { + throw new IOException("Snapshot created successfully"); + } + + // ensure all scheduled procedures are successfully completed + TEST_UTIL.waitFor(4000, 400, + () -> master.getMasterProcedureExecutor().getProcedures().stream() + .filter(masterProcedureEnvProcedure -> masterProcedureEnvProcedure.getState() + == ProcedureProtos.ProcedureState.SUCCESS) + .count() == master.getMasterProcedureExecutor().getProcedures().size()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSuspendTRSPWhenHoldingRegionStateNodeLock.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSuspendTRSPWhenHoldingRegionStateNodeLock.java new file mode 100644 index 000000000000..39427d25b100 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSuspendTRSPWhenHoldingRegionStateNodeLock.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.everyItem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.assignment.RegionStateNode; +import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; + +/** + * Testcase for HBASE-28240. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestSuspendTRSPWhenHoldingRegionStateNodeLock { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSuspendTRSPWhenHoldingRegionStateNodeLock.class); + + private static final HBaseTestingUtil HBTU = new HBaseTestingUtil(); + + private static TableName TABLE_NAME = TableName.valueOf("test"); + + private static byte[] FAMILY = Bytes.toBytes("family"); + + @BeforeClass + public static void setUp() throws Exception { + HBTU.startMiniCluster(2); + HBTU.createTable(TABLE_NAME, FAMILY); + HBTU.waitTableAvailable(TABLE_NAME); + HBTU.getAdmin().balancerSwitch(false, true); + HBTU.waitUntilNoRegionsInTransition(); + } + + @AfterClass + public static void tearDown() throws Exception { + HBTU.shutdownMiniCluster(); + } + + private Matcher> notChildOf(long procId) { + return new BaseMatcher>() { + + @Override + public boolean matches(Object item) { + if (!(item instanceof Procedure)) { + return false; + } + Procedure proc = (Procedure) item; + return !proc.hasParent() || proc.getRootProcId() != procId; + } + + @Override + public void describeTo(Description description) { + description.appendText("not a child of pid=").appendValue(procId); + } + }; + } + + @Test + public void testSuspend() throws Exception { + HMaster master = HBTU.getMiniHBaseCluster().getMaster(); + AssignmentManager am = master.getAssignmentManager(); + RegionInfo ri = Iterables.getOnlyElement(am.getTableRegions(TABLE_NAME, true)); + RegionStateNode rsn = am.getRegionStates().getRegionStateNode(ri); + + ServerName src = rsn.getRegionLocation(); + ServerName dst = HBTU.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(t -> t.getRegionServer().getServerName()).filter(sn -> !sn.equals(src)).findFirst() + .get(); + TransitRegionStateProcedure proc = am.createMoveRegionProcedure(ri, dst); + // lock the region state node manually, so later TRSP can not lock it + rsn.lock(); + ProcedureExecutor procExec = master.getMasterProcedureExecutor(); + long procId = procExec.submitProcedure(proc); + // sleep several seconds to let the procedure be scheduled + Thread.sleep(2000); + // wait until no active procedures + HBTU.waitFor(30000, () -> procExec.getActiveExecutorCount() == 0); + // the procedure should have not finished yet + assertFalse(proc.isFinished()); + // the TRSP should have not scheduled any sub procedures yet + assertThat(procExec.getProcedures(), everyItem(notChildOf(procId))); + // make sure the region is still on the src region server + assertEquals(src, HBTU.getRSForFirstRegionInTable(TABLE_NAME).getServerName()); + + // unlock the region state node lock, the TRSP should be woken up and finish the execution + rsn.unlock(); + HBTU.waitFor(30000, () -> proc.isFinished()); + // make sure the region is on the dst region server + assertEquals(dst, HBTU.getRSForFirstRegionInTable(TABLE_NAME).getServerName()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java index f282c6f9d8f4..4bbc88681290 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java @@ -140,7 +140,8 @@ public void testCleaner() throws Exception { assertEquals("Before cleanup without delay 1", 1, firstFiles.length); String firstFile = firstFiles[0].getPath().getName(); - ts = EnvironmentEdgeManager.currentTime() - 1 * secondsOfDay() * 1000; // 1 day before + // 1.5 day before + ts = (long) (EnvironmentEdgeManager.currentTime() - 1.5 * secondsOfDay() * 1000); putKVAndFlush(table, row2, dummyData, ts); FileStatus[] secondFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath); // now there are 2 mob files diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java index 35a1757115c9..124214e46af3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java @@ -22,6 +22,7 @@ import java.net.InetAddress; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; +import java.security.cert.X509Certificate; import java.util.Collections; import java.util.List; import java.util.Map; @@ -667,13 +668,13 @@ public void testOnlineSlowLogConnectionAttributes() throws Exception { static RpcLogDetails getRpcLogDetails(String userName, String clientAddress, String className, int forcedParamIndex) { RpcCall rpcCall = getRpcCall(userName, forcedParamIndex); - return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, 0, className, true, + return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, 0, 0, className, true, true); } static RpcLogDetails getRpcLogDetails(String userName, String clientAddress, String className) { RpcCall rpcCall = getRpcCall(userName); - return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, 0, className, true, + return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, 0, 0, className, true, true); } @@ -685,8 +686,8 @@ private static RpcLogDetails getRpcLogDetailsOfScan() { private RpcLogDetails getRpcLogDetails(String userName, String clientAddress, String className, boolean isSlowLog, boolean isLargeLog) { RpcCall rpcCall = getRpcCall(userName); - return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, 0, className, isSlowLog, - isLargeLog); + return new RpcLogDetails(rpcCall, rpcCall.getParam(), clientAddress, 0, 0, 0, className, + isSlowLog, isLargeLog); } private static RpcCall getRpcCall(String userName) { @@ -814,6 +815,11 @@ public Optional getRequestUser() { return getUser(userName); } + @Override + public Optional getClientCertificateChain() { + return Optional.empty(); + } + @Override public InetAddress getRemoteAddress() { return null; @@ -859,6 +865,16 @@ public long getResponseExceptionSize() { @Override public void incrementResponseExceptionSize(long exceptionSize) { } + + @Override + public void updateFsReadTime(long latencyMillis) { + + } + + @Override + public long getFsReadTime() { + return 0; + } }; return rpcCall; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java index 8a93f2d0ff54..1de0a0d31a33 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.InetAddress; import java.nio.ByteBuffer; +import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.Collections; import java.util.Map; @@ -80,7 +81,7 @@ public void itDeepCopiesRpcLogDetailsParams() throws IOException { ProtobufUtil.mergeFrom(messageBuilder, cis, buffer.capacity()); Message message = messageBuilder.build(); RpcLogDetails rpcLogDetails = - new RpcLogDetails(getRpcCall(message), message, null, 0L, 0L, null, true, false); + new RpcLogDetails(getRpcCall(message), message, null, 0L, 0L, 0, null, true, false); // log's scan should be equal ClientProtos.Scan logScan = ((ClientProtos.ScanRequest) rpcLogDetails.getParam()).getScan(); @@ -213,6 +214,11 @@ public Optional getRequestUser() { return null; } + @Override + public Optional getClientCertificateChain() { + return Optional.empty(); + } + @Override public InetAddress getRemoteAddress() { return null; @@ -258,6 +264,16 @@ public long getResponseExceptionSize() { @Override public void incrementResponseExceptionSize(long exceptionSize) { } + + @Override + public void updateFsReadTime(long latencyMillis) { + + } + + @Override + public long getFsReadTime() { + return 0; + } }; return rpcCall; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestTooLargeLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestTooLargeLog.java index da3d97547645..fdc3e288bfed 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestTooLargeLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestTooLargeLog.java @@ -118,6 +118,6 @@ public void testLogLargeBlockBytesScanned() throws IOException { record.getBlockBytesScanned() >= 100); assertTrue("expected " + record.getResponseSize() + " to be < 100", record.getResponseSize() < 100); - + assertTrue("expected " + record.getFsReadTime() + " to be > 0", record.getFsReadTime() > 0); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java index 305f0e29e952..fdd5c7d5cf90 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.security.cert.X509Certificate; import java.util.HashSet; import java.util.Map; import java.util.Optional; @@ -275,6 +276,11 @@ public Optional getRequestUser() { return Optional.empty(); } + @Override + public Optional getClientCertificateChain() { + return Optional.empty(); + } + @Override public InetAddress getRemoteAddress() { return null; @@ -320,6 +326,16 @@ public long getResponseExceptionSize() { @Override public void incrementResponseExceptionSize(long exceptionSize) { } + + @Override + public void updateFsReadTime(long latencyMillis) { + + } + + @Override + public long getFsReadTime() { + return 0; + } }; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestAtomicReadQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestAtomicReadQuota.java new file mode 100644 index 000000000000..9b654ac8e6d0 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestAtomicReadQuota.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.CheckAndMutate; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestAtomicReadQuota { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestAtomicReadQuota.class); + private static final Logger LOG = LoggerFactory.getLogger(TestAtomicReadQuota.class); + private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + private static final TableName TABLE_NAME = TableName.valueOf(UUID.randomUUID().toString()); + private static final byte[] FAMILY = Bytes.toBytes("cf"); + private static final byte[] QUALIFIER = Bytes.toBytes("q"); + + @AfterClass + public static void tearDown() throws Exception { + ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL); + EnvironmentEdgeManager.reset(); + TEST_UTIL.deleteTable(TABLE_NAME); + TEST_UTIL.shutdownMiniCluster(); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 1000); + TEST_UTIL.startMiniCluster(1); + TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); + TEST_UTIL.createTable(TABLE_NAME, FAMILY); + TEST_UTIL.waitTableAvailable(TABLE_NAME); + QuotaCache.TEST_FORCE_REFRESH = true; + } + + @Test + public void testIncrementCountedAgainstReadCapacity() throws Exception { + setupQuota(); + + Increment inc = new Increment(Bytes.toBytes(UUID.randomUUID().toString())); + inc.addColumn(FAMILY, QUALIFIER, 1); + testThrottle(table -> table.increment(inc)); + } + + @Test + public void testConditionalRowMutationsCountedAgainstReadCapacity() throws Exception { + setupQuota(); + + byte[] row = Bytes.toBytes(UUID.randomUUID().toString()); + Increment inc = new Increment(row); + inc.addColumn(FAMILY, Bytes.toBytes("doot"), 1); + Put put = new Put(row); + put.addColumn(FAMILY, Bytes.toBytes("doot"), Bytes.toBytes("v")); + + RowMutations rowMutations = new RowMutations(row); + rowMutations.add(inc); + rowMutations.add(put); + testThrottle(table -> table.mutateRow(rowMutations)); + } + + @Test + public void testNonConditionalRowMutationsOmittedFromReadCapacity() throws Exception { + setupQuota(); + + byte[] row = Bytes.toBytes(UUID.randomUUID().toString()); + Put put = new Put(row); + put.addColumn(FAMILY, Bytes.toBytes("doot"), Bytes.toBytes("v")); + + RowMutations rowMutations = new RowMutations(row); + rowMutations.add(put); + try (Table table = getTable()) { + for (int i = 0; i < 100; i++) { + table.mutateRow(rowMutations); + } + } + } + + @Test + public void testNonAtomicPutOmittedFromReadCapacity() throws Exception { + setupQuota(); + + byte[] row = Bytes.toBytes(UUID.randomUUID().toString()); + Put put = new Put(row); + put.addColumn(FAMILY, Bytes.toBytes("doot"), Bytes.toBytes("v")); + try (Table table = getTable()) { + for (int i = 0; i < 100; i++) { + table.put(put); + } + } + } + + @Test + public void testNonAtomicMultiPutOmittedFromReadCapacity() throws Exception { + setupQuota(); + + Put put1 = new Put(Bytes.toBytes(UUID.randomUUID().toString())); + put1.addColumn(FAMILY, Bytes.toBytes("doot"), Bytes.toBytes("v")); + Put put2 = new Put(Bytes.toBytes(UUID.randomUUID().toString())); + put2.addColumn(FAMILY, Bytes.toBytes("doot"), Bytes.toBytes("v")); + + Increment inc = new Increment(Bytes.toBytes(UUID.randomUUID().toString())); + inc.addColumn(FAMILY, Bytes.toBytes("doot"), 1); + + List puts = new ArrayList<>(2); + puts.add(put1); + puts.add(put2); + + try (Table table = getTable()) { + for (int i = 0; i < 100; i++) { + table.put(puts); + } + } + } + + @Test + public void testCheckAndMutateCountedAgainstReadCapacity() throws Exception { + setupQuota(); + + byte[] row = Bytes.toBytes(UUID.randomUUID().toString()); + byte[] value = Bytes.toBytes("v"); + Put put = new Put(row); + put.addColumn(FAMILY, Bytes.toBytes("doot"), value); + CheckAndMutate checkAndMutate = + CheckAndMutate.newBuilder(row).ifEquals(FAMILY, QUALIFIER, value).build(put); + + testThrottle(table -> table.checkAndMutate(checkAndMutate)); + } + + @Test + public void testAtomicBatchCountedAgainstReadCapacity() throws Exception { + setupQuota(); + + byte[] row = Bytes.toBytes(UUID.randomUUID().toString()); + Increment inc = new Increment(row); + inc.addColumn(FAMILY, Bytes.toBytes("doot"), 1); + + List incs = new ArrayList<>(2); + incs.add(inc); + incs.add(inc); + + testThrottle(table -> { + Object[] results = new Object[] {}; + table.batch(incs, results); + return results; + }); + } + + private void setupQuota() throws Exception { + try (Admin admin = TEST_UTIL.getAdmin()) { + admin.setQuota(QuotaSettingsFactory.throttleUser(User.getCurrent().getShortName(), + ThrottleType.READ_NUMBER, 1, TimeUnit.MINUTES)); + } + ThrottleQuotaTestUtil.triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); + } + + private void cleanupQuota() throws Exception { + try (Admin admin = TEST_UTIL.getAdmin()) { + admin.setQuota(QuotaSettingsFactory.unthrottleUser(User.getCurrent().getShortName())); + } + ThrottleQuotaTestUtil.triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); + } + + private void testThrottle(ThrowingFunction request) throws Exception { + try (Table table = getTable()) { + // we have a read quota configured, so this should fail + TEST_UTIL.waitFor(60_000, () -> { + try { + request.run(table); + return false; + } catch (Exception e) { + boolean success = e.getCause() instanceof RpcThrottlingException; + if (!success) { + LOG.error("Unexpected exception", e); + } + return success; + } + }); + } finally { + cleanupQuota(); + } + } + + private Table getTable() throws IOException { + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 100); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); + return TEST_UTIL.getConnection().getTableBuilder(TABLE_NAME, null).setOperationTimeout(250) + .build(); + } + + @FunctionalInterface + private interface ThrowingFunction { + O run(I input) throws Exception; + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestBlockBytesScannedQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestBlockBytesScannedQuota.java new file mode 100644 index 000000000000..e27ba123381c --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestBlockBytesScannedQuota.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doGets; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doMultiGets; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doPuts; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doScans; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.waitMinuteQuota; + +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestBlockBytesScannedQuota { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBlockBytesScannedQuota.class); + + private final static Logger LOG = LoggerFactory.getLogger(TestBlockBytesScannedQuota.class); + + private static final int REFRESH_TIME = 5000; + private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + private static final byte[] FAMILY = Bytes.toBytes("cf"); + private static final byte[] QUALIFIER = Bytes.toBytes("q"); + + private static final TableName TABLE_NAME = TableName.valueOf("BlockBytesScannedQuotaTest"); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // client should fail fast + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 10); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); + + // quotas enabled, using block bytes scanned + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME); + + // don't cache blocks to make IO predictable + TEST_UTIL.getConfiguration().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); + + TEST_UTIL.startMiniCluster(1); + TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); + TEST_UTIL.createTable(TABLE_NAME, FAMILY); + TEST_UTIL.waitTableAvailable(TABLE_NAME); + QuotaCache.TEST_FORCE_REFRESH = true; + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + EnvironmentEdgeManager.reset(); + TEST_UTIL.deleteTable(TABLE_NAME); + TEST_UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL); + } + + @Test + public void testBBSGet() throws Exception { + final Admin admin = TEST_UTIL.getAdmin(); + final String userName = User.getCurrent().getShortName(); + int blockSize = admin.getDescriptor(TABLE_NAME).getColumnFamily(FAMILY).getBlocksize(); + Table table = admin.getConnection().getTable(TABLE_NAME); + + doPuts(10_000, FAMILY, QUALIFIER, table); + TEST_UTIL.flush(TABLE_NAME); + + // Add ~10 block/min limit + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.READ_SIZE, + Math.round(10.1 * blockSize), TimeUnit.MINUTES)); + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); + + // should execute at max 10 requests + testTraffic(() -> doGets(20, FAMILY, QUALIFIER, table), 10, 1); + + // wait a minute and you should get another 10 requests executed + waitMinuteQuota(); + testTraffic(() -> doGets(20, FAMILY, QUALIFIER, table), 10, 1); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); + testTraffic(() -> doGets(100, FAMILY, QUALIFIER, table), 100, 0); + testTraffic(() -> doGets(100, FAMILY, QUALIFIER, table), 100, 0); + } + + @Test + public void testBBSScan() throws Exception { + final Admin admin = TEST_UTIL.getAdmin(); + final String userName = User.getCurrent().getShortName(); + int blockSize = admin.getDescriptor(TABLE_NAME).getColumnFamily(FAMILY).getBlocksize(); + Table table = admin.getConnection().getTable(TABLE_NAME); + + doPuts(10_000, FAMILY, QUALIFIER, table); + TEST_UTIL.flush(TABLE_NAME); + + // Add 1 block/min limit. + // This should only allow 1 scan per minute, because we estimate 1 block per scan + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_SIZE, blockSize, + TimeUnit.MINUTES)); + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); + waitMinuteQuota(); + + // should execute 1 request + testTraffic(() -> doScans(5, table), 1, 0); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); + testTraffic(() -> doScans(100, table), 100, 0); + testTraffic(() -> doScans(100, table), 100, 0); + + // Add ~3 block/min limit. This should support >1 scans + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_SIZE, + Math.round(3.1 * blockSize), TimeUnit.MINUTES)); + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); + + // should execute some requests, but not all + testTraffic(() -> doScans(100, table), 100, 90); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); + testTraffic(() -> doScans(100, table), 100, 0); + testTraffic(() -> doScans(100, table), 100, 0); + } + + @Test + public void testBBSMultiGet() throws Exception { + final Admin admin = TEST_UTIL.getAdmin(); + final String userName = User.getCurrent().getShortName(); + int blockSize = admin.getDescriptor(TABLE_NAME).getColumnFamily(FAMILY).getBlocksize(); + Table table = admin.getConnection().getTable(TABLE_NAME); + int rowCount = 10_000; + + doPuts(rowCount, FAMILY, QUALIFIER, table); + TEST_UTIL.flush(TABLE_NAME); + + // Add 1 block/min limit. + // This should only allow 1 multiget per minute, because we estimate 1 block per multiget + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_SIZE, blockSize, + TimeUnit.MINUTES)); + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); + waitMinuteQuota(); + + // should execute 1 request + testTraffic(() -> doMultiGets(10, 10, rowCount, FAMILY, QUALIFIER, table), 1, 1); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); + testTraffic(() -> doMultiGets(100, 10, rowCount, FAMILY, QUALIFIER, table), 100, 0); + testTraffic(() -> doMultiGets(100, 10, rowCount, FAMILY, QUALIFIER, table), 100, 0); + + // Add ~100 block/min limit + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_SIZE, + Math.round(100.1 * blockSize), TimeUnit.MINUTES)); + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); + + // should execute approximately 10 batches of 10 requests + testTraffic(() -> doMultiGets(20, 10, rowCount, FAMILY, QUALIFIER, table), 10, 1); + + // wait a minute and you should get another ~10 batches of 10 requests + waitMinuteQuota(); + testTraffic(() -> doMultiGets(20, 10, rowCount, FAMILY, QUALIFIER, table), 10, 1); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); + testTraffic(() -> doMultiGets(100, 10, rowCount, FAMILY, QUALIFIER, table), 100, 0); + testTraffic(() -> doMultiGets(100, 10, rowCount, FAMILY, QUALIFIER, table), 100, 0); + } + + private void testTraffic(Callable trafficCallable, long expectedSuccess, long marginOfError) + throws Exception { + TEST_UTIL.waitFor(90_000, () -> { + long actualSuccess; + try { + actualSuccess = trafficCallable.call(); + } catch (Exception e) { + throw new RuntimeException(e); + } + LOG.info("Traffic test yielded {} successful requests. Expected {} +/- {}", actualSuccess, + expectedSuccess, marginOfError); + boolean success = (actualSuccess >= expectedSuccess - marginOfError) + && (actualSuccess <= expectedSuccess + marginOfError); + if (!success) { + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); + waitMinuteQuota(); + Thread.sleep(15_000L); + } + return success; + }); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultQuota.java new file mode 100644 index 000000000000..9a2200731f60 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultQuota.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.waitMinuteQuota; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestDefaultQuota { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDefaultQuota.class); + private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + private static final TableName TABLE_NAME = TableName.valueOf(UUID.randomUUID().toString()); + private static final int REFRESH_TIME = 5000; + private static final byte[] FAMILY = Bytes.toBytes("cf"); + private static final byte[] QUALIFIER = Bytes.toBytes("q"); + + @After + public void tearDown() throws Exception { + ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL); + EnvironmentEdgeManager.reset(); + TEST_UTIL.deleteTable(TABLE_NAME); + TEST_UTIL.shutdownMiniCluster(); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // quotas enabled, using block bytes scanned + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME); + TEST_UTIL.getConfiguration().setInt(QuotaUtil.QUOTA_DEFAULT_USER_MACHINE_READ_NUM, 1); + + // don't cache blocks to make IO predictable + TEST_UTIL.getConfiguration().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); + + TEST_UTIL.startMiniCluster(1); + TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); + TEST_UTIL.createTable(TABLE_NAME, FAMILY); + TEST_UTIL.waitTableAvailable(TABLE_NAME); + QuotaCache.TEST_FORCE_REFRESH = true; + + try (Admin admin = TEST_UTIL.getAdmin()) { + ThrottleQuotaTestUtil.doPuts(1_000, FAMILY, QUALIFIER, + admin.getConnection().getTable(TABLE_NAME)); + } + TEST_UTIL.flush(TABLE_NAME); + } + + @Test + public void testDefaultUserReadNum() throws Exception { + // Should have a strict throttle by default + TEST_UTIL.waitFor(60_000, () -> runGetsTest(100) < 100); + + // Add big quota and should be effectively unlimited + configureLenientThrottle(); + refreshQuotas(); + // Should run without error + TEST_UTIL.waitFor(60_000, () -> runGetsTest(100) == 100); + + // Remove all the limits, and should revert to strict default + unsetQuota(); + TEST_UTIL.waitFor(60_000, () -> runGetsTest(100) < 100); + } + + private void configureLenientThrottle() throws IOException { + try (Admin admin = TEST_UTIL.getAdmin()) { + admin.setQuota(QuotaSettingsFactory.throttleUser(getUserName(), ThrottleType.READ_NUMBER, + 100_000, TimeUnit.SECONDS)); + } + } + + private static String getUserName() throws IOException { + return User.getCurrent().getShortName(); + } + + private void refreshQuotas() throws Exception { + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); + waitMinuteQuota(); + } + + private void unsetQuota() throws Exception { + try (Admin admin = TEST_UTIL.getAdmin()) { + admin.setQuota(QuotaSettingsFactory.unthrottleUser(getUserName())); + } + refreshQuotas(); + } + + private long runGetsTest(int attempts) throws Exception { + refreshQuotas(); + try (Table table = getTable()) { + return ThrottleQuotaTestUtil.doGets(attempts, FAMILY, QUALIFIER, table); + } + } + + private Table getTable() throws IOException { + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 100); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); + return TEST_UTIL.getConnection().getTableBuilder(TABLE_NAME, null).setOperationTimeout(250) + .build(); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java index a6e93b663c04..bc2d0ae0713e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java @@ -18,11 +18,17 @@ package org.apache.hadoop.hbase.quotas; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; +import java.util.Random; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; @@ -104,6 +110,64 @@ static long doGets(int maxOps, final Table... tables) { return count; } + static long doGets(int maxOps, byte[] family, byte[] qualifier, final Table... tables) { + int count = 0; + try { + while (count < maxOps) { + Get get = new Get(Bytes.toBytes("row-" + count)); + get.addColumn(family, qualifier); + for (final Table table : tables) { + table.get(get); + } + count += tables.length; + } + } catch (IOException e) { + LOG.error("get failed after nRetries=" + count, e); + } + return count; + } + + static long doMultiGets(int maxOps, int batchSize, int rowCount, byte[] family, byte[] qualifier, + final Table... tables) { + int opCount = 0; + Random random = new Random(); + try { + while (opCount < maxOps) { + List gets = new ArrayList<>(batchSize); + while (gets.size() < batchSize) { + Get get = new Get(Bytes.toBytes("row-" + random.nextInt(rowCount))); + get.addColumn(family, qualifier); + gets.add(get); + } + for (final Table table : tables) { + table.get(gets); + } + opCount += tables.length; + } + } catch (IOException e) { + LOG.error("multiget failed after nRetries=" + opCount, e); + } + return opCount; + } + + static long doScans(int maxOps, Table table) { + int count = 0; + int caching = 100; + try { + Scan scan = new Scan(); + scan.setCaching(caching); + scan.setCacheBlocks(false); + ResultScanner scanner = table.getScanner(scan); + while (count < (maxOps * caching)) { + scanner.next(); + count += 1; + } + } catch (IOException e) { + LOG.error("scan failed after nRetries=" + count, e); + } + return count / caching; + } + static void triggerUserCacheRefresh(HBaseTestingUtil testUtil, boolean bypass, TableName... tables) throws Exception { triggerCacheRefresh(testUtil, bypass, true, false, false, false, false, tables); @@ -137,51 +201,79 @@ private static void triggerCacheRefresh(HBaseTestingUtil testUtil, boolean bypas RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager(); QuotaCache quotaCache = quotaManager.getQuotaCache(); - quotaCache.triggerCacheRefresh(); - // sleep for cache update Thread.sleep(250); + testUtil.waitFor(60000, 250, new ExplainingPredicate() { - for (TableName table : tables) { - quotaCache.getTableLimiter(table); - } - - boolean isUpdated = false; - while (!isUpdated) { - quotaCache.triggerCacheRefresh(); - isUpdated = true; - for (TableName table : tables) { - boolean isBypass = true; - if (userLimiter) { - isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass(); + @Override + public boolean evaluate() throws Exception { + boolean isUpdated = true; + for (TableName table : tables) { + if (userLimiter) { + boolean isUserBypass = + quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass(); + if (isUserBypass != bypass) { + LOG.info( + "User limiter for user={}, table={} not refreshed, bypass expected {}, actual {}", + User.getCurrent(), table, bypass, isUserBypass); + envEdge.incValue(100); + isUpdated = false; + break; + } + } + if (tableLimiter) { + boolean isTableBypass = quotaCache.getTableLimiter(table).isBypass(); + if (isTableBypass != bypass) { + LOG.info("Table limiter for table={} not refreshed, bypass expected {}, actual {}", + table, bypass, isTableBypass); + envEdge.incValue(100); + isUpdated = false; + break; + } + } + if (nsLimiter) { + boolean isNsBypass = + quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass(); + if (isNsBypass != bypass) { + LOG.info( + "Namespace limiter for namespace={} not refreshed, bypass expected {}, actual {}", + table.getNamespaceAsString(), bypass, isNsBypass); + envEdge.incValue(100); + isUpdated = false; + break; + } + } } - if (tableLimiter) { - isBypass &= quotaCache.getTableLimiter(table).isBypass(); + if (rsLimiter) { + boolean rsIsBypass = quotaCache + .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY).isBypass(); + if (rsIsBypass != bypass) { + LOG.info("RegionServer limiter not refreshed, bypass expected {}, actual {}", bypass, + rsIsBypass); + envEdge.incValue(100); + isUpdated = false; + } } - if (nsLimiter) { - isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass(); + if (exceedThrottleQuota) { + if (quotaCache.isExceedThrottleQuotaEnabled() != bypass) { + LOG.info("ExceedThrottleQuotaEnabled not refreshed, bypass expected {}, actual {}", + bypass, quotaCache.isExceedThrottleQuotaEnabled()); + envEdge.incValue(100); + isUpdated = false; + } } - if (isBypass != bypass) { - envEdge.incValue(100); - isUpdated = false; - break; + if (isUpdated) { + return true; } + quotaCache.triggerCacheRefresh(); + return false; } - if (rsLimiter) { - boolean rsIsBypass = quotaCache - .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY).isBypass(); - if (rsIsBypass != bypass) { - envEdge.incValue(100); - isUpdated = false; - } - } - if (exceedThrottleQuota) { - if (quotaCache.isExceedThrottleQuotaEnabled() != bypass) { - envEdge.incValue(100); - isUpdated = false; - } + + @Override + public String explainFailure() throws Exception { + return "Quota cache is still not refreshed"; } - } + }); LOG.debug("QuotaCache"); LOG.debug(Objects.toString(quotaCache.getNamespaceQuotaCache())); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index 01187475944d..c88e2deee4a8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -82,6 +82,11 @@ public long getStoreFileSize() { return 104; } + @Override + public float getCurrentRegionCacheRatio() { + return 0; + } + @Override public long getReadRequestCount() { return 105; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 6a1c285bf8db..3e4199439f7f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -5806,6 +5806,35 @@ public void testFlushResult() throws IOException { } } + @Test + public void testFlushWithSpecifiedFamily() throws IOException { + byte[] family = Bytes.toBytes("family"); + this.region = initHRegion(tableName, method, family); + + Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); + this.region.put(put); + + HRegion.FlushResult fr = + this.region.flushcache(Arrays.asList(family), false, FlushLifeCycleTracker.DUMMY); + assertTrue(fr.isFlushSucceeded()); + } + + @Test + public void testFlushWithSpecifiedNoSuchFamily() throws IOException { + byte[] family = Bytes.toBytes("family"); + byte[] noSuchFamily = Bytes.toBytes("noSuchFamily"); + this.region = initHRegion(tableName, method, family); + + Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); + this.region.put(put); + + HRegion.FlushResult fr = this.region.flushcache(Arrays.asList(family, noSuchFamily), false, + FlushLifeCycleTracker.DUMMY); + assertEquals(HRegion.FlushResult.Result.CANNOT_FLUSH, fr.getResult()); + assertFalse("Cannot flush the Region because of non-existing column families.", + fr.isFlushSucceeded()); + } + protected Configuration initSplit() { // Always compact if there is more than one store file. CONF.setInt("hbase.hstore.compactionThreshold", 2); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerAggregate.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerAggregate.java new file mode 100644 index 000000000000..428416833875 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServerAggregate.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.OptionalDouble; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.ipc.RpcServerInterface; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.apache.hadoop.hbase.wal.WALFactory; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +@Category({ SmallTests.class, RegionServerTests.class }) +public class TestMetricsRegionServerAggregate { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetricsRegionServerAggregate.class); + + @Test + public void test() { + AtomicInteger retVal = new AtomicInteger(0); + Answer defaultAnswer = invocation -> { + Class returnType = invocation.getMethod().getReturnType(); + + if (returnType.equals(Integer.TYPE) || returnType.equals(Integer.class)) { + return retVal.get(); + } else if (returnType.equals(Long.TYPE) || returnType.equals(Long.class)) { + return (long) retVal.get(); + } + return Mockito.RETURNS_DEFAULTS.answer(invocation); + }; + + ServerName serverName = mock(ServerName.class); + when(serverName.getHostname()).thenReturn("foo"); + WALFactory walFactory = mock(WALFactory.class); + RpcServerInterface rpcServer = mock(RpcServerInterface.class); + AtomicInteger storeFileCount = new AtomicInteger(1); + HRegion regionOne = getMockedRegion(defaultAnswer, "a", "foo", true, storeFileCount); + HRegion regionTwo = getMockedRegion(defaultAnswer, "b", "bar", true, storeFileCount); + HRegion regionThree = getMockedRegion(defaultAnswer, "c", "foo", false, storeFileCount); + HRegion regionFour = getMockedRegion(defaultAnswer, "d", "bar", false, storeFileCount); + List regions = Lists.newArrayList(regionOne, regionTwo, regionThree, regionFour); + + int numStoresPerRegion = 2; + for (HRegion region : regions) { + // if adding more stores, update numStoresPerRegion so that tests below continue working + assertEquals(numStoresPerRegion, region.getStores().size()); + } + + HRegionServer regionServer = mock(HRegionServer.class, defaultAnswer); + when(regionServer.getWalFactory()).thenReturn(walFactory); + when(regionServer.getOnlineRegionsLocalContext()).thenReturn(regions); + when(regionServer.getServerName()).thenReturn(serverName); + Configuration conf = HBaseConfiguration.create(); + int metricsPeriodSec = 600; + // set a very long period so that it doesn't actually run during our very quick test + conf.setLong(HConstants.REGIONSERVER_METRICS_PERIOD, metricsPeriodSec * 1000); + when(regionServer.getConfiguration()).thenReturn(conf); + when(regionServer.getRpcServer()).thenReturn(rpcServer); + + MetricsRegionServerWrapperImpl wrapper = new MetricsRegionServerWrapperImpl(regionServer); + + // we need to control the edge because rate calculations expect a + // stable interval relative to the configured period + ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); + EnvironmentEdgeManager.injectEdge(edge); + + try { + for (int i = 1; i <= 10; i++) { + edge.incValue(wrapper.getPeriod()); + retVal.incrementAndGet(); + wrapper.forceRecompute(); + + int numRegions = regions.size(); + int totalStores = numRegions * numStoresPerRegion; + + // there are N regions, and each has M stores. everything gets aggregated, so + // multiply expected values accordingly + int expectedForRegions = retVal.get() * numRegions; + int expectedForStores = retVal.get() * totalStores; + + assertEquals(totalStores, wrapper.getNumStores()); + assertEquals(expectedForStores, wrapper.getFlushedCellsCount()); + assertEquals(expectedForStores, wrapper.getCompactedCellsCount()); + assertEquals(expectedForStores, wrapper.getMajorCompactedCellsCount()); + assertEquals(expectedForStores, wrapper.getFlushedCellsSize()); + assertEquals(expectedForStores, wrapper.getCompactedCellsSize()); + assertEquals(expectedForStores, wrapper.getMajorCompactedCellsSize()); + assertEquals(expectedForRegions, wrapper.getCellsCountCompactedFromMob()); + assertEquals(expectedForRegions, wrapper.getCellsCountCompactedToMob()); + assertEquals(expectedForRegions, wrapper.getCellsSizeCompactedFromMob()); + assertEquals(expectedForRegions, wrapper.getCellsSizeCompactedToMob()); + assertEquals(expectedForRegions, wrapper.getMobFlushCount()); + assertEquals(expectedForRegions, wrapper.getMobFlushedCellsCount()); + assertEquals(expectedForRegions, wrapper.getMobFlushedCellsSize()); + assertEquals(expectedForRegions, wrapper.getMobScanCellsCount()); + assertEquals(expectedForRegions, wrapper.getMobScanCellsSize()); + assertEquals(expectedForRegions, wrapper.getCheckAndMutateChecksFailed()); + assertEquals(expectedForRegions, wrapper.getCheckAndMutateChecksPassed()); + assertEquals(expectedForStores, wrapper.getStoreFileIndexSize()); + assertEquals(expectedForStores, wrapper.getTotalStaticIndexSize()); + assertEquals(expectedForStores, wrapper.getTotalStaticBloomSize()); + assertEquals(expectedForStores, wrapper.getBloomFilterRequestsCount()); + assertEquals(expectedForStores, wrapper.getBloomFilterNegativeResultsCount()); + assertEquals(expectedForStores, wrapper.getBloomFilterEligibleRequestsCount()); + assertEquals(expectedForRegions, wrapper.getNumMutationsWithoutWAL()); + assertEquals(expectedForRegions, wrapper.getDataInMemoryWithoutWAL()); + assertEquals(expectedForRegions, wrapper.getAverageRegionSize()); + assertEquals(expectedForRegions, wrapper.getBlockedRequestsCount()); + assertEquals(expectedForStores, wrapper.getNumReferenceFiles()); + assertEquals(expectedForStores, wrapper.getMemStoreSize()); + assertEquals(expectedForStores, wrapper.getOnHeapMemStoreSize()); + assertEquals(expectedForStores, wrapper.getOffHeapMemStoreSize()); + assertEquals(expectedForStores, wrapper.getStoreFileSize()); + assertEquals(expectedForRegions, wrapper.getReadRequestsCount()); + assertEquals(expectedForRegions, wrapper.getCpRequestsCount()); + assertEquals(expectedForRegions, wrapper.getFilteredReadRequestsCount()); + assertEquals(expectedForRegions, wrapper.getWriteRequestsCount()); + assertEquals(expectedForRegions * 2, wrapper.getTotalRowActionRequestCount()); + + // If we have N regions, each with M stores. That's N*M stores in total. In creating those + // stores, we increment the number and age of storefiles for each one. So the first + // store has 1 file of 1 age, then 2 files of 2 age, etc. + // formula for 1+2+3..+n + assertEquals((totalStores * (totalStores + 1)) / 2, wrapper.getNumStoreFiles()); + assertEquals(totalStores, wrapper.getMaxStoreFiles()); + assertEquals(totalStores, wrapper.getMaxStoreFileAge()); + assertEquals(1, wrapper.getMinStoreFileAge()); + assertEquals(totalStores / 2, wrapper.getAvgStoreFileAge()); + + // there are four regions, two are primary and the other two secondary + // for each type, one region has 100% locality, the other has 0%. + // this just proves we correctly aggregate for each + assertEquals(50.0, wrapper.getPercentFileLocal(), 0.0001); + assertEquals(50.0, wrapper.getPercentFileLocalSecondaryRegions(), 0.0001); + + // readRequestCount and writeRequestCount are tracking the value of i, which increases by 1 + // each interval. There are N regions, so the delta each interval is N*i=N. So the rate is + // simply N / period. + assertEquals((double) numRegions / metricsPeriodSec, wrapper.getReadRequestsRatePerSecond(), + 0.0001); + assertEquals((double) numRegions / metricsPeriodSec, + wrapper.getWriteRequestsRatePerSecond(), 0.0001); + // total of above, so multiply by 2 + assertEquals((double) numRegions / metricsPeriodSec * 2, wrapper.getRequestsPerSecond(), + 0.0001); + // Similar logic to above, except there are M totalStores and each one is of + // size tracking i. So the rate is just M / period. + assertEquals((double) totalStores / metricsPeriodSec, wrapper.getStoreFileSizeGrowthRate(), + 0.0001); + } + } finally { + EnvironmentEdgeManager.reset(); + } + } + + private HRegion getMockedRegion(Answer defaultAnswer, String name, String localOnHost, + boolean isPrimary, AtomicInteger storeFileCount) { + RegionInfo regionInfo = mock(RegionInfo.class); + when(regionInfo.getEncodedName()).thenReturn(name); + if (!isPrimary) { + when(regionInfo.getReplicaId()).thenReturn(RegionInfo.DEFAULT_REPLICA_ID + 1); + } + HDFSBlocksDistribution distribution = new HDFSBlocksDistribution(); + distribution.addHostsAndBlockWeight(new String[] { localOnHost }, 100); + + HStore store = getMockedStore(HStore.class, defaultAnswer, storeFileCount); + HMobStore mobStore = getMockedStore(HMobStore.class, defaultAnswer, storeFileCount); + + HRegion region = mock(HRegion.class, defaultAnswer); + when(region.getRegionInfo()).thenReturn(regionInfo); + when(region.getHDFSBlocksDistribution()).thenReturn(distribution); + when(region.getStores()).thenReturn(Lists.newArrayList(store, mobStore)); + return region; + } + + private T getMockedStore(Class clazz, Answer defaultAnswer, + AtomicInteger storeFileCount) { + T store = mock(clazz, defaultAnswer); + int storeFileCountVal = storeFileCount.getAndIncrement(); + when(store.getStorefilesCount()).thenReturn(storeFileCountVal); + when(store.getAvgStoreFileAge()).thenReturn(OptionalDouble.of(storeFileCountVal)); + when(store.getMaxStoreFileAge()).thenReturn(OptionalLong.of(storeFileCountVal)); + when(store.getMinStoreFileAge()).thenReturn(OptionalLong.of(storeFileCountVal)); + MemStoreSize memStore = mock(MemStoreSize.class, defaultAnswer); + when(store.getMemStoreSize()).thenReturn(memStore); + return store; + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSQosFunction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSQosFunction.java index c6bd8967e124..15ee32397e9a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSQosFunction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSQosFunction.java @@ -72,5 +72,8 @@ public void testAnnotations() { checkMethod(conf, "CloseRegion", HConstants.ADMIN_QOS, qosFunction); checkMethod(conf, "CompactRegion", HConstants.ADMIN_QOS, qosFunction); checkMethod(conf, "FlushRegion", HConstants.ADMIN_QOS, qosFunction); + checkMethod(conf, "UpdateConfiguration", HConstants.ADMIN_QOS, qosFunction); + checkMethod(conf, "CompactionSwitch", HConstants.ADMIN_QOS, qosFunction); + checkMethod(conf, "RollWALWriter", HConstants.ADMIN_QOS, qosFunction); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java index 940dbebf614b..2a5aec458828 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java @@ -20,9 +20,13 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HBaseTestingUtil; @@ -31,6 +35,7 @@ import org.apache.hadoop.hbase.SingleProcessHBaseCluster; import org.apache.hadoop.hbase.StartTestingClusterOption; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; @@ -48,8 +53,10 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hbase.wal.WALProvider; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -59,6 +66,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; + /** * Test log deletion as logs are rolled. */ @@ -74,6 +83,10 @@ public abstract class AbstractTestLogRolling { protected static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); @Rule public final TestName name = new TestName(); + protected static int syncLatencyMillis; + private static int rowNum = 1; + private static final AtomicBoolean slowSyncHookCalled = new AtomicBoolean(); + protected static ScheduledExecutorService EXECUTOR; public AbstractTestLogRolling() { this.server = null; @@ -118,6 +131,17 @@ public static void setUpBeforeClass() throws Exception { // disable low replication check for log roller to get a more stable result // TestWALOpenAfterDNRollingStart will test this option. conf.setLong("hbase.regionserver.hlog.check.lowreplication.interval", 24L * 60 * 60 * 1000); + + // For slow sync threshold test: roll after 5 slow syncs in 10 seconds + conf.setInt(FSHLog.SLOW_SYNC_ROLL_THRESHOLD, 5); + conf.setInt(FSHLog.SLOW_SYNC_ROLL_INTERVAL_MS, 10 * 1000); + // For slow sync threshold test: roll once after a sync above this threshold + conf.setInt(FSHLog.ROLL_ON_SYNC_TIME_MS, 5000); + + // Slow sync executor. + EXECUTOR = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("Slow-sync-%d") + .setDaemon(true).setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build()); } @Before @@ -139,6 +163,11 @@ public void tearDown() throws Exception { TEST_UTIL.shutdownMiniCluster(); } + @AfterClass + public static void tearDownAfterClass() { + EXECUTOR.shutdownNow(); + } + private void startAndWriteData() throws IOException, InterruptedException { this.server = cluster.getRegionServerThreads().get(0).getRegionServer(); @@ -158,6 +187,74 @@ private void startAndWriteData() throws IOException, InterruptedException { } } + private static void setSyncLatencyMillis(int latency) { + syncLatencyMillis = latency; + } + + protected final AbstractFSWAL getWALAndRegisterSlowSyncHook(RegionInfo region) + throws IOException { + // Get a reference to the wal. + final AbstractFSWAL log = (AbstractFSWAL) server.getWAL(region); + + // Register a WALActionsListener to observe if a SLOW_SYNC roll is requested + log.registerWALActionsListener(new WALActionsListener() { + @Override + public void logRollRequested(RollRequestReason reason) { + switch (reason) { + case SLOW_SYNC: + slowSyncHookCalled.lazySet(true); + break; + default: + break; + } + } + }); + return log; + } + + protected final void checkSlowSync(AbstractFSWAL log, Table table, int slowSyncLatency, + int writeCount, boolean slowSync) throws Exception { + if (slowSyncLatency > 0) { + setSyncLatencyMillis(slowSyncLatency); + setSlowLogWriter(log.conf); + } else { + setDefaultLogWriter(log.conf); + } + + // Set up for test + log.rollWriter(true); + slowSyncHookCalled.set(false); + + final WALProvider.WriterBase oldWriter = log.getWriter(); + + // Write some data + for (int i = 0; i < writeCount; i++) { + writeData(table, rowNum++); + } + + if (slowSync) { + TEST_UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate() { + @Override + public boolean evaluate() throws Exception { + return log.getWriter() != oldWriter; + } + + @Override + public String explainFailure() throws Exception { + return "Waited too long for our test writer to get rolled out"; + } + }); + + assertTrue("Should have triggered log roll due to SLOW_SYNC", slowSyncHookCalled.get()); + } else { + assertFalse("Should not have triggered log roll due to SLOW_SYNC", slowSyncHookCalled.get()); + } + } + + protected abstract void setSlowLogWriter(Configuration conf); + + protected abstract void setDefaultLogWriter(Configuration conf); + /** * Tests that log rolling doesn't hang when no data is written. */ @@ -239,12 +336,10 @@ void validateData(Table table, int rownum) throws IOException { */ @Test public void testCompactionRecordDoesntBlockRolling() throws Exception { - Table table = null; // When the hbase:meta table can be opened, the region servers are running - Table t = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME); - try { - table = createTestTable(getName()); + try (Table t = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME); + Table table = createTestTable(getName())) { server = TEST_UTIL.getRSForFirstRegionInTable(table.getName()); HRegion region = server.getRegions(table.getName()).get(0); @@ -286,9 +381,6 @@ public void testCompactionRecordDoesntBlockRolling() throws Exception { log.rollWriter(); // Now 2nd WAL is deleted and 3rd is added. assertEquals("Should have 1 WALs at the end", 1, AbstractFSWALProvider.getNumRolledLogFiles(log)); - } finally { - if (t != null) t.close(); - if (table != null) table.close(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java index 9dc27a693a7f..804e93eb8f56 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java @@ -20,10 +20,17 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; @@ -36,6 +43,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import org.apache.hbase.thirdparty.io.netty.channel.Channel; +import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; + @Category({ VerySlowRegionServerTests.class, LargeTests.class }) public class TestAsyncLogRolling extends AbstractTestLogRolling { @@ -51,6 +61,61 @@ public static void setUpBeforeClass() throws Exception { AbstractTestLogRolling.setUpBeforeClass(); } + public static class SlowSyncLogWriter extends AsyncProtobufLogWriter { + + public SlowSyncLogWriter(EventLoopGroup eventLoopGroup, Class channelClass) { + super(eventLoopGroup, channelClass); + } + + @Override + public CompletableFuture sync(boolean forceSync) { + CompletableFuture future = new CompletableFuture<>(); + super.sync(forceSync).whenCompleteAsync((lengthAfterFlush, error) -> { + EXECUTOR.schedule(() -> { + if (error != null) { + future.completeExceptionally(error); + } else { + future.complete(lengthAfterFlush); + } + }, syncLatencyMillis, TimeUnit.MILLISECONDS); + }); + return future; + } + } + + @Override + protected void setSlowLogWriter(Configuration conf) { + conf.set(AsyncFSWALProvider.WRITER_IMPL, SlowSyncLogWriter.class.getName()); + } + + @Override + protected void setDefaultLogWriter(Configuration conf) { + conf.set(AsyncFSWALProvider.WRITER_IMPL, AsyncProtobufLogWriter.class.getName()); + } + + @Test + public void testSlowSyncLogRolling() throws Exception { + // Create the test table + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName())) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build(); + admin.createTable(desc); + try (Table table = TEST_UTIL.getConnection().getTable(desc.getTableName())) { + server = TEST_UTIL.getRSForFirstRegionInTable(desc.getTableName()); + RegionInfo region = server.getRegions(desc.getTableName()).get(0).getRegionInfo(); + final AbstractFSWAL log = getWALAndRegisterSlowSyncHook(region); + + // Set default log writer, no additional latency to any sync on the hlog. + checkSlowSync(log, table, -1, 10, false); + + // Adds 5000 ms of latency to any sync on the hlog. This will trip the other threshold. + // Write some data. Should only take one sync. + checkSlowSync(log, table, 5000, 1, true); + + // Set default log writer, no additional latency to any sync on the hlog. + checkSlowSync(log, table, -1, 10, false); + } + } + @Test public void testLogRollOnDatanodeDeath() throws IOException, InterruptedException { dfsCluster.startDataNodes(TEST_UTIL.getConfiguration(), 3, true, null, null); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java index f07a02cb25d1..9caa47e8614b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; @@ -56,10 +55,9 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; +import org.apache.hadoop.hbase.wal.FSHLogProvider; import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALProvider.Writer; import org.apache.hadoop.hbase.wal.WALStreamReader; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -98,192 +96,30 @@ public static void setUpBeforeClass() throws Exception { conf.setInt("hbase.regionserver.hlog.lowreplication.rolllimit", 3); conf.set(WALFactory.WAL_PROVIDER, "filesystem"); AbstractTestLogRolling.setUpBeforeClass(); - - // For slow sync threshold test: roll after 5 slow syncs in 10 seconds - TEST_UTIL.getConfiguration().setInt(FSHLog.SLOW_SYNC_ROLL_THRESHOLD, 5); - TEST_UTIL.getConfiguration().setInt(FSHLog.SLOW_SYNC_ROLL_INTERVAL_MS, 10 * 1000); - // For slow sync threshold test: roll once after a sync above this threshold - TEST_UTIL.getConfiguration().setInt(FSHLog.ROLL_ON_SYNC_TIME_MS, 5000); } - @Test - public void testSlowSyncLogRolling() throws Exception { - // Create the test table - TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName())) - .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build(); - admin.createTable(desc); - Table table = TEST_UTIL.getConnection().getTable(desc.getTableName()); - int row = 1; - try { - // Get a reference to the FSHLog - server = TEST_UTIL.getRSForFirstRegionInTable(desc.getTableName()); - RegionInfo region = server.getRegions(desc.getTableName()).get(0).getRegionInfo(); - final FSHLog log = (FSHLog) server.getWAL(region); - - // Register a WALActionsListener to observe if a SLOW_SYNC roll is requested - - final AtomicBoolean slowSyncHookCalled = new AtomicBoolean(); - log.registerWALActionsListener(new WALActionsListener() { - @Override - public void logRollRequested(WALActionsListener.RollRequestReason reason) { - switch (reason) { - case SLOW_SYNC: - slowSyncHookCalled.lazySet(true); - break; - default: - break; - } - } - }); - - // Write some data - - for (int i = 0; i < 10; i++) { - writeData(table, row++); - } - - assertFalse("Should not have triggered log roll due to SLOW_SYNC", slowSyncHookCalled.get()); - - // Set up for test - slowSyncHookCalled.set(false); - - // Wrap the current writer with the anonymous class below that adds 200 ms of - // latency to any sync on the hlog. This should be more than sufficient to trigger - // slow sync warnings. - final Writer oldWriter1 = log.getWriter(); - final Writer newWriter1 = new Writer() { - @Override - public void close() throws IOException { - oldWriter1.close(); - } - - @Override - public void sync(boolean forceSync) throws IOException { - try { - Thread.sleep(200); - } catch (InterruptedException e) { - InterruptedIOException ex = new InterruptedIOException(); - ex.initCause(e); - throw ex; - } - oldWriter1.sync(forceSync); - } - - @Override - public void append(Entry entry) throws IOException { - oldWriter1.append(entry); - } - - @Override - public long getLength() { - return oldWriter1.getLength(); - } - - @Override - public long getSyncedLength() { - return oldWriter1.getSyncedLength(); - } - }; - log.setWriter(newWriter1); - - // Write some data. - // We need to write at least 5 times, but double it. We should only request - // a SLOW_SYNC roll once in the current interval. - for (int i = 0; i < 10; i++) { - writeData(table, row++); - } - - // Wait for our wait injecting writer to get rolled out, as needed. - - TEST_UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate() { - @Override - public boolean evaluate() throws Exception { - return log.getWriter() != newWriter1; - } - - @Override - public String explainFailure() throws Exception { - return "Waited too long for our test writer to get rolled out"; - } - }); - - assertTrue("Should have triggered log roll due to SLOW_SYNC", slowSyncHookCalled.get()); - - // Set up for test - slowSyncHookCalled.set(false); - - // Wrap the current writer with the anonymous class below that adds 5000 ms of - // latency to any sync on the hlog. - // This will trip the other threshold. - final Writer oldWriter2 = (Writer) log.getWriter(); - final Writer newWriter2 = new Writer() { - @Override - public void close() throws IOException { - oldWriter2.close(); - } - - @Override - public void sync(boolean forceSync) throws IOException { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - InterruptedIOException ex = new InterruptedIOException(); - ex.initCause(e); - throw ex; - } - oldWriter2.sync(forceSync); - } - - @Override - public void append(Entry entry) throws IOException { - oldWriter2.append(entry); - } - - @Override - public long getLength() { - return oldWriter2.getLength(); - } - - @Override - public long getSyncedLength() { - return oldWriter2.getSyncedLength(); - } - }; - log.setWriter(newWriter2); - - // Write some data. Should only take one sync. - - writeData(table, row++); - - // Wait for our wait injecting writer to get rolled out, as needed. - - TEST_UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate() { - @Override - public boolean evaluate() throws Exception { - return log.getWriter() != newWriter2; - } - - @Override - public String explainFailure() throws Exception { - return "Waited too long for our test writer to get rolled out"; - } - }); - - assertTrue("Should have triggered log roll due to SLOW_SYNC", slowSyncHookCalled.get()); - - // Set up for test - slowSyncHookCalled.set(false); - - // Write some data - for (int i = 0; i < 10; i++) { - writeData(table, row++); + public static class SlowSyncLogWriter extends ProtobufLogWriter { + @Override + public void sync(boolean forceSync) throws IOException { + try { + Thread.sleep(syncLatencyMillis); + } catch (InterruptedException e) { + InterruptedIOException ex = new InterruptedIOException(); + ex.initCause(e); + throw ex; } + super.sync(forceSync); + } + } - assertFalse("Should not have triggered log roll due to SLOW_SYNC", slowSyncHookCalled.get()); + @Override + protected void setSlowLogWriter(Configuration conf) { + conf.set(FSHLogProvider.WRITER_IMPL, SlowSyncLogWriter.class.getName()); + } - } finally { - table.close(); - } + @Override + protected void setDefaultLogWriter(Configuration conf) { + conf.set(FSHLogProvider.WRITER_IMPL, ProtobufLogWriter.class.getName()); } void batchWriteAndWait(Table table, final FSHLog log, int start, boolean expect, int timeout) @@ -313,6 +149,36 @@ void batchWriteAndWait(Table table, final FSHLog log, int start, boolean expect, } } + @Test + public void testSlowSyncLogRolling() throws Exception { + // Create the test table + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName())) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build(); + admin.createTable(desc); + try (Table table = TEST_UTIL.getConnection().getTable(desc.getTableName())) { + server = TEST_UTIL.getRSForFirstRegionInTable(desc.getTableName()); + RegionInfo region = server.getRegions(desc.getTableName()).get(0).getRegionInfo(); + final AbstractFSWAL log = getWALAndRegisterSlowSyncHook(region); + + // Set default log writer, no additional latency to any sync on the hlog. + checkSlowSync(log, table, -1, 10, false); + + // Adds 200 ms of latency to any sync on the hlog. This should be more than sufficient to + // trigger slow sync warnings. + // Write some data. + // We need to write at least 5 times, but double it. We should only request + // a SLOW_SYNC roll once in the current interval. + checkSlowSync(log, table, 200, 10, true); + + // Adds 5000 ms of latency to any sync on the hlog. This will trip the other threshold. + // Write some data. Should only take one sync. + checkSlowSync(log, table, 5000, 1, true); + + // Set default log writer, no additional latency to any sync on the hlog. + checkSlowSync(log, table, -1, 10, false); + } + } + /** * Tests that logs are rolled upon detecting datanode death Requires an HDFS jar with HDFS-826 & * syncFs() support (HDFS-200) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java index 93fa22c00fd3..d6c7a0250015 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java @@ -34,15 +34,17 @@ import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.AdvancedScanResultConsumer; import org.apache.hadoop.hbase.client.AsyncClusterConnection; import org.apache.hadoop.hbase.client.AsyncConnection; import org.apache.hadoop.hbase.client.AsyncTable; import org.apache.hadoop.hbase.client.ClusterConnectionFactory; +import org.apache.hadoop.hbase.client.ConnectionRegistry; +import org.apache.hadoop.hbase.client.DoNothingConnectionRegistry; import org.apache.hadoop.hbase.client.DummyAsyncClusterConnection; import org.apache.hadoop.hbase.client.DummyAsyncTable; -import org.apache.hadoop.hbase.client.DummyConnectionRegistry; import org.apache.hadoop.hbase.client.Row; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ReplicationTests; @@ -84,8 +86,8 @@ public class TestWALEntrySinkFilter { public void testWALEntryFilter() throws IOException { Configuration conf = HBaseConfiguration.create(); // Make it so our filter is instantiated on construction of ReplicationSink. - conf.setClass(DummyConnectionRegistry.REGISTRY_IMPL_CONF_KEY, DevNullConnectionRegistry.class, - DummyConnectionRegistry.class); + conf.setClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, + DevNullConnectionRegistry.class, ConnectionRegistry.class); conf.setClass(WALEntrySinkFilter.WAL_ENTRY_FILTER_KEY, IfTimeIsGreaterThanBOUNDARYWALEntrySinkFilterImpl.class, WALEntrySinkFilter.class); conf.setClass(ClusterConnectionFactory.HBASE_SERVER_CLUSTER_CONNECTION_IMPL, @@ -166,9 +168,10 @@ public boolean filter(TableName table, long writeTime) { } } - public static class DevNullConnectionRegistry extends DummyConnectionRegistry { + public static class DevNullConnectionRegistry extends DoNothingConnectionRegistry { - public DevNullConnectionRegistry(Configuration conf) { + public DevNullConnectionRegistry(Configuration conf, User user) { + super(conf, user); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java index 72fc7141680a..403a538f024b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java @@ -109,14 +109,15 @@ public static List data() { @BeforeClass public static void setUpBeforeClass() throws IOException { Security.addProvider(new BouncyCastleProvider()); - File dir = new File(UTIL.getDataTestDir(TestNettyTlsIPC.class.getSimpleName()).toString()) - .getCanonicalFile(); + File dir = + new File(UTIL.getDataTestDir(TestNettyTLSIPCFileWatcher.class.getSimpleName()).toString()) + .getCanonicalFile(); FileUtils.forceMkdir(dir); // server must enable tls CONF.setBoolean(X509Util.HBASE_SERVER_NETTY_TLS_ENABLED, true); PROVIDER = new X509TestContextProvider(CONF, dir); EVENT_LOOP_GROUP_CONFIG = - NettyEventLoopGroupConfig.setup(CONF, TestNettyTlsIPC.class.getSimpleName()); + NettyEventLoopGroupConfig.setup(CONF, TestNettyTLSIPCFileWatcher.class.getSimpleName()); SERVER = mock(HBaseServerBase.class); when(SERVER.getEventLoopGroupConfig()).thenReturn(EVENT_LOOP_GROUP_CONFIG); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSaslTlsIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSaslTlsIPC.java index 1477e8aa0fca..1120d56fb9fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSaslTlsIPC.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSaslTlsIPC.java @@ -97,7 +97,7 @@ public static List data() { @BeforeClass public static void setUpBeforeClass() throws Exception { Security.addProvider(new BouncyCastleProvider()); - File dir = new File(TEST_UTIL.getDataTestDir(TestNettyTlsIPC.class.getSimpleName()).toString()) + File dir = new File(TEST_UTIL.getDataTestDir(TestSaslTlsIPC.class.getSimpleName()).toString()) .getCanonicalFile(); FileUtils.forceMkdir(dir); initKDCAndConf(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecurityInfoAndHBasePolicyProviderMatch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecurityInfoAndHBasePolicyProviderMatch.java new file mode 100644 index 000000000000..4a664d4d4d0e --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecurityInfoAndHBasePolicyProviderMatch.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertNotNull; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface; +import org.apache.hadoop.hbase.ipc.RpcServerInterface; +import org.apache.hadoop.hbase.testclassification.SecurityTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.security.authorize.Service; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Make sure that all rpc services for master and region server are properly configured in + * {@link SecurityInfo} and {@link HBasePolicyProvider}. + */ +@Category({ SecurityTests.class, SmallTests.class }) +public class TestSecurityInfoAndHBasePolicyProviderMatch { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSecurityInfoAndHBasePolicyProviderMatch.class); + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + private void assertServiceMatches(RpcServerInterface rpcServer) { + HBasePolicyProvider provider = new HBasePolicyProvider(); + Set> serviceClasses = + Stream.of(provider.getServices()).map(Service::getProtocol).collect(Collectors.toSet()); + for (BlockingServiceAndInterface bsai : ((RpcServer) rpcServer).getServices()) { + assertNotNull( + "no security info for " + bsai.getBlockingService().getDescriptorForType().getName(), + SecurityInfo.getInfo(bsai.getBlockingService().getDescriptorForType().getName())); + assertThat(serviceClasses, hasItem(bsai.getServiceInterface())); + } + } + + @Test + public void testMatches() { + assertServiceMatches( + UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().getRpcServer()); + assertServiceMatches(UTIL.getMiniHBaseCluster().getRegionServer(0).getRpcServer()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 11ae3b3ecf09..cc895d21ac61 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -3086,6 +3086,41 @@ public Object run() throws Exception { verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); } + @Test + public void testUpdateMasterConfiguration() throws Exception { + AccessTestAction action = () -> { + ACCESS_CONTROLLER.preUpdateMasterConfiguration(ObserverContextImpl.createAndPrepare(CP_ENV), + null); + return null; + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testUpdateRegionServerConfiguration() throws Exception { + AccessTestAction action = () -> { + ACCESS_CONTROLLER + .preUpdateRegionServerConfiguration(ObserverContextImpl.createAndPrepare(RSCP_ENV), null); + return null; + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testClearRegionBlockCache() throws Exception { + AccessTestAction action = () -> { + ACCESS_CONTROLLER.preClearRegionBlockCache(ObserverContextImpl.createAndPrepare(RSCP_ENV)); + return null; + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + @Test public void testTransitSyncReplicationPeerState() throws Exception { AccessTestAction action = new AccessTestAction() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java index 16ac215acaff..f132eb6964b1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java @@ -21,22 +21,26 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.AsyncConnection; +import org.apache.hadoop.hbase.client.AsyncTable; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.ipc.NettyRpcClient; import org.apache.hadoop.hbase.ipc.RpcClientFactory; import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; @@ -51,11 +55,9 @@ import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; -import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; - -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.AuthenticationService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.GetAuthenticationTokenRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.GetAuthenticationTokenResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.WhoAmIRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.WhoAmIResponse; @@ -92,24 +94,58 @@ public void setUpBeforeMethod() { rpcClientImpl); } - @Test - public void test() throws Exception { - try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - Table table = conn.getTable(TableName.META_TABLE_NAME)) { - CoprocessorRpcChannel rpcChannel = table.coprocessorService(HConstants.EMPTY_START_ROW); - AuthenticationProtos.AuthenticationService.BlockingInterface service = - AuthenticationProtos.AuthenticationService.newBlockingStub(rpcChannel); - WhoAmIResponse response = service.whoAmI(null, WhoAmIRequest.getDefaultInstance()); + private void testToken() throws Exception { + try (AsyncConnection conn = + ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get()) { + AsyncTable table = conn.getTable(TableName.META_TABLE_NAME); + WhoAmIResponse response = + table. coprocessorService( + AuthenticationService::newStub, + (s, c, r) -> s.whoAmI(c, WhoAmIRequest.getDefaultInstance(), r), + HConstants.EMPTY_START_ROW).get(); assertEquals(USERNAME, response.getUsername()); assertEquals(AuthenticationMethod.TOKEN.name(), response.getAuthMethod()); - try { - service.getAuthenticationToken(null, GetAuthenticationTokenRequest.getDefaultInstance()); - } catch (ServiceException e) { - IOException ioe = ProtobufUtil.getRemoteException(e); - assertThat(ioe, instanceOf(AccessDeniedException.class)); - assertThat(ioe.getMessage(), - containsString("Token generation only allowed for Kerberos authenticated clients")); - } + IOException ioe = + assertThrows(IOException.class, + () -> FutureUtils.get(table. coprocessorService(AuthenticationService::newStub, + (s, c, r) -> s.getAuthenticationToken(c, + GetAuthenticationTokenRequest.getDefaultInstance(), r), + HConstants.EMPTY_START_ROW))); + assertThat(ioe, instanceOf(AccessDeniedException.class)); + assertThat(ioe.getMessage(), + containsString("Token generation only allowed for Kerberos authenticated clients")); } + + } + + /** + * Confirm that we will use delegation token first if token and kerberos tickets are both present + */ + @Test + public void testTokenFirst() throws Exception { + testToken(); + } + + /** + * Confirm that we can connect to cluster successfully when there is only token present, i.e, no + * kerberos ticket + */ + @Test + public void testOnlyToken() throws Exception { + User user = + User.createUserForTesting(TEST_UTIL.getConfiguration(), "no_krb_user", new String[0]); + for (Token token : User.getCurrent().getUGI().getCredentials() + .getAllTokens()) { + user.getUGI().addToken(token); + } + user.getUGI().doAs(new PrivilegedExceptionAction() { + + @Override + public Void run() throws Exception { + testToken(); + return null; + } + }); } } diff --git a/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml b/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml index 928a3276df65..441d38ccd47f 100644 --- a/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml +++ b/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml @@ -81,7 +81,6 @@ org.apache.hbase hbase-shaded-testing-util - ${project.version} test diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 5c6d778bcf37..b04a79229831 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1580,6 +1580,8 @@ def list_locks # Parse arguments and update TableDescriptorBuilder accordingly # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity def update_tdb_from_arg(tdb, arg) + tdb.setErasureCodingPolicy(arg.delete(TableDescriptorBuilder::ERASURE_CODING_POLICY)) \ + if arg.include?(TableDescriptorBuilder::ERASURE_CODING_POLICY) tdb.setMaxFileSize(arg.delete(TableDescriptorBuilder::MAX_FILESIZE)) if arg.include?(TableDescriptorBuilder::MAX_FILESIZE) tdb.setReadOnly(JBoolean.valueOf(arg.delete(TableDescriptorBuilder::READONLY))) if arg.include?(TableDescriptorBuilder::READONLY) tdb.setCompactionEnabled(JBoolean.valueOf(arg.delete(TableDescriptorBuilder::COMPACTION_ENABLED))) if arg.include?(TableDescriptorBuilder::COMPACTION_ENABLED) diff --git a/hbase-shell/src/main/ruby/jar-bootstrap.rb b/hbase-shell/src/main/ruby/jar-bootstrap.rb index 3f0e650947cf..63cb0a755449 100644 --- a/hbase-shell/src/main/ruby/jar-bootstrap.rb +++ b/hbase-shell/src/main/ruby/jar-bootstrap.rb @@ -37,6 +37,14 @@ # hbase hacking. include Java +# Required to access JRuby-specific internal features, such as `JRuby.runtime` +# Loading 'java' was automatically loading 'jruby' until JRuby 9.2. +# But, it has changed since JRuby 9.3. JRuby 9.3+ needs loading 'jruby' explicitly. +# +# See also: https://github.com/jruby/jruby/issues/7221#issuecomment-1133646241 +# +require 'jruby' + # Some goodies for hirb. Should these be left up to the user's discretion? if $stdin.tty? require 'irb/completion' @@ -104,7 +112,7 @@ def add_to_configuration(c, arg) opts.ordering = GetoptLong::REQUIRE_ORDER script2run = nil -log_level = org.apache.logging.log4j.Level::ERROR +log_level = 'ERROR' @shell_debug = false interactive = true full_backtrace = false @@ -118,7 +126,7 @@ def add_to_configuration(c, arg) when D_ARG conf_from_cli = add_to_configuration(conf_from_cli, arg) when '--debug' - log_level = org.apache.logging.log4j.Level::DEBUG + log_level = 'DEBUG' full_backtrace = true @shell_debug = true puts 'Setting DEBUG log level...' @@ -138,8 +146,8 @@ def add_to_configuration(c, arg) ARGV.unshift('-d') if @shell_debug # Set logging level to avoid verboseness -org.apache.logging.log4j.core.config.Configurator.setAllLevels('org.apache.zookeeper', log_level) -org.apache.logging.log4j.core.config.Configurator.setAllLevels('org.apache.hadoop', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) +org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level) # Require HBase now after setting log levels require 'hbase_constants' @@ -165,14 +173,14 @@ def debug if @shell_debug @shell_debug = false conf.back_trace_limit = 0 - log_level = org.apache.logging.log4j.Level::ERROR + log_level = 'ERROR' else @shell_debug = true conf.back_trace_limit = 100 - log_level = org.apache.logging.log4j.Level::DEBUG + log_level = 'DEBUG' end - org.apache.logging.log4j.core.config.Configurator.setAllLevels('org.apache.zookeeper', log_level) - org.apache.logging.log4j.core.config.Configurator.setAllLevels('org.apache.hadoop', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level) debug? end diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 414ab9d2bd51..46b38dd96b89 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -318,6 +318,13 @@ def get_workspace hbase_receiver.send :define_singleton_method, :exit, lambda { |rc = 0| @shell.exit(rc) } + at_exit do + # Non-deamon Netty threadpool in ZK ClientCnxnSocketNetty cannot be shut down otherwise + begin + hbase.shutdown + rescue Exception + end + end ::IRB::WorkSpace.new(hbase_receiver.get_binding) end diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java index 58ce051930d8..9270715fd8da 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java @@ -21,11 +21,13 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.jruby.embed.PathType; import org.jruby.embed.ScriptingContainer; import org.junit.AfterClass; @@ -99,7 +101,11 @@ public static void setUpBeforeClass() throws Exception { setUpConfig(); // Start mini cluster - TEST_UTIL.startMiniCluster(1); + // 3 datanodes needed for erasure coding checks + TEST_UTIL.startMiniCluster(3); + DistributedFileSystem dfs = + (DistributedFileSystem) FileSystem.get(TEST_UTIL.getConfiguration()); + dfs.enableErasureCodingPolicy("XOR-2-1-1024k"); setUpJRubyRuntime(); } diff --git a/hbase-shell/src/test/ruby/hbase/admin2_test.rb b/hbase-shell/src/test/ruby/hbase/admin2_test.rb index 83eaf38a7e59..b840231f7fc4 100644 --- a/hbase-shell/src/test/ruby/hbase/admin2_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin2_test.rb @@ -324,19 +324,24 @@ def teardown define_test 'list decommissioned regionservers' do server_name = admin.getServerNames([], true)[0].getServerName() command(:decommission_regionservers, server_name) + initial_number_of_rows = -1 begin output = capture_stdout { command(:list_decommissioned_regionservers) } puts "#{output}" assert output.include? 'DECOMMISSIONED REGION SERVERS' assert output.include? "#{server_name}" - assert output.include? '1 row(s)' + matches = output.match(/(\d+) row\(s\)/) + initial_number_of_rows = matches[1].to_i unless matches.nil? + assert initial_number_of_rows > 0 ensure command(:recommission_regionserver, server_name) output = capture_stdout { command(:list_decommissioned_regionservers) } puts "#{output}" assert output.include? 'DECOMMISSIONED REGION SERVERS' assert (output.include? "#{server_name}") ? false : true - assert output.include? '0 row(s)' + matches = output.match(/(\d+) row\(s\)/) + final_number_of_rows = matches[1].to_i unless matches.nil? + assert (initial_number_of_rows - final_number_of_rows) == 1 end end diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb index 4efcbb112765..14f393247217 100644 --- a/hbase-shell/src/test/ruby/hbase/admin_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb @@ -422,7 +422,7 @@ def teardown define_test 'clear slowlog responses should work' do output = capture_stdout { command(:clear_slowlog_responses, nil) } - assert(output.include?('Cleared Slowlog responses from 0/1 RegionServers')) + assert(output.include?('Cleared Slowlog responses from 0/3 RegionServers')) end #------------------------------------------------------------------------------- @@ -477,6 +477,7 @@ def teardown define_test "create should be able to set table options" do drop_test_table(@create_test_name) command(:create, @create_test_name, 'a', 'b', 'MAX_FILESIZE' => 12345678, + ERASURE_CODING_POLICY => 'XOR-2-1-1024k', PRIORITY => '77', FLUSH_POLICY => 'org.apache.hadoop.hbase.regionserver.FlushAllLargeStoresPolicy', REGION_MEMSTORE_REPLICATION => 'TRUE', @@ -486,6 +487,7 @@ def teardown MERGE_ENABLED => 'false') assert_equal(['a:', 'b:'], table(@create_test_name).get_all_columns.sort) assert_match(/12345678/, admin.describe(@create_test_name)) + assert_match(/XOR-2-1-1024k/, admin.describe(@create_test_name)) assert_match(/77/, admin.describe(@create_test_name)) assert_match(/'COMPACTION_ENABLED' => 'false'/, admin.describe(@create_test_name)) assert_match(/'SPLIT_ENABLED' => 'false'/, admin.describe(@create_test_name)) @@ -964,6 +966,28 @@ def teardown assert_match(/12345678/, admin.describe(@test_name)) end + define_test 'alter should be able to change EC policy' do + command(:alter, @test_name, METHOD => 'table_att', 'ERASURE_CODING_POLICY' => 'XOR-2-1-1024k') + assert_match(/XOR-2-1-1024k/, admin.describe(@test_name)) + end + + define_test 'alter should be able to remove EC policy' do + command(:alter, @test_name, METHOD => 'table_att', 'ERASURE_CODING_POLICY' => 'XOR-2-1-1024k') + command(:alter, @test_name, METHOD => 'table_att_unset', NAME => 'ERASURE_CODING_POLICY') + assert_not_match(/ERASURE_CODING_POLICY/, admin.describe(@test_name)) + end + + define_test 'alter should be able to change EC POLICY w/o table_att' do + command(:alter, @test_name, 'ERASURE_CODING_POLICY' => 'XOR-2-1-1024k') + assert_match(/XOR-2-1-1024k/, admin.describe(@test_name)) + end + + define_test 'alter should be able to remove EC POLICY w/o table_att' do + command(:alter, @test_name, 'ERASURE_CODING_POLICY' => 'XOR-2-1-1024k') + command(:alter, @test_name, 'ERASURE_CODING_POLICY' => nil) + assert_not_match(/ERASURE_CODING_POLICY/, admin.describe(@test_name)) + end + define_test "alter should be able to specify coprocessor attributes with spec string" do drop_test_table(@test_name) create_test_table(@test_name) diff --git a/hbase-shell/src/test/ruby/hbase/table_test.rb b/hbase-shell/src/test/ruby/hbase/table_test.rb index 05b80725efc8..8ed6f663dbd4 100644 --- a/hbase-shell/src/test/ruby/hbase/table_test.rb +++ b/hbase-shell/src/test/ruby/hbase/table_test.rb @@ -681,11 +681,15 @@ def teardown create_test_table(@test_name_raw) @test_table = table(@test_name_raw) - # Instert test data + # Insert test data @test_table.put(1, "x:a", 1) + sleep(1.0/1000.0) @test_table.put(2, "x:raw1", 11) + sleep(1.0/1000.0) @test_table.put(2, "x:raw1", 11) + sleep(1.0/1000.0) @test_table.put(2, "x:raw1", 11) + sleep(1.0/1000.0) @test_table.put(2, "x:raw1", 11) args = {} diff --git a/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb b/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb index 77b16df02e7c..0d2f1901438f 100644 --- a/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb +++ b/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb @@ -28,11 +28,13 @@ include Java # Set logging level to avoid verboseness - org.apache.log4j.Logger.getRootLogger.setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.hadoop.hdfs").setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.hadoop.ipc.HBaseServer").setLevel(org.apache.log4j.Level::OFF) + log_level = 'OFF' + org.apache.hadoop.hbase.logging.Log4jUtils.setRootLevel(log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hdfs', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils + .setAllLevels('org.apache.hadoop.ipc.HBaseServer', log_level) java_import org.apache.hadoop.hbase.HBaseTestingUtility diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb b/hbase-shell/src/test/ruby/shell/list_locks_test.rb index 20a910c485dd..89c6940db2a6 100644 --- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb +++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb @@ -81,7 +81,8 @@ def create_shared_lock(proc_id) "\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \ "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \ "\"lastUpdate\"=>\"0\", " \ - "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}]" \ + "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}], " \ + "\"executed\"=>false" \ "}\n\n", output) end @@ -101,7 +102,8 @@ def create_shared_lock(proc_id) "\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \ "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \ "\"lastUpdate\"=>\"0\", " \ - "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}]" \ + "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}], " \ + "\"executed\"=>false" \ "}\n\n", output) end @@ -119,7 +121,8 @@ def create_shared_lock(proc_id) "\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \ "\"procId\"=>\"1\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \ "\"lastUpdate\"=>\"0\", " \ - "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}]" \ + "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}], " \ + "\"executed\"=>false" \ "}\n\n" \ "TABLE(hbase:namespace)\n" \ "Lock type: SHARED, count: 1\n\n", @@ -143,7 +146,8 @@ def create_shared_lock(proc_id) "\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \ "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \ "\"lastUpdate\"=>\"0\", " \ - "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}]" \ + "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}], " \ + "\"executed\"=>false" \ "}\n\n", output) end @@ -168,7 +172,8 @@ def create_shared_lock(proc_id) "\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \ "\"procId\"=>\"3\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \ "\"lastUpdate\"=>\"0\", " \ - "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}]" \ + "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}], " \ + "\"executed\"=>false" \ "}\n\n", output) end @@ -198,14 +203,14 @@ def create_shared_lock(proc_id) "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \ "\"lockType\"=>\"EXCLUSIVE\", " \ "\"tableName\"=>{\"namespace\"=>\"bnM0\", \"qualifier\"=>\"dGFibGU0\"" \ - "}, \"description\"=>\"description\"}]}\n" \ + "}, \"description\"=>\"description\"}], \"executed\"=>false}\n" \ "Waiting procedures\n" \ "{\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \ "\"procId\"=>\"2\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \ "\"lastUpdate\"=>\"0\", \"stateMessage\"=>[{" \ "\"lockType\"=>\"SHARED\", " \ "\"tableName\"=>{\"namespace\"=>\"bnM0\", \"qualifier\"=>\"dGFibGU0\"}, " \ - "\"description\"=>\"description\"}]}\n" \ + "\"description\"=>\"description\"}], \"executed\"=>false}\n" \ "1 row(s)\n\n", output) end diff --git a/hbase-shell/src/test/ruby/tests_runner.rb b/hbase-shell/src/test/ruby/tests_runner.rb index 147d68103f5e..e05d11117e58 100644 --- a/hbase-shell/src/test/ruby/tests_runner.rb +++ b/hbase-shell/src/test/ruby/tests_runner.rb @@ -27,11 +27,13 @@ include Java # Set logging level to avoid verboseness - org.apache.log4j.Logger.getRootLogger.setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.hadoop.hdfs").setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(org.apache.log4j.Level::OFF) - org.apache.log4j.Logger.getLogger("org.apache.hadoop.ipc.HBaseServer").setLevel(org.apache.log4j.Level::OFF) + log_level = 'OFF' + org.apache.hadoop.hbase.logging.Log4jUtils.setRootLevel(log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hdfs', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) + org.apache.hadoop.hbase.logging.Log4jUtils + .setAllLevels('org.apache.hadoop.ipc.HBaseServer', log_level) java_import org.apache.hadoop.hbase.HBaseTestingUtility diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java index 76bc96df05c5..a02f944e12a7 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -1647,6 +1647,7 @@ public static LogQueryFilter getSlowLogQueryFromThrift(TLogQueryFilter tLogQuery tOnlineLogRecord.setRegionName(slowLogRecord.getRegionName()); tOnlineLogRecord.setResponseSize(slowLogRecord.getResponseSize()); tOnlineLogRecord.setBlockBytesScanned(slowLogRecord.getBlockBytesScanned()); + tOnlineLogRecord.setFsReadTime(slowLogRecord.getFsReadTime()); tOnlineLogRecord.setServerClass(slowLogRecord.getServerClass()); tOnlineLogRecord.setStartTime(slowLogRecord.getStartTime()); tOnlineLogRecord.setUserName(slowLogRecord.getUserName()); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java index 672b8b96d551..c3d6cba7ad20 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.client.OnlineLogRecord */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2023-02-04") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2024-01-12") public class TOnlineLogRecord implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOnlineLogRecord"); @@ -30,6 +30,7 @@ public class TOnlineLogRecord implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -112,6 +115,8 @@ public static _Fields findByThriftId(int fieldId) { return REGION_NAME; case 15: // BLOCK_BYTES_SCANNED return BLOCK_BYTES_SCANNED; + case 16: // FS_READ_TIME + return FS_READ_TIME; default: return null; } @@ -161,8 +166,9 @@ public java.lang.String getFieldName() { private static final int __MULTIMUTATIONSCOUNT_ISSET_ID = 5; private static final int __MULTISERVICECALLS_ISSET_ID = 6; private static final int __BLOCKBYTESSCANNED_ISSET_ID = 7; - private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.REGION_NAME,_Fields.BLOCK_BYTES_SCANNED}; + private static final int __FSREADTIME_ISSET_ID = 8; + private short __isset_bitfield = 0; + private static final _Fields optionals[] = {_Fields.REGION_NAME,_Fields.BLOCK_BYTES_SCANNED,_Fields.FS_READ_TIME}; public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -196,6 +202,8 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.BLOCK_BYTES_SCANNED, new org.apache.thrift.meta_data.FieldMetaData("blockBytesScanned", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.FS_READ_TIME, new org.apache.thrift.meta_data.FieldMetaData("fsReadTime", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TOnlineLogRecord.class, metaDataMap); } @@ -275,6 +283,7 @@ public TOnlineLogRecord(TOnlineLogRecord other) { this.regionName = other.regionName; } this.blockBytesScanned = other.blockBytesScanned; + this.fsReadTime = other.fsReadTime; } public TOnlineLogRecord deepCopy() { @@ -306,6 +315,8 @@ public void clear() { this.regionName = null; setBlockBytesScannedIsSet(false); this.blockBytesScanned = 0; + setFsReadTimeIsSet(false); + this.fsReadTime = 0; } public long getStartTime() { @@ -667,6 +678,29 @@ public void setBlockBytesScannedIsSet(boolean value) { __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __BLOCKBYTESSCANNED_ISSET_ID, value); } + public long getFsReadTime() { + return this.fsReadTime; + } + + public TOnlineLogRecord setFsReadTime(long fsReadTime) { + this.fsReadTime = fsReadTime; + setFsReadTimeIsSet(true); + return this; + } + + public void unsetFsReadTime() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __FSREADTIME_ISSET_ID); + } + + /** Returns true if field fsReadTime is set (has been assigned a value) and false otherwise */ + public boolean isSetFsReadTime() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __FSREADTIME_ISSET_ID); + } + + public void setFsReadTimeIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __FSREADTIME_ISSET_ID, value); + } + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { case START_TIME: @@ -789,6 +823,14 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; + case FS_READ_TIME: + if (value == null) { + unsetFsReadTime(); + } else { + setFsReadTime((java.lang.Long)value); + } + break; + } } @@ -840,6 +882,9 @@ public java.lang.Object getFieldValue(_Fields field) { case BLOCK_BYTES_SCANNED: return getBlockBytesScanned(); + case FS_READ_TIME: + return getFsReadTime(); + } throw new java.lang.IllegalStateException(); } @@ -881,6 +926,8 @@ public boolean isSet(_Fields field) { return isSetRegionName(); case BLOCK_BYTES_SCANNED: return isSetBlockBytesScanned(); + case FS_READ_TIME: + return isSetFsReadTime(); } throw new java.lang.IllegalStateException(); } @@ -1033,6 +1080,15 @@ public boolean equals(TOnlineLogRecord that) { return false; } + boolean this_present_fsReadTime = true && this.isSetFsReadTime(); + boolean that_present_fsReadTime = true && that.isSetFsReadTime(); + if (this_present_fsReadTime || that_present_fsReadTime) { + if (!(this_present_fsReadTime && that_present_fsReadTime)) + return false; + if (this.fsReadTime != that.fsReadTime) + return false; + } + return true; } @@ -1086,6 +1142,10 @@ public int hashCode() { if (isSetBlockBytesScanned()) hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(blockBytesScanned); + hashCode = hashCode * 8191 + ((isSetFsReadTime()) ? 131071 : 524287); + if (isSetFsReadTime()) + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(fsReadTime); + return hashCode; } @@ -1247,6 +1307,16 @@ public int compareTo(TOnlineLogRecord other) { return lastComparison; } } + lastComparison = java.lang.Boolean.compare(isSetFsReadTime(), other.isSetFsReadTime()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFsReadTime()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.fsReadTime, other.fsReadTime); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -1359,6 +1429,12 @@ public java.lang.String toString() { sb.append(this.blockBytesScanned); first = false; } + if (isSetFsReadTime()) { + if (!first) sb.append(", "); + sb.append("fsReadTime:"); + sb.append(this.fsReadTime); + first = false; + } sb.append(")"); return sb.toString(); } @@ -1549,6 +1625,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TOnlineLogRecord st org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 16: // FS_READ_TIME + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.fsReadTime = iprot.readI64(); + struct.setFsReadTimeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1648,6 +1732,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TOnlineLogRecord s oprot.writeI64(struct.blockBytesScanned); oprot.writeFieldEnd(); } + if (struct.isSetFsReadTime()) { + oprot.writeFieldBegin(FS_READ_TIME_FIELD_DESC); + oprot.writeI64(struct.fsReadTime); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1685,13 +1774,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TOnlineLogRecord st if (struct.isSetBlockBytesScanned()) { optionals.set(1); } - oprot.writeBitSet(optionals, 2); + if (struct.isSetFsReadTime()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); if (struct.isSetRegionName()) { oprot.writeString(struct.regionName); } if (struct.isSetBlockBytesScanned()) { oprot.writeI64(struct.blockBytesScanned); } + if (struct.isSetFsReadTime()) { + oprot.writeI64(struct.fsReadTime); + } } @Override @@ -1723,7 +1818,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TOnlineLogRecord str struct.setMultiMutationsCountIsSet(true); struct.multiServiceCalls = iprot.readI32(); struct.setMultiServiceCallsIsSet(true); - java.util.BitSet incoming = iprot.readBitSet(2); + java.util.BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { struct.regionName = iprot.readString(); struct.setRegionNameIsSet(true); @@ -1732,6 +1827,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TOnlineLogRecord str struct.blockBytesScanned = iprot.readI64(); struct.setBlockBytesScannedIsSet(true); } + if (incoming.get(2)) { + struct.fsReadTime = iprot.readI64(); + struct.setFsReadTimeIsSet(true); + } } } diff --git a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift index a32f266cf313..ed3fdf32b973 100644 --- a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift +++ b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift @@ -499,6 +499,7 @@ struct TOnlineLogRecord { 13: required i32 multiServiceCalls 14: optional string regionName 15: optional i64 blockBytesScanned + 16: optional i64 fsReadTime } // diff --git a/pom.xml b/pom.xml index 2b1da88e056f..b8ac1067cf4a 100644 --- a/pom.xml +++ b/pom.xml @@ -758,6 +758,7 @@ hbase-asyncfs hbase-logging hbase-compression + hbase-extensions scm:git:git://gitbox.apache.org/repos/asf/hbase.git @@ -803,7 +804,7 @@ 3.5.0 ${compileSource} - 3.2.4 + 3.3.5 ${hadoop-three.version} @@ -839,14 +840,14 @@ 4.5.13 4.4.13 3.2.6 - 2.14.1 - 2.14.1 + 2.15.2 + 2.15.2 2.3.1 3.1.0 2.1.1 2.3.2 3.0.1-b08 - 9.3.9.0 + 9.3.13.0 4.13.2 1.3 1.15.0 @@ -863,8 +864,8 @@ 2.4.1 1.5.4 - 2.1.43 - 1.0.57 + 2.2.1 + 1.0.58 2.12.2 1.76 1.5.1 @@ -971,8 +972,11 @@ "-Djava.library.path=${hadoop.library.path};${java.library.path}" -Dorg.apache.hbase.thirdparty.io.netty.leakDetection.level=advanced -Dio.opentelemetry.context.enableStrictContext=true + -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true --add-modules jdk.unsupported + --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED @@ -981,8 +985,14 @@ --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-exports java.base/jdk.internal.misc=ALL-UNNAMED - --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED - --add-opens java.base/jdk.internal.util.random=ALL-UNNAMED + --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED + --add-exports java.base/sun.net.dns=ALL-UNNAMED + --add-exports java.base/sun.net.util=ALL-UNNAMED + + --add-opens java.base/jdk.internal.util.random=ALL-UNNAMED + --add-opens java.base/sun.security.x509=ALL-UNNAMED + --add-opens java.base/sun.security.util=ALL-UNNAMED ${hbase-surefire.argLine} @{jacocoArgLine} 1.5.1 @@ -1277,6 +1287,11 @@ hbase-shaded-mapreduce ${project.version} + + org.apache.hbase + hbase-shaded-testing-util + ${project.version} + org.apache.hbase hbase-asyncfs @@ -1319,6 +1334,11 @@ hbase-compression-zstd ${project.version} + + org.apache.hbase + hbase-openssl + ${project.version} + com.github.stephenc.findbugs @@ -1737,6 +1757,11 @@ hbase-shaded-netty ${hbase-thirdparty.version} + + org.apache.hbase.thirdparty + hbase-shaded-netty-tcnative + ${hbase-thirdparty.version} + org.apache.hbase.thirdparty hbase-shaded-protobuf @@ -2991,6 +3016,7 @@ devapidocs Developer API The full HBase API, including private and unstable APIs + Apache HBase™ ${project.version} API **/generated/* **/protobuf/* @@ -3039,8 +3065,9 @@ testdevapidocs - Developer API + Test Developer API The full HBase API test code, including private and unstable APIs + Apache HBase™ ${project.version} Test API **/generated/* **/protobuf/* @@ -3100,6 +3127,7 @@ apidocs User API The HBase Application Programmer's API + Apache HBase™ ${project.version} API org.apache.hadoop.hbase.backup*:org.apache.hadoop.hbase.catalog:org.apache.hadoop.hbase.client.coprocessor:org.apache.hadoop.hbase.client.metrics:org.apache.hadoop.hbase.codec*:org.apache.hadoop.hbase.constraint:org.apache.hadoop.hbase.coprocessor.*:org.apache.hadoop.hbase.executor:org.apache.hadoop.hbase.fs:*.generated.*:org.apache.hadoop.hbase.io.hfile.*:org.apache.hadoop.hbase.mapreduce.hadoopbackport:org.apache.hadoop.hbase.mapreduce.replication:org.apache.hadoop.hbase.master.*:org.apache.hadoop.hbase.metrics*:org.apache.hadoop.hbase.migration:org.apache.hadoop.hbase.monitoring:org.apache.hadoop.hbase.p*:org.apache.hadoop.hbase.regionserver.compactions:org.apache.hadoop.hbase.regionserver.handler:org.apache.hadoop.hbase.regionserver.snapshot:org.apache.hadoop.hbase.replication.*:org.apache.hadoop.hbase.rest.filter:org.apache.hadoop.hbase.rest.model:org.apache.hadoop.hbase.rest.p*:org.apache.hadoop.hbase.security.*:org.apache.hadoop.hbase.thrift*:org.apache.hadoop.hbase.tmpl.*:org.apache.hadoop.hbase.tool:org.apache.hadoop.hbase.trace:org.apache.hadoop.hbase.util.byterange*:org.apache.hadoop.hbase.util.test:org.apache.hadoop.hbase.util.vint:org.apache.hadoop.metrics2*:org.apache.hadoop.hbase.io.compress* false @@ -3155,8 +3183,9 @@ true testapidocs - User API - The HBase Application Programmer's API + Test User API + The HBase Application Programmer's API test code + Apache HBase™ ${project.version} Test API org.apache.hadoop.hbase.backup*:org.apache.hadoop.hbase.catalog:org.apache.hadoop.hbase.client.coprocessor:org.apache.hadoop.hbase.client.metrics:org.apache.hadoop.hbase.codec*:org.apache.hadoop.hbase.constraint:org.apache.hadoop.hbase.coprocessor.*:org.apache.hadoop.hbase.executor:org.apache.hadoop.hbase.fs:*.generated.*:org.apache.hadoop.hbase.io.hfile.*:org.apache.hadoop.hbase.mapreduce.hadoopbackport:org.apache.hadoop.hbase.mapreduce.replication:org.apache.hadoop.hbase.master.*:org.apache.hadoop.hbase.metrics*:org.apache.hadoop.hbase.migration:org.apache.hadoop.hbase.monitoring:org.apache.hadoop.hbase.p*:org.apache.hadoop.hbase.regionserver.compactions:org.apache.hadoop.hbase.regionserver.handler:org.apache.hadoop.hbase.regionserver.snapshot:org.apache.hadoop.hbase.replication.*:org.apache.hadoop.hbase.rest.filter:org.apache.hadoop.hbase.rest.model:org.apache.hadoop.hbase.rest.p*:org.apache.hadoop.hbase.security.*:org.apache.hadoop.hbase.thrift*:org.apache.hadoop.hbase.tmpl.*:org.apache.hadoop.hbase.tool:org.apache.hadoop.hbase.trace:org.apache.hadoop.hbase.util.byterange*:org.apache.hadoop.hbase.util.test:org.apache.hadoop.hbase.util.vint:org.apache.hadoop.metrics2*:org.apache.hadoop.hbase.io.compress* false diff --git a/src/main/asciidoc/_chapters/bulk_data_generator_tool.adoc b/src/main/asciidoc/_chapters/bulk_data_generator_tool.adoc index 3ac6ca693121..b04fcdeb7264 100644 --- a/src/main/asciidoc/_chapters/bulk_data_generator_tool.adoc +++ b/src/main/asciidoc/_chapters/bulk_data_generator_tool.adoc @@ -18,8 +18,8 @@ * limitations under the License. */ //// - -== Bulk Data Generator Tool +[[BulkDataGeneratorTool]] += Bulk Data Generator Tool :doctype: book :numbered: :toc: left @@ -29,7 +29,7 @@ This is a random data generator tool for HBase tables leveraging Hbase bulk load. It can create pre-splited HBase table and the generated data is *uniformly distributed* to all the regions of the table. -=== How to Use +== Usage [source] ---- @@ -53,7 +53,7 @@ hbase org.apache.hadoop.hbase.util.bulkdatagenerator.BulkDataGeneratorTool -t TE hbase org.apache.hadoop.hbase.util.bulkdatagenerator.BulkDataGeneratorTool -t TEST_TABLE -mc 10 -r 100 -sc 10 -Dmapreduce.map.memory.mb=8192 ---- -=== How it Works +== Overview ==== Table Schema Tool generates a HBase table with single column family, i.e. *cf* and 9 columns i.e. diff --git a/src/main/asciidoc/_chapters/hbase-default.adoc b/src/main/asciidoc/_chapters/hbase-default.adoc index 69fb4a0eae66..03391cc38b1a 100644 --- a/src/main/asciidoc/_chapters/hbase-default.adoc +++ b/src/main/asciidoc/_chapters/hbase-default.adoc @@ -761,6 +761,17 @@ Period at which the region balancer runs in the Master. `300000` +[[hbase.master.oldwals.dir.updater.period]] +*`hbase.master.oldwals.dir.updater.period`*:: ++ +.Description +Period at which the oldWALs directory size calculator/updater will run in the Master. + ++ +.Default +`300000` + + [[hbase.regions.slop]] *`hbase.regions.slop`*:: + diff --git a/src/main/asciidoc/_chapters/store_file_tracking.adoc b/src/main/asciidoc/_chapters/store_file_tracking.adoc index 74d802f386c5..b6c1f7e73399 100644 --- a/src/main/asciidoc/_chapters/store_file_tracking.adoc +++ b/src/main/asciidoc/_chapters/store_file_tracking.adoc @@ -143,3 +143,22 @@ example, that would be as follows: ---- alter 'my-table', CONFIGURATION => {'hbase.store.file-tracker.impl' => 'FILE'} ---- + +### Specifying trackers during snapshot recovery + +It's also possible to specify a given store file tracking implementation when recovering a snapshot +using the _CLONE_SFT_ option of _clone_snasphot_ command. This is useful when recovering old +snapshots, taken prior to a change in the global configuration, or if the snapshot has been +imported from a different cluster that had a different store file tracking setting. +Because snapshots preserve table and colum family descriptors, a simple restore would reload +the original configuration, requiring the additional steps described above to convert the +table/column family to the desired tracker implementation. +An example of how to use _clone_snapshot_ to specify the *FILE* tracker implementation +is shown below: + +---- +clone_snapshot 'snapshotName', 'namespace:tableName', {CLONE_SFT=>'FILE'} +---- + +NOTE: The option to specify the tracker during snapshot recovery is only available for the +_clone_snapshot_ command. The _restore_snapshot_ command does not support this parameter. diff --git a/src/site/xdoc/downloads.xml b/src/site/xdoc/downloads.xml index 7e163612f96b..e9e9d97ebaed 100644 --- a/src/site/xdoc/downloads.xml +++ b/src/site/xdoc/downloads.xml @@ -45,51 +45,51 @@ under the License. - 3.0.0-alpha-4 + 3.0.0-beta-1 - 2023/06/07 + 2024/01/14 - 3.0.0-alpha-4 vs 2.0.0 + 3.0.0-beta-1 vs 2.0.0 - Changes + Changes - Release Notes + Release Notes - src (sha512 asc)
- bin (sha512 asc)
- client-bin (sha512 asc) + src (sha512 asc)
+ bin (sha512 asc)
+ client-bin (sha512 asc) - Testing only, not production ready + Feature freeze, passed a 10B ITBLL run, use with caution - 2.5.6 + 2.5.7 - 2023/10/20 + 2023/12/24 - 2.5.6 vs 2.5.5 + 2.5.7 vs 2.5.6 - Changes + Changes - Release Notes + Release Notes - src (sha512 asc)
- bin (sha512 asc)
- client-bin (sha512 asc)
- hadoop3-bin (sha512 asc)
- hadoop3-client-bin (sha512 asc) + src (sha512 asc)
+ bin (sha512 asc)
+ client-bin (sha512 asc)
+ hadoop3-bin (sha512 asc)
+ hadoop3-client-bin (sha512 asc) stable release