apache · guluo2016 · Nov 30, 2023 · Nov 30, 2023 · Nov 30, 2023 · Dec 1, 2023
diff --git a/.asf.yaml b/.asf.yaml
@@ -38,4 +38,4 @@ notifications:
   commits:      [email protected]
   issues:       [email protected]
   pullrequests: [email protected]
-  jira_options: link
+  jira_options: link label
diff --git a/bin/draining_servers.rb b/bin/draining_servers.rb
@@ -25,7 +25,6 @@
 
 java_import org.apache.hadoop.hbase.HBaseConfiguration
 java_import org.apache.hadoop.hbase.client.ConnectionFactory
-java_import org.apache.hadoop.hbase.client.HBaseAdmin
 java_import org.apache.hadoop.hbase.zookeeper.ZKUtil
 java_import org.apache.hadoop.hbase.zookeeper.ZNodePaths
 java_import org.slf4j.LoggerFactory
@@ -136,8 +135,12 @@ def listServers(_options)
 
 hostOrServers = ARGV[1..ARGV.size]
 
-# Create a logger and save it to ruby global
-$LOG = LoggerFactory.getLogger(NAME)
+# disable debug/info logging on this script for clarity
+log_level = 'ERROR'
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level)
+
 case ARGV[0]
 when 'add'
   if ARGV.length < 2

diff --git a/bin/get-active-master.rb b/bin/get-active-master.rb
@@ -24,9 +24,10 @@
 java_import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker
 
 # disable debug/info logging on this script for clarity
-log_level = org.apache.log4j.Level::ERROR
-org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level)
-org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level)
+log_level = 'ERROR'
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level)
 
 config = HBaseConfiguration.create
 

diff --git a/bin/hbase b/bin/hbase
@@ -496,7 +496,8 @@ add_jdk11_deps_to_classpath() {
 }
 
 add_jdk11_jvm_flags() {
-  HBASE_OPTS="$HBASE_OPTS -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true --add-modules jdk.unsupported --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED"
+  # Keep in sync with hbase-surefire.jdk11.flags in the root pom.xml
+  HBASE_OPTS="$HBASE_OPTS -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true --add-modules jdk.unsupported --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-exports java.base/sun.net.dns=ALL-UNNAMED --add-exports java.base/sun.net.util=ALL-UNNAMED"
 }
 
 add_opentelemetry_agent() {
@@ -786,7 +787,7 @@ fi
 # Add lib/jdk11 jars to the classpath
 
 if [ "${DEBUG}" = "true" ]; then
-  echo "Deciding on addition of lib/jdk11 jars to the classpath"
+  echo "Deciding on addition of lib/jdk11 jars to the classpath and setting JVM module flags"
 fi
 
 addJDK11Jars=false
@@ -879,6 +880,7 @@ export CLASSPATH
 if [ "${DEBUG}" = "true" ]; then
   echo "classpath=${CLASSPATH}" >&2
   HBASE_OPTS="${HBASE_OPTS} -Xdiag"
+  echo "HBASE_OPTS=${HBASE_OPTS}"
 fi
 
 # resolve the command arguments

diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
@@ -178,8 +178,12 @@ EOF
 fi
 
 function read_java_version() {
-  properties="$("${JAVA_HOME}/bin/java" -XshowSettings:properties -version 2>&1)"
-  echo "${properties}" | "${GREP}" java.runtime.version | head -1 | "${SED}" -e 's/.* = \([^ ]*\)/\1/'
+  # Avoid calling java repeatedly
+  if [ -z "$read_java_version_cached" ]; then
+    properties="$("${JAVA_HOME}/bin/java" -XshowSettings:properties -version 2>&1)"
+    read_java_version_cached="$(echo "${properties}" | "${GREP}" java.runtime.version | head -1 | "${SED}" -e 's/.* = \([^ ]*\)/\1/')"
+  fi
+  echo "$read_java_version_cached"
 }
 
 # Inspect the system properties exposed by this JVM to identify the major

diff --git a/bin/replication/copy_tables_desc.rb b/bin/replication/copy_tables_desc.rb
@@ -27,11 +27,8 @@
 java_import org.apache.hadoop.conf.Configuration
 java_import org.apache.hadoop.hbase.HBaseConfiguration
 java_import org.apache.hadoop.hbase.HConstants
-java_import org.apache.hadoop.hbase.HTableDescriptor
 java_import org.apache.hadoop.hbase.TableName
 java_import org.apache.hadoop.hbase.client.ConnectionFactory
-java_import org.apache.hadoop.hbase.client.HBaseAdmin
-java_import org.slf4j.LoggerFactory
 
 # Name of this script
 NAME = 'copy_tables_desc'.freeze
@@ -45,7 +42,7 @@ def usage
 def copy(src, dst, table)
   # verify if table exists in source cluster
   begin
-    t = src.getTableDescriptor(TableName.valueOf(table))
+    t = src.getDescriptor(TableName.valueOf(table))
   rescue org.apache.hadoop.hbase.TableNotFoundException
     puts format("Source table \"%s\" doesn't exist, skipping.", table)
     return
@@ -62,9 +59,13 @@ def copy(src, dst, table)
   puts format('Schema for table "%s" was succesfully copied to remote cluster.', table)
 end
 
-usage if ARGV.size < 2 || ARGV.size > 3
+# disable debug/info logging on this script for clarity
+log_level = 'ERROR'
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level)
 
-LOG = LoggerFactory.getLogger(NAME)
+usage if ARGV.size < 2 || ARGV.size > 3
 
 parts1 = ARGV[0].split(':')
 

diff --git a/bin/shutdown_regionserver.rb b/bin/shutdown_regionserver.rb
@@ -24,7 +24,6 @@
 
 include Java
 java_import org.apache.hadoop.hbase.HBaseConfiguration
-java_import org.apache.hadoop.hbase.client.HBaseAdmin
 java_import org.apache.hadoop.hbase.client.ConnectionFactory
 
 def usage(msg = nil)
@@ -35,6 +34,12 @@ def usage(msg = nil)
   abort
 end
 
+# disable debug/info logging on this script for clarity
+log_level = 'ERROR'
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level)
+org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop', log_level)
+
 usage if ARGV.empty?
 
 ARGV.each do |x|

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
@@ -91,7 +91,7 @@ pipeline {
                   rm -rf "${YETUS_DIR}"
                   "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
                       --working-dir "${WORKSPACE}/downloads-yetus" \
-                      --keys 'https://www.apache.org/dist/yetus/KEYS' \
+                      --keys 'https://downloads.apache.org/yetus/KEYS' \
                       --verify-tar-gz \
                       "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
                       "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
@@ -138,7 +138,7 @@ pipeline {
               echo "Ensure we have a copy of Hadoop ${HADOOP2_VERSION}"
               "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
                   --working-dir "${WORKSPACE}/downloads-hadoop-2" \
-                  --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
+                  --keys 'https://downloads.apache.org/hadoop/common/KEYS' \
                   --verify-tar-gz \
                   "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \
                   "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz"
@@ -152,7 +152,7 @@ pipeline {
         }
         stage ('hadoop 3 cache') {
           environment {
-            HADOOP3_VERSION="3.1.1"
+            HADOOP3_VERSION="3.3.5"
           }
           steps {
             // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :(
@@ -166,7 +166,7 @@ pipeline {
               echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}"
               "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
                   --working-dir "${WORKSPACE}/downloads-hadoop-3" \
-                  --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
+                  --keys 'https://downloads.apache.org/hadoop/common/KEYS' \
                   --verify-tar-gz \
                   "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \
                   "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz"
@@ -661,6 +661,9 @@ pipeline {
               rm -rf "hadoop-3" && mkdir "hadoop-3"
               rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
               rm -rf ".m2-for-src" && mkdir ".m2-for-src"
+              # remove old hadoop tarballs in workspace
+              rm -rf hadoop-2*.tar.gz
+              rm -rf hadoop-3*.tar.gz
               echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile
               echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile
             '''
@@ -726,6 +729,8 @@ pipeline {
               echo "Attempting to use run an instance on top of Hadoop 3."
               artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
               tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
+              # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471
+              ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3
               if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
                   --single-process \
                   --working-dir output-integration/hadoop-3 \

diff --git a/...-26220 Use P2P communicate between region servers to sync the list for bootstrap node.pdf b/...-26220 Use P2P communicate between region servers to sync the list for bootstrap node.pdf
diff --git a/dev-support/design-docs/HBASE-28196 Yield SCP and TRSP when they are blocked.pdf b/dev-support/design-docs/HBASE-28196 Yield SCP and TRSP when they are blocked.pdf
diff --git a/dev-support/flaky-tests/python-requirements.txt b/dev-support/flaky-tests/python-requirements.txt
@@ -17,6 +17,6 @@
 #
 requests==2.31.0
 future==0.18.3
-gitpython==3.1.37
+gitpython==3.1.41
 rbtools==4.0
-jinja2==3.1.2
+jinja2==3.1.3
diff --git a/dev-support/git-jira-release-audit/requirements.txt b/dev-support/git-jira-release-audit/requirements.txt
@@ -19,11 +19,11 @@ blessed==1.17.0
 certifi==2023.7.22
 cffi==1.13.2
 chardet==3.0.4
-cryptography==41.0.6
+cryptography==42.0.2
 defusedxml==0.6.0
 enlighten==1.4.0
 gitdb2==2.0.6
-GitPython==3.1.37
+GitPython==3.1.41
 idna==2.8
 jira==2.0.0
 oauthlib==3.1.0

diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
@@ -602,13 +602,20 @@ function hadoopcheck_rebuild
     else
       hbase_hadoop3_versions="3.1.1 3.1.2 3.1.3 3.1.4 3.2.0 3.2.1 3.2.2 3.2.3 3.2.4 3.3.0 3.3.1 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6"
     fi
-  else
-    yetus_info "Setting Hadoop 3 versions to test based on branch-2.5+/master/feature branch rules"
+  elif [[ "${PATCH_BRANCH}" = branch-2.5 ]]; then
+    yetus_info "Setting Hadoop 3 versions to test based on branch-2.5 rules"
     if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
       hbase_hadoop3_versions="3.2.4 3.3.6"
     else
       hbase_hadoop3_versions="3.2.3 3.2.4 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6"
     fi
+  else
+    yetus_info "Setting Hadoop 3 versions to test based on branch-2.6+/master/feature branch rules"
+    if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
+      hbase_hadoop3_versions="3.3.6"
+    else
+      hbase_hadoop3_versions="3.3.5 3.3.6"
+    fi
   fi
 
   export MAVEN_OPTS="${MAVEN_OPTS}"

diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -324,7 +324,7 @@ redirect_and_run "${working_dir}/hadoop_cluster_smoke" \
     "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
 
 echo "Starting up HBase"
-HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/start-hbase.sh"
+HBASE_CONF_DIR="${working_dir}/hbase-conf/" HBASE_LOG_DIR="${working_dir}" "${component_install}/bin/start-hbase.sh"
 
 sleep_time=2
 until "${component_install}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/waiting_hbase_startup.log" 2>&1 <<EOF

diff --git a/dev-support/patch-hadoop3.sh b/dev-support/patch-hadoop3.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+hadoop_dir=$1
+
+sed -i "s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g" "$hadoop_dir/libexec/hadoop-functions.sh"
+sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" "$hadoop_dir/bin/mapred"
+curl https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar -o "$hadoop_dir/share/hadoop/tools/lib/mockito-core-2.28.2.jar"
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
@@ -61,6 +61,10 @@
       <!-- Overriding the scope in depMgmt -->
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-openssl</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-server</artifactId>

diff --git a/...fs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java b/...fs/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestFanOutOneBlockAsyncDFSOutput.java
@@ -52,10 +52,12 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
+import org.junit.FixMethodOrder;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
+import org.junit.runners.MethodSorters;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -65,6 +67,7 @@
 import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
 import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
 
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
 @Category({ MiscTests.class, MediumTests.class })
 public class TestFanOutOneBlockAsyncDFSOutput extends AsyncFSTestBase {
 
@@ -142,8 +145,12 @@ public void test() throws IOException, InterruptedException, ExecutionException
     writeAndVerify(FS, f, out);
   }
 
+  /**
+   * Test method has been renamed to be the first in NAME_ASCENDING. It's an ugly hack to avoid
+   * flakiness.
+   */
   @Test
-  public void testRecover() throws IOException, InterruptedException, ExecutionException {
+  public void test0Recover() throws IOException, InterruptedException, ExecutionException {
     Path f = new Path("/" + name.getMethodName());
     EventLoop eventLoop = EVENT_LOOP_GROUP.next();
     FanOutOneBlockAsyncDFSOutput out = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true,

diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml
@@ -105,6 +105,12 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <!-- Required by tests using MiniMRCluster -->
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpkix-jdk18on</artifactId>
+      <scope>test</scope>
+    </dependency>
     <!-- General dependencies -->
     <dependency>
       <groupId>org.apache.commons</groupId>

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/Abortable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/Abortable.java
@@ -37,7 +37,7 @@ public interface Abortable {
   void abort(String why, Throwable e);
 
   /**
-   * It just call another abort method and the Throwable parameter is null.
+   * It just calls another abort method and the Throwable parameter is null.
    * @param why Why we're aborting.
    * @see Abortable#abort(String, Throwable)
    */