HBASE-26097: to resolve hbase version 2.x protobuf version conflict

.gitignore

@@ -20,3 +20,5 @@ linklint-*.zip
 linklint/
 .checkstyle
 **/.checkstyle
+.java-version
+tmp

.pylintrc

@@ -133,6 +133,9 @@ dummy-variables-rgx=_|dummy
 # you should avoid to define new builtins when possible.
 additional-builtins=
 
+# List of modules that can redefine builtins. (For python 2/3 compatibility)
+redefining-builtins-modules=builtins
+
 
 [TYPECHECK]
 

.rubocop.yml

@@ -0,0 +1,11 @@
+Naming/HeredocDelimiterNaming:
+  Enabled: false
+
+Layout/HeredocIndentation:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 100
+
+Metrics/MethodLength:
+  Max: 75

NOTICE.txt

@@ -1,5 +1,5 @@
 Apache HBase
-Copyright 2007-2018 The Apache Software Foundation
+Copyright 2007-2020 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
@@ -45,3 +45,6 @@ This product includes portions of the Guava project v14 and v21, specifically
 Copyright (C) 2007 The Guava Authors
 
 Licensed under the Apache License, Version 2.0
+--
+This product includes portions of Jetty project, specially
+'hbase-shaded-hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/mortbay/jetty/webapp/webdefault.xml'

bin/chaos-daemon.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+
+usage="Usage: chaos-daemon.sh (start|stop) chaosagent"
+
+# if no args specified, show usage
+if [ $# -le 1 ]; then
+  echo "$usage"
+  exit 1
+fi
+
+# get arguments
+startStop=$1
+shift
+
+command=$1
+shift
+
+check_before_start(){
+    #ckeck if the process is not running
+    mkdir -p "$HBASE_PID_DIR"
+    if [ -f "$CHAOS_PID" ]; then
+      if kill -0 "$(cat "$CHAOS_PID")" > /dev/null 2>&1; then
+        echo "$command" running as process "$(cat "$CHAOS_PID")".  Stop it first.
+        exit 1
+      fi
+    fi
+}
+
+bin=`dirname "${BASH_SOURCE-$0}"`
+bin=$(cd "$bin">/dev/null || exit; pwd)
+
+. "$bin"/hbase-config.sh
+. "$bin"/hbase-common.sh
+
+CLASSPATH=$HBASE_CONF_DIR
+for f in ../lib/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f
+done
+
+# get log directory
+if [ "$HBASE_LOG_DIR" = "" ]; then
+  export HBASE_LOG_DIR="$HBASE_HOME/logs"
+fi
+
+if [ "$HBASE_PID_DIR" = "" ]; then
+  HBASE_PID_DIR=/tmp
+fi
+
+if [ "$HBASE_IDENT_STRING" = "" ]; then
+  export HBASE_IDENT_STRING="$USER"
+fi
+
+if [ "$JAVA_HOME" != "" ]; then
+  #echo "run java in $JAVA_HOME"
+  JAVA_HOME=$JAVA_HOME
+fi
+if [ "$JAVA_HOME" = "" ]; then
+  echo "Error: JAVA_HOME is not set."
+  exit 1
+fi
+
+export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME
+export CHAOS_LOGFILE=$HBASE_LOG_PREFIX.log
+
+if [ -z "${HBASE_ROOT_LOGGER}" ]; then
+export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"}
+fi
+
+if [ -z "${HBASE_SECURITY_LOGGER}" ]; then
+export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"}
+fi
+
+CHAOS_LOGLOG=${CHAOS_LOGLOG:-"${HBASE_LOG_DIR}/${CHAOS_LOGFILE}"}
+CHAOS_PID=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid
+
+if [ -z "$CHAOS_JAVA_OPTS" ]; then
+  CHAOS_JAVA_OPTS="-Xms1024m -Xmx4096m"
+fi
+
+case $startStop in
+
+(start)
+    check_before_start
+    echo running $command
+    CMD="${JAVA_HOME}/bin/java -Dapp.home=${HBASE_CONF_DIR}/../  ${CHAOS_JAVA_OPTS} -cp ${CLASSPATH} org.apache.hadoop.hbase.chaos.ChaosService -$command start &>> ${CHAOS_LOGLOG} &"
+
+    eval $CMD
+    PID=$(echo $!)
+    echo ${PID} >${CHAOS_PID}
+
+    echo "Chaos ${1} process Started with ${PID} !"
+    now=$(date)
+    echo "${now} Chaos ${1} process Started with ${PID} !" >>${CHAOS_LOGLOG}
+    ;;
+
+(stop)
+    echo stopping $command
+    if [ -f $CHAOS_PID ]; then
+      pidToKill=`cat $CHAOS_PID`
+      # kill -0 == see if the PID exists
+      if kill -0 $pidToKill > /dev/null 2>&1; then
+        echo -n stopping $command
+        echo "`date` Terminating $command" >> $CHAOS_LOGLOG
+        kill $pidToKill > /dev/null 2>&1
+        waitForProcessEnd $pidToKill $command
+      else
+        retval=$?
+        echo no $command to stop because kill -0 of pid $pidToKill failed with status $retval
+      fi
+    else
+      echo no $command to stop because no pid file $CHAOS_PID
+    fi
+    rm -f $CHAOS_PID
+  ;;
+
+(*)
+  echo $usage
+  exit 1
+  ;;
+
+esac

bin/draining_servers.rb

@@ -27,6 +27,7 @@
 java_import org.apache.hadoop.hbase.client.ConnectionFactory
 java_import org.apache.hadoop.hbase.client.HBaseAdmin
 java_import org.apache.hadoop.hbase.zookeeper.ZKUtil
+java_import org.apache.hadoop.hbase.zookeeper.ZNodePaths
 java_import org.slf4j.LoggerFactory
 
 # Name of this script
@@ -50,28 +51,29 @@
 def getServers(admin)
   serverInfos = admin.getClusterStatus.getServers
   servers = []
-  for server in serverInfos
+  serverInfos.each do |server|
     servers << server.getServerName
   end
   servers
 end
 
+# rubocop:disable Metrics/AbcSize
 def getServerNames(hostOrServers, config)
   ret = []
   connection = ConnectionFactory.createConnection(config)
 
-  for hostOrServer in hostOrServers
+  hostOrServers.each do |host_or_server|
     # check whether it is already serverName. No need to connect to cluster
-    parts = hostOrServer.split(',')
+    parts = host_or_server.split(',')
     if parts.size == 3
-      ret << hostOrServer
+      ret << host_or_server
     else
-      admin = connection.getAdmin unless admin
+      admin ||= connection.getAdmin
       servers = getServers(admin)
 
-      hostOrServer = hostOrServer.tr(':', ',')
-      for server in servers
-        ret << server if server.start_with?(hostOrServer)
+      host_or_server = host_or_server.tr(':', ',')
+      servers.each do |server|
+        ret << server if server.start_with?(host_or_server)
       end
     end
   end
@@ -86,11 +88,11 @@ def addServers(_options, hostOrServers)
   servers = getServerNames(hostOrServers, config)
 
   zkw = org.apache.hadoop.hbase.zookeeper.ZKWatcher.new(config, 'draining_servers', nil)
-  parentZnode = zkw.znodePaths.drainingZNode
 
   begin
-    for server in servers
-      node = ZKUtil.joinZNode(parentZnode, server)
+    parentZnode = zkw.getZNodePaths.drainingZNode
+    servers.each do |server|
+      node = ZNodePaths.joinZNode(parentZnode, server)
       ZKUtil.createAndFailSilent(zkw, node)
     end
   ensure
@@ -103,27 +105,32 @@ def removeServers(_options, hostOrServers)
   servers = getServerNames(hostOrServers, config)
 
   zkw = org.apache.hadoop.hbase.zookeeper.ZKWatcher.new(config, 'draining_servers', nil)
-  parentZnode = zkw.znodePaths.drainingZNode
 
   begin
-    for server in servers
-      node = ZKUtil.joinZNode(parentZnode, server)
+    parentZnode = zkw.getZNodePaths.drainingZNode
+    servers.each do |server|
+      node = ZNodePaths.joinZNode(parentZnode, server)
       ZKUtil.deleteNodeFailSilent(zkw, node)
     end
   ensure
     zkw.close
   end
 end
+# rubocop:enable Metrics/AbcSize
 
 # list servers in draining mode
 def listServers(_options)
   config = HBaseConfiguration.create
 
   zkw = org.apache.hadoop.hbase.zookeeper.ZKWatcher.new(config, 'draining_servers', nil)
-  parentZnode = zkw.znodePaths.drainingZNode
 
-  servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode)
-  servers.each { |server| puts server }
+  begin
+    parentZnode = zkw.getZNodePaths.drainingZNode
+    servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode)
+    servers.each { |server| puts server }
+  ensure
+    zkw.close
+  end
 end
 
 hostOrServers = ARGV[1..ARGV.size]

bin/graceful_stop.sh

@@ -22,7 +22,8 @@
 # Turn off the balancer before running this script.
 function usage {
   echo "Usage: graceful_stop.sh [--config <conf-dir>] [-e] [--restart [--reload]] [--thrift] \
-[--rest]  [-nob |--nobalancer ] <hostname>"
+[--rest] [-n |--noack] [--maxthreads <number of threads>] [--movetimeout <timeout in seconds>] \
+[-nob |--nobalancer] [-d |--designatedfile <file path>] [-x |--excludefile <file path>] <hostname>"
   echo " thrift         If we should stop/start thrift before/after the hbase stop/start"
   echo " rest           If we should stop/start rest before/after the hbase stop/start"
   echo " restart        If we should restart after graceful stop"
@@ -32,10 +33,13 @@ moving regions"
   echo " maxthreads xx  Limit the number of threads used by the region mover. Default value is 1."
   echo " movetimeout xx Timeout for moving regions. If regions are not moved by the timeout value,\
 exit with error. Default value is INT_MAX."
-  echo " hostname       Hostname of server we are to stop"
+  echo " hostname       Hostname to stop; match what HBase uses; pass 'localhost' if local to avoid ssh"
   echo " e|failfast     Set -e so exit immediately if any command exits with non-zero status"
-  echo " nob| nobalancer Do not manage balancer states. This is only used as optimization in \
+  echo " nob|nobalancer Do not manage balancer states. This is only used as optimization in \
 rolling_restart.sh to avoid multiple calls to hbase shell"
+  echo " d|designatedfile xx Designated file with <hostname:port> per line as unload targets"
+  echo " x|excludefile xx Exclude file should have <hostname:port> per line. We do not unload \
+regions to hostnames given in exclude file"
   exit 1
 }
 
@@ -57,6 +61,8 @@ movetimeout=2147483647
 maxthreads=1
 failfast=
 nob=false
+designatedfile=
+excludefile=
 while [ $# -gt 0 ]
 do
   case "$1" in
@@ -69,6 +75,8 @@ do
     --maxthreads) shift; maxthreads=$1; shift;;
     --movetimeout) shift; movetimeout=$1; shift;;
     --nobalancer | -nob) nob=true; shift;;
+    --designatedfile | -d) shift; designatedfile=$1; shift;;
+    --excludefile | -x) shift; excludefile=$1; shift;;
     --) shift; break;;
     -*) usage ;;
     *)  break;;	# terminate while loop
@@ -95,10 +103,11 @@ hostname=$1
 filename="/tmp/$hostname"
 
 local=
-localhostname=`/bin/hostname`
+localhostname=`/bin/hostname -f`
 
-if [ "$localhostname" == "$hostname" ]; then
+if [ "$localhostname" == "$hostname" ] || [ "$hostname" == "localhost" ]; then
   local=true
+  hostname=$localhostname
 fi
 
 if [ "$nob" == "true"  ]; then
@@ -110,10 +119,20 @@ else
   log "Previous balancer state was $HBASE_BALANCER_STATE"
 fi
 
+unload_args="--filename $filename --maxthreads $maxthreads $noack --operation unload \
+--timeout $movetimeout --regionserverhost $hostname"
+
+if [ "$designatedfile" != "" ]; then
+  unload_args="$unload_args --designatedfile $designatedfile"
+fi
+
+if [ "$excludefile" != "" ]; then
+  unload_args="$unload_args --excludefile $excludefile"
+fi
+
 log "Unloading $hostname region(s)"
 HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.apache.hadoop.hbase.util.RegionMover \
---filename $filename --maxthreads $maxthreads $noack --operation "unload" --timeout $movetimeout \
---regionserverhost $hostname
+$unload_args
 log "Unloaded $hostname region(s)"
 
 # Stop the server(s). Have to put hostname into its own little file for hbase-daemons.sh
@@ -183,4 +202,4 @@ else
 fi
 
 # Cleanup tmp files.
-trap "rm -f  "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT
+trap "rm -f  /tmp/$(basename $0).*.tmp &> /dev/null" EXIT