HBASE-19577 Use log4j2 instead of log4j for logging

bin/hbase

@@ -301,10 +301,13 @@ else
   # make it easier to check for shaded/not later on.
   shaded_jar=""
 fi
+# here we will add slf4j-api, commons-logging, jul-to-slf4j, jcl-over-slf4j
+# to classpath, as they are all logging bridges. Only exclude log4j* so we
+# will not actually log anything out. Add it later if necessary
 for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do
   if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \
-     [ "${f}" != "htrace-core.jar$" ] && \
-     [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then
+     [[ "${f}" != "htrace-core.jar$" ]] && \
+     [[ ! "${f}" =~ ^.*/log4j.*$ ]]; then
     CLASSPATH="${CLASSPATH}:${f}"
   fi
 done
@@ -658,7 +661,7 @@ elif [ "$COMMAND" = "mapredcp" ] ; then
     for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do
       if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \
          [ "${f}" != "htrace-core.jar$" ] && \
-         [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then
+         [[ ! "${f}" =~ ^.*/log4j.*$ ]]; then
         echo -n ":${f}"
       fi
     done
@@ -775,25 +778,31 @@ HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR"
 HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE"
 HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME"
 HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING"
-HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}"
+# log4j2 does not support setting log level and appender at once, so we need to split HBASE_ROOT_LOGGER
+HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-INFO,console}
+array=(${HBASE_ROOT_LOGGER//,/ })
+HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger.level=${array[0]}"
+HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger.appender=${array[1]}"
 if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
   export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH"
 fi
 
 # Enable security logging on the master and regionserver only
 if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then
-  HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,RFAS}"
+  HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-INFO,RFAS}
 else
-  HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}"
+  HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}
 fi
+array=(${HBASE_SECURITY_LOGGER//,/ })
+HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger.level=${array[0]}"
+HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger.appender=${array[1]}"
 
 HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX"
 # by now if we're running a command it means we need logging
-for f in ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j*.jar; do
+for f in ${HBASE_HOME}/lib/client-facing-thirdparty/log4j*.jar; do
   if [ -f "${f}" ]; then
     CLASSPATH="${CLASSPATH}:${f}"
-    break
   fi
 done
 

bin/hbase-daemon.sh

@@ -155,12 +155,20 @@ JAVA=$JAVA_HOME/bin/java
 export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME
 export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log
 
-if [ -z "${HBASE_ROOT_LOGGER}" ]; then
-export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"}
+if [ -z "${HBASE_ROOT_LOGGER_LEVEL}" ]; then
+export HBASE_ROOT_LOGGER_LEVEL=${HBASE_ROOT_LOGGER_LEVEL:-"INFO"}
 fi
 
-if [ -z "${HBASE_SECURITY_LOGGER}" ]; then
-export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"}
+if [ -z "${HBASE_ROOT_LOGGER_APPENDER}" ]; then
+export HBASE_ROOT_LOGGER_APPENDER=${HBASE_ROOT_LOGGER_APPENDER:-"RFA"}
+fi
+
+if [ -z "${HBASE_SECURITY_LOGGER_LEVEL}" ]; then
+export HBASE_SECURITY_LOGGER_LEVEL=${HBASE_SECURITY_LOGGER_LEVEL:-"INFO"}
+fi
+
+if [ -z "${HBASE_SECURITY_LOGGER_APPENDER}" ]; then
+export HBASE_SECURITY_LOGGER_APPENDER=${HBASE_SECURITY_LOGGER_APPENDER:-"RFAS"}
 fi
 
 HBASE_LOGOUT=${HBASE_LOGOUT:-"$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out"}

bin/hbase.cmd

@@ -329,7 +329,13 @@ set HBASE_OPTS=%HBASE_OPTS% -XX:OnOutOfMemoryError="taskkill /F /PID %p"
 if not defined HBASE_ROOT_LOGGER (
   set HBASE_ROOT_LOGGER=INFO,console
 )
-set HBASE_OPTS=%HBASE_OPTS% -Dhbase.root.logger="%HBASE_ROOT_LOGGER%"
+
+for /F "tokens=1,2 delims=," %%a in ("%HBASE_ROOT_LOGGER%") do (
+  set HBASE_ROOT_LOGGER_LEVEL=%%a
+  set HBASE_ROOT_LOGGER_APPENDER=%%b
+)
+
+set HBASE_OPTS=%HBASE_OPTS% -Dhbase.root.logger.level="%HBASE_ROOT_LOGGER_LEVEL% -Dhbase.root.logger.appender="%HBASE_ROOT_LOGGER_APPENDER% "
 
 if defined JAVA_LIBRARY_PATH (
   set HBASE_OPTS=%HBASE_OPTS% -Djava.library.path="%JAVA_LIBRARY_PATH%"
@@ -345,7 +351,13 @@ if not defined HBASE_SECURITY_LOGGER (
     set HBASE_SECURITY_LOGGER=INFO,DRFAS
   )
 )
-set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger="%HBASE_SECURITY_LOGGER%"
+
+for /F "tokens=1,2 delims=," %%a in ("%HBASE_SECURITY_LOGGER%") do (
+  set HBASE_SECURITY_LOGGER_LEVEL=%%a
+  set HBASE_SECURITY_LOGGER_APPENDER=%%b
+)
+
+set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger.level="%HBASE_SECURITY_LOGGER_LEVEL% -Dhbase.security.logger.appender="%HBASE_SECURITY_LOGGER_APPENDER%"
 
 set HEAP_SETTINGS=%JAVA_HEAP_MAX% %JAVA_OFFHEAP_MAX%
 set java_arguments=%HEAP_SETTINGS% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments%

hbase-logging/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java → conf/log4j2-hbtop.xml

@@ -1,3 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -15,32 +17,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hbase;
-
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Logger class that buffers before trying to log to the specified console.
- */
-@InterfaceAudience.Private
-public class AsyncConsoleAppender extends org.apache.log4j.AsyncAppender {
-  private final org.apache.log4j.ConsoleAppender consoleAppender;
-
-  public AsyncConsoleAppender() {
-    super();
-    consoleAppender = new org.apache.log4j.ConsoleAppender(
-      new org.apache.log4j.PatternLayout("%d{ISO8601} %-5p [%t] %c{2}: %m%n"));
-    this.addAppender(consoleAppender);
-  }
-
-  public void setTarget(String value) {
-    consoleAppender.setTarget(value);
-  }
-
-  @Override
-  public void activateOptions() {
-    consoleAppender.activateOptions();
-    super.activateOptions();
-  }
-
-}
+-->
+<Configuration>
+  <Appenders>
+    <!-- Console appender -->
+    <Console name="console" target="SYSTEM_ERR">
+      <PatternLayout pattern="%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n" />
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="warn">
+      <AppenderRef ref="console" />
+    </Root>
+    <!-- ZooKeeper will still put stuff at WARN -->
+    <Logger name="org.apache.zookeeper" level="error" />
+  </Loggers>
+</Configuration>