Merge branch 'trunk' into HADOOP-16085-squashed

Author: Ben Roling

LICENSE.txt

@@ -1636,7 +1636,6 @@ derived.)
 
 The binary distribution of this product bundles these dependencies under the
 following license:
-JUnit 4.11
 Eclipse JDT Core 3.1.1
 --------------------------------------------------------------------------------
 (EPL v1.0)

dev-support/docker/Dockerfile

@@ -207,23 +207,6 @@ ENV MAVEN_OPTS -Xms256m -Xmx1536m
 # YETUS CUT HERE
 ###
 
-####
-# Install svn & Forrest (for Apache Hadoop website)
-###
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends subversion \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /opt/apache-forrest \
-    && curl -L -s -S \
-      https://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz \
-      -o /opt/forrest.tar.gz \
-    && tar xzf /opt/forrest.tar.gz --strip-components 1 -C /opt/apache-forrest
-RUN echo 'forrest.home=/opt/apache-forrest' > build.properties
-ENV FORREST_HOME=/opt/apache-forrest
-
 # Hugo static website generator (for new hadoop site and Ozone docs)
 RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.30.2/hugo_0.30.2_Linux-64bit.deb \
     && dpkg --install hugo.deb \

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java

@@ -24,7 +24,6 @@
 import java.io.UnsupportedEncodingException;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
@@ -44,6 +43,7 @@
 import org.ietf.jgss.Oid;
 
 import javax.security.auth.Subject;
+import javax.security.auth.kerberos.KerberosPrincipal;
 import javax.security.auth.kerberos.KerberosTicket;
 import javax.security.auth.kerberos.KeyTab;
 
@@ -90,36 +90,37 @@ public static Oid getOidInstance(String oidName)
     return (Oid)oidField.get(oidClass);
   }
 
-  public static String getDefaultRealm() 
-      throws ClassNotFoundException, NoSuchMethodException, 
-      IllegalArgumentException, IllegalAccessException, 
+  /**
+   * Return the default realm for this JVM.
+   *
+   * @return The default realm
+   * @throws IllegalArgumentException If the default realm does not exist.
+   * @throws ClassNotFoundException Not thrown. Exists for compatibility.
+   * @throws NoSuchMethodException Not thrown. Exists for compatibility.
+   * @throws IllegalAccessException Not thrown. Exists for compatibility.
+   * @throws InvocationTargetException Not thrown. Exists for compatibility.
+   */
+  public static String getDefaultRealm()
+      throws ClassNotFoundException, NoSuchMethodException,
+      IllegalArgumentException, IllegalAccessException,
       InvocationTargetException {
-    Object kerbConf;
-    Class<?> classRef;
-    Method getInstanceMethod;
-    Method getDefaultRealmMethod;
-    if (IBM_JAVA) {
-      classRef = Class.forName("com.ibm.security.krb5.internal.Config");
-    } else {
-      classRef = Class.forName("sun.security.krb5.Config");
-    }
-    getInstanceMethod = classRef.getMethod("getInstance", new Class[0]);
-    kerbConf = getInstanceMethod.invoke(classRef, new Object[0]);
-    getDefaultRealmMethod = classRef.getDeclaredMethod("getDefaultRealm",
-         new Class[0]);
-    return (String)getDefaultRealmMethod.invoke(kerbConf, new Object[0]);
+    // Any name is okay.
+    return new KerberosPrincipal("tmp", 1).getRealm();
   }
 
+  /**
+   * Return the default realm for this JVM.
+   * If the default realm does not exist, this method returns null.
+   *
+   * @return The default realm
+   */
   public static String getDefaultRealmProtected() {
-    String realmString = null;
     try {
-      realmString = getDefaultRealm();
-    } catch (RuntimeException rte) {
-      //silently catch everything
+      return getDefaultRealm();
     } catch (Exception e) {
-      //silently return null
+      //silently catch everything
+      return null;
     }
-    return realmString;
   }
 
   /*

hadoop-common-project/hadoop-common/pom.xml

@@ -308,7 +308,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
+      <artifactId>kerb-core</artifactId>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>

hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh

@@ -585,7 +585,7 @@ function hadoop_bootstrap
 
   #
   # short-cuts. vendors may redefine these as well, preferably
-  # in hadoop-layouts.sh
+  # in hadoop-layout.sh
   #
   HADOOP_COMMON_DIR=${HADOOP_COMMON_DIR:-"share/hadoop/common"}
   HADOOP_COMMON_LIB_JARS_DIR=${HADOOP_COMMON_LIB_JARS_DIR:-"share/hadoop/common/lib"}
@@ -2530,7 +2530,7 @@ function hadoop_do_classpath_subcommand
   fi
 }
 
-## @description  generic shell script opton parser.  sets
+## @description  generic shell script option parser.  sets
 ## @description  HADOOP_PARSE_COUNTER to set number the
 ## @description  caller should shift
 ## @audience     private

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java

@@ -139,7 +139,7 @@ protected int init(String[] args) throws IOException {
         }
         if (attributes.containsKey(attr)) {
           getOut().println("\nEach attribute must correspond to only one " +
-              "value:\natttribute \"" + attr + "\" was repeated\n");
+              "value:\nattribute \"" + attr + "\" was repeated\n");
           return 1;
         }
         attributes.put(attr, val);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java

@@ -32,8 +32,7 @@
  *
  * This class is immutable.
  */
-@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "Hive", "MapReduce",
-    "Pig", "Spark", "YARN"})
+@InterfaceAudience.Public
 @InterfaceStability.Evolving
 public final class CallerContext {
   public static final Charset SIGNATURE_ENCODING = StandardCharsets.UTF_8;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java

@@ -833,7 +833,7 @@ private static <T extends IOException> T wrapWithMessage(
       Throwable t = ctor.newInstance(msg);
       return (T)(t.initCause(exception));
     } catch (Throwable e) {
-      LOG.warn("Unable to wrap exception of type {}: it has no (String) "
+      LOG.trace("Unable to wrap exception of type {}: it has no (String) "
           + "constructor", clazz, e);
       throw exception;
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java

@@ -97,10 +97,10 @@ public AccessControlList(String users, String groups) {
   }
 
   /**
-   * Build ACL from the given two Strings.
-   * The Strings contain comma separated values.
+   * Build ACL from the given array of strings.
+   * The strings contain comma separated values.
    *
-   * @param aclString build ACL from array of Strings
+   * @param userGroupStrings build ACL from array of Strings
    */
   private void buildACL(String[] userGroupStrings) {
     users = new HashSet<String>();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java

@@ -167,7 +167,7 @@ protected SecretKey generateSecret() {
    * @param key the secret key
    * @return the bytes of the generated password
    */
-  protected static byte[] createPassword(byte[] identifier, 
+  public static byte[] createPassword(byte[] identifier,
                                          SecretKey key) {
     Mac mac = threadLocalMac.get();
     try {