Merge branch 'apache:trunk' into HADOOP-18427

Author: slfan1989

hadoop-client-modules/hadoop-client-api/pom.xml

@@ -98,13 +98,6 @@
               <createSourcesJar>true</createSourcesJar>
               <shadeSourcesContent>true</shadeSourcesContent>
             </configuration>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-maven-plugins</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -254,8 +247,7 @@
                     </relocation>
                   </relocations>
                   <transformers>
-                    <!-- Needed until MSHADE-182 -->
-                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resource>NOTICE.txt</resource>

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -671,13 +671,6 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-maven-plugins</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -1052,8 +1045,7 @@
                     </relocation>
                   </relocations>
                   <transformers>
-                    <!-- Needed until MSHADE-182 -->
-                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resources>

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -128,13 +128,6 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-maven-plugins</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -397,8 +390,7 @@
     -->
                   </relocations>
                   <transformers>
-                    <!-- Needed until MSHADE-182 -->
-                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resources>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java

@@ -27,41 +27,37 @@
 public class SampleStat {
   private final MinMax minmax = new MinMax();
   private long numSamples = 0;
-  private double a0, a1, s0, s1, total;
+  private double mean, s;
 
   /**
    * Construct a new running sample stat
    */
   public SampleStat() {
-    a0 = s0 = 0.0;
-    total = 0.0;
+    mean = 0.0;
+    s = 0.0;
   }
 
   public void reset() {
     numSamples = 0;
-    a0 = s0 = 0.0;
-    total = 0.0;
+    mean = 0.0;
+    s = 0.0;
     minmax.reset();
   }
 
   // We want to reuse the object, sometimes.
-  void reset(long numSamples, double a0, double a1, double s0, double s1,
-      double total, MinMax minmax) {
-    this.numSamples = numSamples;
-    this.a0 = a0;
-    this.a1 = a1;
-    this.s0 = s0;
-    this.s1 = s1;
-    this.total = total;
-    this.minmax.reset(minmax);
+  void reset(long numSamples1, double mean1, double s1, MinMax minmax1) {
+    numSamples = numSamples1;
+    mean = mean1;
+    s = s1;
+    minmax.reset(minmax1);
   }
 
   /**
    * Copy the values to other (saves object creation and gc.)
    * @param other the destination to hold our values
    */
   public void copyTo(SampleStat other) {
-    other.reset(numSamples, a0, a1, s0, s1, total, minmax);
+    other.reset(numSamples, mean, s, minmax);
   }
 
   /**
@@ -78,24 +74,22 @@ public SampleStat add(double x) {
    * Add some sample and a partial sum to the running stat.
    * Note, min/max is not evaluated using this method.
    * @param nSamples  number of samples
-   * @param x the partial sum
+   * @param xTotal the partial sum
    * @return  self
    */
-  public SampleStat add(long nSamples, double x) {
+  public SampleStat add(long nSamples, double xTotal) {
     numSamples += nSamples;
-    total += x;
 
-    if (numSamples == 1) {
-      a0 = a1 = x;
-      s0 = 0.0;
-    }
-    else {
-      // The Welford method for numerical stability
-      a1 = a0 + (x - a0) / numSamples;
-      s1 = s0 + (x - a0) * (x - a1);
-      a0 = a1;
-      s0 = s1;
-    }
+    // use the weighted incremental version of Welford's algorithm to get
+    // numerical stability while treating the samples as being weighted
+    // by nSamples
+    // see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+
+    double x = xTotal / nSamples;
+    double meanOld = mean;
+
+    mean += ((double) nSamples / numSamples) * (x - meanOld);
+    s += nSamples * (x - meanOld) * (x - mean);
     return this;
   }
 
@@ -110,21 +104,21 @@ public long numSamples() {
    * @return the total of all samples added
    */
   public double total() {
-    return total;
+    return mean * numSamples;
   }
 
   /**
    * @return  the arithmetic mean of the samples
    */
   public double mean() {
-    return numSamples > 0 ? (total / numSamples) : 0.0;
+    return numSamples > 0 ? mean : 0.0;
   }
 
   /**
    * @return  the variance of the samples
    */
   public double variance() {
-    return numSamples > 1 ? s1 / (numSamples - 1) : 0.0;
+    return numSamples > 1 ? s / (numSamples - 1) : 0.0;
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java

@@ -30,6 +30,7 @@
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Hashtable;
 import java.util.Iterator;
@@ -252,6 +253,10 @@ public class LdapGroupsMapping
   public static final String POSIX_GID_ATTR_KEY = LDAP_CONFIG_PREFIX + ".posix.attr.gid.name";
   public static final String POSIX_GID_ATTR_DEFAULT = "gidNumber";
 
+  public static final String GROUP_SEARCH_FILTER_PATTERN =
+      LDAP_CONFIG_PREFIX + ".group.search.filter.pattern";
+  public static final String GROUP_SEARCH_FILTER_PATTERN_DEFAULT = "";
+
   /*
    * Posix attributes
    */
@@ -337,6 +342,7 @@ public class LdapGroupsMapping
   private int numAttempts;
   private volatile int numAttemptsBeforeFailover;
   private volatile String ldapCtxFactoryClassName;
+  private volatile String[] groupSearchFilterParams;
 
   /**
    * Returns list of groups for a user.
@@ -437,8 +443,14 @@ Set<String> lookupGroup(SearchResult result, DirContext c,
     Set<String> groupDNs = new HashSet<>();
 
     NamingEnumeration<SearchResult> groupResults;
-    // perform the second LDAP query
-    if (isPosix) {
+
+    String[] resolved = resolveCustomGroupFilterArgs(result);
+    // If custom group filter argument is supplied, use that!!!
+    if (resolved != null) {
+      groupResults =
+          c.search(groupbaseDN, groupSearchFilter, resolved, SEARCH_CONTROLS);
+    } else if (isPosix) {
+      // perform the second LDAP query
       groupResults = lookupPosixGroup(result, c);
     } else {
       String userDn = result.getNameInNamespace();
@@ -462,6 +474,25 @@ Set<String> lookupGroup(SearchResult result, DirContext c,
     return groups;
   }
 
+  private String[] resolveCustomGroupFilterArgs(SearchResult result)
+      throws NamingException {
+    if (groupSearchFilterParams != null) {
+      String[] filterElems = new String[groupSearchFilterParams.length];
+      for (int i = 0; i < groupSearchFilterParams.length; i++) {
+        // Specific handling for userDN.
+        if (groupSearchFilterParams[i].equalsIgnoreCase("userDN")) {
+          filterElems[i] = result.getNameInNamespace();
+        } else {
+          filterElems[i] =
+              result.getAttributes().get(groupSearchFilterParams[i]).get()
+                  .toString();
+        }
+      }
+      return filterElems;
+    }
+    return null;
+  }
+
   /**
    * Perform LDAP queries to get group names of a user.
    *
@@ -781,6 +812,12 @@ public synchronized void setConf(Configuration conf) {
         conf.get(POSIX_UID_ATTR_KEY, POSIX_UID_ATTR_DEFAULT);
     posixGidAttr =
         conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT);
+    String groupSearchFilterParamCSV = conf.get(GROUP_SEARCH_FILTER_PATTERN,
+        GROUP_SEARCH_FILTER_PATTERN_DEFAULT);
+    if(groupSearchFilterParamCSV!=null && !groupSearchFilterParamCSV.isEmpty()) {
+      LOG.debug("Using custom group search filters: {}", groupSearchFilterParamCSV);
+      groupSearchFilterParams = groupSearchFilterParamCSV.split(",");
+    }
 
     int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT,
         DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
@@ -795,7 +832,16 @@ public synchronized void setConf(Configuration conf) {
       returningAttributes = new String[] {
           groupNameAttr, posixUidAttr, posixGidAttr};
     }
-    SEARCH_CONTROLS.setReturningAttributes(returningAttributes);
+
+    // If custom group filter is being used, fetch attributes in the filter
+    // as well.
+    ArrayList<String> customAttributes = new ArrayList<>();
+    if (groupSearchFilterParams != null) {
+      customAttributes.addAll(Arrays.asList(groupSearchFilterParams));
+    }
+    customAttributes.addAll(Arrays.asList(returningAttributes));
+    SEARCH_CONTROLS
+        .setReturningAttributes(customAttributes.toArray(new String[0]));
 
     // LDAP_CTX_FACTORY_CLASS_DEFAULT is not open to unnamed modules
     // in Java 11+, so the default value is set to null to avoid

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -585,6 +585,18 @@
   </description>
 </property>
 
+<property>
+  <name>hadoop.security.group.mapping.ldap.group.search.filter.pattern</name>
+  <value></value>
+  <description>
+    Comma separated values that needs to be substituted in the group search
+    filter during group lookup. The values are substituted in the order they
+    appear in the list, the first value will replace {0} the second {1} and
+    so on.
+  </description>
+</property>
+
+
 <property>
   <name>hadoop.security.group.mapping.providers</name>
   <value></value>

hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md

@@ -85,6 +85,14 @@ This is the limit for each ldap query.  If `hadoop.security.group.mapping.ldap.s
 `hadoop.security.group.mapping.ldap.base` configures how far to walk up the groups hierarchy when resolving groups.
 By default, with a limit of 0, in order to be considered a member of a group, the user must be an explicit member in LDAP.  Otherwise, it will traverse the group hierarchy `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` levels up.
 
+It is possible to have custom group search filters with different arguments using
+the configuration `hadoop.security.group.mapping.ldap.group.search.filter.pattern`, we can configure comma separated values here and the values configured will be fetched from the LDAP attributes and will be replaced in the group
+search filter in the order they appear here, say if the first entry here is uid, so uid will be fetched from the attributes and the value fetched
+will be used in place of {0} in the group search filter, similarly the second value configured will replace {1} and so on.
+
+Note: If `hadoop.security.group.mapping.ldap.group.search.filter.pattern` is configured, the group search will always be done assuming this group
+search filter pattern irrespective of any other parameters.
+
 ### Bind user(s) ###
 If the LDAP server does not support anonymous binds,
 set the distinguished name of the user to bind in `hadoop.security.group.mapping.ldap.bind.user`.