YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well. (Sunil G via wangda)

(cherry picked from commit 07b0fb9)

Author: wangdatan

hadoop-yarn-project/CHANGES.txt

@@ -564,6 +564,9 @@ Release 2.8.0 - UNRELEASED
     YARN-4309. Add container launch related debug information to container logs
     when a container fails. (Varun Vasudev via wangda)
 
+    YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well.
+    (Sunil G via wangda)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java

@@ -61,7 +61,7 @@ private enum ResourceType {
     //CACHED_USED and CACHED_PENDING may be read by anyone, but must only
     //be written by ordering policies
     USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
-      CACHED_PENDING(5);
+      CACHED_PENDING(5), AMLIMIT(6);
 
     private int idx;
 
@@ -92,6 +92,7 @@ public String toString() {
       sb.append("pending=" + resArr[1] + "%, ");
       sb.append("am_used=" + resArr[2] + "%, ");
       sb.append("reserved=" + resArr[3] + "%}");
+      sb.append("am_limit=" + resArr[6] + "%, ");
       return sb.toString();
     }
   }
@@ -106,11 +107,19 @@ public Resource getUsed() {
   public Resource getUsed(String label) {
     return _get(label, ResourceType.USED);
   }
-  
+
+  public Resource getCachedUsed() {
+    return _get(NL, ResourceType.CACHED_USED);
+  }
+
   public Resource getCachedUsed(String label) {
     return _get(label, ResourceType.CACHED_USED);
   }
-  
+
+  public Resource getCachedPending() {
+    return _get(NL, ResourceType.CACHED_PENDING);
+  }
+
   public Resource getCachedPending(String label) {
     return _get(label, ResourceType.CACHED_PENDING);
   }
@@ -149,15 +158,23 @@ public void copyAllUsed(ResourceUsage other) {
   public void setUsed(String label, Resource res) {
     _set(label, ResourceType.USED, res);
   }
-  
+
   public void setCachedUsed(String label, Resource res) {
     _set(label, ResourceType.CACHED_USED, res);
   }
-  
+
+  public void setCachedUsed(Resource res) {
+    _set(NL, ResourceType.CACHED_USED, res);
+  }
+
   public void setCachedPending(String label, Resource res) {
     _set(label, ResourceType.CACHED_PENDING, res);
   }
 
+  public void setCachedPending(Resource res) {
+    _set(NL, ResourceType.CACHED_PENDING, res);
+  }
+
   /*
    * Pending
    */
@@ -263,6 +280,41 @@ public void setAMUsed(String label, Resource res) {
     _set(label, ResourceType.AMUSED, res);
   }
 
+  /*
+   * AM-Resource Limit
+   */
+  public Resource getAMLimit() {
+    return getAMLimit(NL);
+  }
+
+  public Resource getAMLimit(String label) {
+    return _get(label, ResourceType.AMLIMIT);
+  }
+
+  public void incAMLimit(String label, Resource res) {
+    _inc(label, ResourceType.AMLIMIT, res);
+  }
+
+  public void incAMLimit(Resource res) {
+    incAMLimit(NL, res);
+  }
+
+  public void decAMLimit(Resource res) {
+    decAMLimit(NL, res);
+  }
+
+  public void decAMLimit(String label, Resource res) {
+    _dec(label, ResourceType.AMLIMIT, res);
+  }
+
+  public void setAMLimit(Resource res) {
+    setAMLimit(NL, res);
+  }
+
+  public void setAMLimit(String label, Resource res) {
+    _set(label, ResourceType.AMLIMIT, res);
+  }
+
   private static Resource normalize(Resource res) {
     if (res == null) {
       return Resources.none();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -606,6 +606,7 @@ public synchronized Resource getAMResourceLimitPerPartition(
         minimumAllocation);
 
     metrics.setAMResouceLimit(amResouceLimit);
+    queueUsage.setAMLimit(nodePartition, amResouceLimit);
     return amResouceLimit;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java

@@ -217,6 +217,11 @@ public void setAbsoluteMaximumCapacity(String label, float value) {
   }
 
   /* Absolute Maximum AM resource percentage Getter and Setter */
+
+  public float getMaxAMResourcePercentage() {
+    return _get(NL, CapacityType.MAX_AM_PERC);
+  }
+
   public float getMaxAMResourcePercentage(String label) {
     return _get(label, CapacityType.MAX_AM_PERC);
   }
@@ -225,6 +230,10 @@ public void setMaxAMResourcePercentage(String label, float value) {
     _set(label, CapacityType.MAX_AM_PERC, value);
   }
 
+  public void setMaxAMResourcePercentage(float value) {
+    _set(NL, CapacityType.MAX_AM_PERC, value);
+  }
+
   /**
    * Clear configurable fields, like
    * (absolute)capacity/(absolute)maximum-capacity, this will be used by queue

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java

@@ -37,8 +37,8 @@ public class TestResourceUsage {
 
   @Parameterized.Parameters
   public static Collection<String[]> getParameters() {
-    return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
-        { "Reserved" }, { "AMUsed" } });
+    return Arrays.asList(new String[][]{{"Pending"}, {"Used"}, {"Reserved"},
+        {"AMUsed"}, {"AMLimit"}, {"CachedUsed"}, {"CachedPending"}});
   }
 
   public TestResourceUsage(String suffix) {
@@ -112,16 +112,26 @@ private void internalTestModifyAndRead(String label) throws Exception {
     check(0, 0, res);
 
     // Add 1,1 should returns 1,1
-    inc(usage, suffix, Resource.newInstance(1, 1), label);
-    check(1, 1, get(usage, suffix, label));
+    try {
+      inc(usage, suffix, Resource.newInstance(1, 1), label);
+      check(1, 1, get(usage, suffix, label));
+    } catch (NoSuchMethodException e) {
+      // Few operations need not have to be verified as some resources doesn't
+      // inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
+    }
 
     // Set 2,2
     set(usage, suffix, Resource.newInstance(2, 2), label);
     check(2, 2, get(usage, suffix, label));
 
     // dec 2,2
-    dec(usage, suffix, Resource.newInstance(2, 2), label);
-    check(0, 0, get(usage, suffix, label));
+    try {
+      dec(usage, suffix, Resource.newInstance(2, 2), label);
+      check(0, 0, get(usage, suffix, label));
+    } catch (NoSuchMethodException e) {
+      // Few operations need not have to be verified, as some resources doesn't
+      // inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
+    }
   }
 
   void check(int mem, int cpu, Resource res) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java

@@ -43,7 +43,8 @@ public static Collection<String[]> getParameters() {
         { "UsedCapacity" }, 
         { "AbsoluteUsedCapacity" },
         { "MaximumCapacity" }, 
-        { "AbsoluteMaximumCapacity" } });
+        { "AbsoluteMaximumCapacity" },
+        { "MaxAMResourcePercentage" } });
   }
 
   public TestQueueCapacities(String suffix) {