YARN-10596. Allow static definition of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed by Andras Gyori

Author: szilard-nemeth

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java

@@ -238,14 +238,21 @@ static CSQueue parseQueue(
     boolean isReservableQueue = conf.isReservable(fullQueueName);
     boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(
         fullQueueName);
+    // if a queue is eligible for auto queue creation v2
+    // it must be a ParentQueue (even if it is empty)
+    boolean isAutoQueueCreationV2Enabled = conf.isAutoQueueCreationV2Enabled(
+        fullQueueName);
     boolean isDynamicParent = false;
 
+    // Auto created parent queues might not have static children, but they
+    // must be kept as a ParentQueue
     CSQueue oldQueue = oldQueues.get(fullQueueName);
     if (oldQueue instanceof ParentQueue) {
       isDynamicParent = ((ParentQueue) oldQueue).isDynamicQueue();
     }
 
-    if (childQueueNames.size() == 0 && !isDynamicParent) {
+    if (childQueueNames.size() == 0 && !isDynamicParent &&
+        !isAutoQueueCreationV2Enabled) {
       if (null == parent) {
         throw new IllegalStateException(
             "Queue configuration missing child queue names for " + queueName);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java

@@ -422,6 +422,46 @@ public void testAutoQueueCreationOnAppSubmission() throws Exception {
     Assert.assertTrue(user0.isDynamicQueue());
   }
 
+  @Test
+  public void testChildlessParentQueueWhenAutoQueueCreationEnabled()
+      throws Exception {
+    startScheduler();
+    csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
+    csConf.setNonLabeledQueueWeight("root", 1f);
+    csConf.setNonLabeledQueueWeight("root.a", 1f);
+    csConf.setNonLabeledQueueWeight("root.b", 1f);
+    csConf.setQueues("root.a", new String[]{"a1"});
+    csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
+    csConf.setAutoQueueCreationV2Enabled("root", true);
+    csConf.setAutoQueueCreationV2Enabled("root.a", true);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    CSQueue empty = cs.getQueue("root.empty-auto-parent");
+    Assert.assertTrue("empty-auto-parent is not a LeafQueue",
+        empty instanceof LeafQueue);
+    empty.stopQueue();
+
+    csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
+    csConf.setNonLabeledQueueWeight("root", 1f);
+    csConf.setNonLabeledQueueWeight("root.a", 1f);
+    csConf.setNonLabeledQueueWeight("root.b", 1f);
+    csConf.setQueues("root.a", new String[]{"a1"});
+    csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
+    csConf.setAutoQueueCreationV2Enabled("root", true);
+    csConf.setAutoQueueCreationV2Enabled("root.a", true);
+    csConf.setAutoQueueCreationV2Enabled("root.empty-auto-parent", true);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    empty = cs.getQueue("root.empty-auto-parent");
+    Assert.assertTrue("empty-auto-parent is not a ParentQueue",
+        empty instanceof ParentQueue);
+    Assert.assertEquals("empty-auto-parent has children",
+        0, empty.getChildQueues().size());
+    Assert.assertTrue("empty-auto-parent is not eligible " +
+            "for auto queue creation",
+        ((ParentQueue)empty).isEligibleForAutoQueueCreation());
+  }
+
   private LeafQueue createQueue(String queuePath) throws YarnException {
     return autoQueueHandler.autoCreateQueue(
         CSQueueUtils.extractQueuePath(queuePath));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java

@@ -507,8 +507,6 @@ private static Configuration createWeightConfigInternal(boolean enableAqc) {
       if (enableAqc) {
         conf.put("yarn.scheduler.capacity.root.auto-queue-creation-v2.enabled",
             "true");
-        conf.put("yarn.scheduler.capacity.root.default." +
-                "auto-queue-creation-v2.enabled", "true");
       }
       return createConfiguration(conf);
     }