YARN-2635. TestRM, TestRMRestart, TestClientToAMTokens should run with both CS and FS. (Wei Yan and kasha via kasha)

Author: kambatla

hadoop-yarn-project/CHANGES.txt

@@ -41,6 +41,9 @@ Release 2.7.0 - UNRELEASED
     YARN-1979. TestDirectoryCollection fails when the umask is unusual.
     (Vinod Kumar Vavilapalli and Tsuyoshi OZAWA via junping_du)
 
+    YARN-2635. TestRM, TestRMRestart, TestClientToAMTokens should run 
+    with both CS and FS. (Wei Yan and kasha via kasha)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.Collection;
+
+@RunWith(Parameterized.class)
+public abstract class ParameterizedSchedulerTestBase {
+  protected final static String TEST_DIR =
+      new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath();
+  private final static String FS_ALLOC_FILE =
+      new File(TEST_DIR, "test-fs-queues.xml").getAbsolutePath();
+
+  private SchedulerType schedulerType;
+  private YarnConfiguration conf = null;
+
+  public enum SchedulerType {
+    CAPACITY, FAIR
+  }
+
+  public ParameterizedSchedulerTestBase(SchedulerType type) {
+    schedulerType = type;
+  }
+
+  public YarnConfiguration getConf() {
+    return conf;
+  }
+
+  @Parameterized.Parameters
+  public static Collection<SchedulerType[]> getParameters() {
+    return Arrays.asList(new SchedulerType[][]{
+        {SchedulerType.CAPACITY}, {SchedulerType.FAIR}});
+  }
+
+  @Before
+  public void configureScheduler() throws IOException {
+    conf = new YarnConfiguration();
+    switch (schedulerType) {
+      case CAPACITY:
+        conf.set(YarnConfiguration.RM_SCHEDULER,
+            CapacityScheduler.class.getName());
+        break;
+      case FAIR:
+        configureFairScheduler(conf);
+        break;
+    }
+  }
+
+  private void configureFairScheduler(YarnConfiguration conf) throws IOException {
+    // Disable queueMaxAMShare limitation for fair scheduler
+    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
+    out.println("</allocations>");
+    out.close();
+
+    conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager;
 
+import org.junit.Before;
 import static org.mockito.Matchers.argThat;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.spy;
@@ -65,7 +66,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
 import org.apache.log4j.Level;
@@ -75,13 +75,23 @@
 import org.mockito.ArgumentMatcher;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
-public class TestRM {
-
+public class TestRM extends ParameterizedSchedulerTestBase {
   private static final Log LOG = LogFactory.getLog(TestRM.class);
 
   // Milliseconds to sleep for when waiting for something to happen
   private final static int WAIT_SLEEP_MS = 100;
 
+  private YarnConfiguration conf;
+
+  public TestRM(SchedulerType type) {
+    super(type);
+  }
+
+  @Before
+  public void setup() {
+    conf = getConf();
+  }
+
   @After
   public void tearDown() {
     ClusterMetrics.destroy();
@@ -93,7 +103,7 @@ public void tearDown() {
   public void testGetNewAppId() throws Exception {
     Logger rootLogger = LogManager.getRootLogger();
     rootLogger.setLevel(Level.DEBUG);
-    MockRM rm = new MockRM();
+    MockRM rm = new MockRM(conf);
     rm.start();
 
     GetNewApplicationResponse resp = rm.getNewAppId();
@@ -106,7 +116,7 @@ public void testGetNewAppId() throws Exception {
   public void testAppWithNoContainers() throws Exception {
     Logger rootLogger = LogManager.getRootLogger();
     rootLogger.setLevel(Level.DEBUG);
-    MockRM rm = new MockRM();
+    MockRM rm = new MockRM(conf);
     rm.start();
     MockNM nm1 = rm.registerNode("h1:1234", 5120);
 
@@ -128,7 +138,6 @@ public void testAppWithNoContainers() throws Exception {
   public void testAppOnMultiNode() throws Exception {
     Logger rootLogger = LogManager.getRootLogger();
     rootLogger.setLevel(Level.DEBUG);
-    YarnConfiguration conf = new YarnConfiguration();
     conf.set("yarn.scheduler.capacity.node-locality-delay", "-1");
     MockRM rm = new MockRM(conf);
     rm.start();
@@ -188,7 +197,6 @@ public void testAppOnMultiNode() throws Exception {
   // corresponding NM Token.
   @Test (timeout = 20000)
   public void testNMTokenSentForNormalContainer() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
     conf.set(YarnConfiguration.RM_SCHEDULER,
         CapacityScheduler.class.getCanonicalName());
     MockRM rm = new MockRM(conf);
@@ -240,7 +248,7 @@ public void testNMTokenSentForNormalContainer() throws Exception {
 
   @Test (timeout = 40000)
   public void testNMToken() throws Exception {
-    MockRM rm = new MockRM();
+    MockRM rm = new MockRM(conf);
     try {
       rm.start();
       MockNM nm1 = rm.registerNode("h1:1234", 10000);
@@ -422,8 +430,6 @@ protected void allocateContainersAndValidateNMTokens(MockAM am,
 
   @Test (timeout = 300000)
   public void testActivatingApplicationAfterAddingNM() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
-
     MockRM rm1 = new MockRM(conf);
 
     // start like normal because state is empty
@@ -469,7 +475,6 @@ public void testActivatingApplicationAfterAddingNM() throws Exception {
   // is killed or failed, so that client doesn't get the wrong information.
   @Test (timeout = 80000)
   public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
     conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
     MockRM rm1 = new MockRM(conf);
     rm1.start();
@@ -522,7 +527,6 @@ public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception {
 
   @Test (timeout = 60000)
   public void testInvalidatedAMHostPortOnAMRestart() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
     MockRM rm1 = new MockRM(conf);
     rm1.start();
     MockNM nm1 =
@@ -555,7 +559,6 @@ public void testInvalidatedAMHostPortOnAMRestart() throws Exception {
   @Test (timeout = 60000)
   public void testApplicationKillAtAcceptedState() throws Exception {
 
-    YarnConfiguration conf = new YarnConfiguration();
     final Dispatcher dispatcher = new AsyncDispatcher() {
       @Override
       public EventHandler getEventHandler() {
@@ -632,15 +635,4 @@ protected Dispatcher createDispatcher() {
     Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
   }
 
-  public static void main(String[] args) throws Exception {
-    TestRM t = new TestRM();
-    t.testGetNewAppId();
-    t.testAppWithNoContainers();
-    t.testAppOnMultiNode();
-    t.testNMToken();
-    t.testActivatingApplicationAfterAddingNM();
-    t.testInvalidateAMHostPortWhenAMFailedOrKilled();
-    t.testInvalidatedAMHostPortOnAMRestart();
-    t.testApplicationKillAtAcceptedState();
-  }
 }