From ec2547d8d1fb5ffcf0399f429b817137e8c6baa1 Mon Sep 17 00:00:00 2001 From: Tamas Domok Date: Mon, 17 Jan 2022 13:18:13 +0100 Subject: [PATCH 1/2] YARN-11036. Do not inherit from TestRMWebServicesCapacitySched Change-Id: I48e5b1e020d5cc6fb5ae4a7edc38e9212f4f069b --- .../TestRMWebServicesCapacitySched.java | 47 +++++++++----- ...WebServicesCapacitySchedDynamicConfig.java | 65 +++++-------------- .../TestRMWebServicesSchedulerActivities.java | 31 +++++++-- 3 files changed, 73 insertions(+), 70 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 1f73870cccec5..99d9d7522f336 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -77,42 +77,37 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { - protected static MockRM rm; + private MockRM rm; + + public static class WebServletModule extends ServletModule { + private final MockRM rm; + + WebServletModule(MockRM rm) { + this.rm = rm; + } - private static class WebServletModule extends ServletModule { @Override protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration( - new Configuration(false)); - setupQueueConfiguration(csConf); - YarnConfiguration conf = new YarnConfiguration(csConf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, - YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); - rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); } } public TestRMWebServicesCapacitySched() { - super(new WebAppDescriptor.Builder( - "org.apache.hadoop.yarn.server.resourcemanager.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); + super(createWebAppDescriptor()); } @Before @Override public void setUp() throws Exception { super.setUp(); + rm = createMockRM(new CapacitySchedulerConfiguration( + new Configuration(false))); GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule())); + Guice.createInjector(new WebServletModule(rm))); } public static void setupQueueConfiguration( @@ -389,4 +384,22 @@ public static void updateTestDataAutomatically(String configFilename, String act Assert.fail("overwrite should not fail " + e.getMessage()); } } + + public static WebAppDescriptor createWebAppDescriptor() { + return new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build(); + } + + public static MockRM createMockRM(CapacitySchedulerConfiguration csConf) { + setupQueueConfiguration(csConf); + YarnConfiguration conf = new YarnConfiguration(csConf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, + YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); + return new MockRM(conf); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index df4f18e1069ea..6bc5489beec1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -19,10 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import com.google.inject.Guice; -import com.google.inject.servlet.ServletModule; import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; import java.io.IOException; import java.util.HashMap; @@ -34,67 +31,28 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; -import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.junit.Test; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor; public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase { - private static final int GB = 1024; - private static MockRM rm; + private MockRM rm; private CapacitySchedulerQueueManager autoQueueHandler; - private static class WebServletModule extends ServletModule { - private final Configuration conf; - - WebServletModule(Configuration conf) { - this.conf = conf; - } - - @Override - protected void configureServlets() { - bind(JAXBContextResolver.class); - bind(RMWebServices.class); - bind(GenericExceptionHandler.class); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, - YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); - rm = new MockRM(conf); - bind(ResourceManager.class).toInstance(rm); - serve("/*").with(GuiceContainer.class); - } - } - - private void initResourceManager(Configuration conf) throws IOException { - GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule(conf))); - rm.start(); - //Need to call reinitialize as - //MutableCSConfigurationProvider with InMemoryConfigurationStore - //somehow does not load the queues properly and falls back to default config. - //Therefore CS will think there's only the default queue there. - ((CapacityScheduler) rm.getResourceScheduler()).reinitialize(conf, - rm.getRMContext(), true); - } - public TestRMWebServicesCapacitySchedDynamicConfig() { - super(new WebAppDescriptor.Builder( - "org.apache.hadoop.yarn.server.resourcemanager.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); + super(createWebAppDescriptor()); } @Test @@ -335,4 +293,17 @@ public static Configuration createConfiguration( return config; } } + + private void initResourceManager(Configuration conf) throws IOException { + rm = createMockRM(new CapacitySchedulerConfiguration(conf)); + GuiceServletConfig.setInjector( + Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm))); + rm.start(); + //Need to call reinitialize as + //MutableCSConfigurationProvider with InMemoryConfigurationStore + //somehow does not load the queues properly and falls back to default config. + //Therefore CS will think there's only the default queue there. + ((CapacityScheduler) rm.getResourceScheduler()).reinitialize(conf, + rm.getRMContext(), true); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index f864794d59266..55d1297086e9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -18,15 +18,20 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import com.google.inject.Guice; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.core.util.MultivaluedMapImpl; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.junit.Before; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -83,6 +88,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfNodes; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyQueueOrder; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -91,11 +98,23 @@ /** * Tests for scheduler/app activities. */ -public class TestRMWebServicesSchedulerActivities - extends TestRMWebServicesCapacitySched { +public class TestRMWebServicesSchedulerActivities extends JerseyTestBase { + + private MockRM rm; - private static final Logger LOG = LoggerFactory.getLogger( - TestRMWebServicesSchedulerActivities.class); + public TestRMWebServicesSchedulerActivities() { + super(createWebAppDescriptor()); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + rm = createMockRM(new CapacitySchedulerConfiguration( + new Configuration(false))); + GuiceServletConfig.setInjector( + Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm))); + } @Test public void testAssignMultipleContainersPerNodeHeartbeat() From a87e59cd71afb54a84899d6607822d1f1b6fd38b Mon Sep 17 00:00:00 2001 From: Tamas Domok Date: Tue, 8 Mar 2022 09:00:04 +0100 Subject: [PATCH 2/2] review fix Change-Id: I73f0396b0230c7cad6fad33b924c601a9303ab54 --- .../resourcemanager/webapp/TestRMWebServicesCapacitySched.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 99d9d7522f336..d380ef95f17da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -387,7 +387,7 @@ public static void updateTestDataAutomatically(String configFilename, String act public static WebAppDescriptor createWebAppDescriptor() { return new WebAppDescriptor.Builder( - "org.apache.hadoop.yarn.server.resourcemanager.webapp") + TestRMWebServicesCapacitySched.class.getPackage().getName()) .contextListenerClass(GuiceServletConfig.class) .filterClass(com.google.inject.servlet.GuiceFilter.class) .contextPath("jersey-guice-filter").servletPath("/").build();