Skip to content

YARN-11036. Do not inherit from TestRMWebServicesCapacitySched #3895

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

YARN-11036. Do not inherit from TestRMWebServicesCapacitySched #3895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ec2547d
YARN-11036. Do not inherit from TestRMWebServicesCapacitySched
tomicooler Jan 17, 2022
a87e59c
review fix
tomicooler Mar 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 30 additions & 17 deletions .../org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,42 +77,37 @@

public class TestRMWebServicesCapacitySched extends JerseyTestBase {

protected static MockRM rm;
private MockRM rm;

public static class WebServletModule extends ServletModule {
private final MockRM rm;

WebServletModule(MockRM rm) {
this.rm = rm;
}

private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
new Configuration(false));
setupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
}

public TestRMWebServicesCapacitySched() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
super(createWebAppDescriptor());
}

@Before
@Override
public void setUp() throws Exception {
super.setUp();
rm = createMockRM(new CapacitySchedulerConfiguration(
new Configuration(false)));
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule()));
Guice.createInjector(new WebServletModule(rm)));
}

public static void setupQueueConfiguration(
Expand Down Expand Up @@ -389,4 +384,22 @@ public static void updateTestDataAutomatically(String configFilename, String act
Assert.fail("overwrite should not fail " + e.getMessage());
}
}

public static WebAppDescriptor createWebAppDescriptor() {
return new WebAppDescriptor.Builder(
TestRMWebServicesCapacitySched.class.getPackage().getName())
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build();
}

public static MockRM createMockRM(CapacitySchedulerConfiguration csConf) {
setupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
return new MockRM(conf);
}
}
65 changes: 18 additions & 47 deletions ...adoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;

import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;

import java.io.IOException;
import java.util.HashMap;
Expand All @@ -34,67 +31,28 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.junit.Test;

import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;

public class TestRMWebServicesCapacitySchedDynamicConfig extends
JerseyTestBase {
private static final int GB = 1024;
private static MockRM rm;
private MockRM rm;

private CapacitySchedulerQueueManager autoQueueHandler;

private static class WebServletModule extends ServletModule {
private final Configuration conf;

WebServletModule(Configuration conf) {
this.conf = conf;
}

@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
}

private void initResourceManager(Configuration conf) throws IOException {
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule(conf)));
rm.start();
//Need to call reinitialize as
//MutableCSConfigurationProvider with InMemoryConfigurationStore
//somehow does not load the queues properly and falls back to default config.
//Therefore CS will think there's only the default queue there.
((CapacityScheduler) rm.getResourceScheduler()).reinitialize(conf,
rm.getRMContext(), true);
}

public TestRMWebServicesCapacitySchedDynamicConfig() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
super(createWebAppDescriptor());
}

@Test
Expand Down Expand Up @@ -335,4 +293,17 @@ public static Configuration createConfiguration(
return config;
}
}

private void initResourceManager(Configuration conf) throws IOException {
rm = createMockRM(new CapacitySchedulerConfiguration(conf));
GuiceServletConfig.setInjector(
Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm)));
rm.start();
//Need to call reinitialize as
//MutableCSConfigurationProvider with InMemoryConfigurationStore
//somehow does not load the queues properly and falls back to default config.
//Therefore CS will think there's only the default queue there.
((CapacityScheduler) rm.getResourceScheduler()).reinitialize(conf,
rm.getRMContext(), true);
}
}
31 changes: 25 additions & 6 deletions ...pache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,20 @@

package org.apache.hadoop.yarn.server.resourcemanager.webapp;

import com.google.inject.Guice;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.junit.Before;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
Expand Down Expand Up @@ -83,6 +88,8 @@
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfNodes;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyQueueOrder;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
Expand All @@ -91,11 +98,23 @@
/**
* Tests for scheduler/app activities.
*/
public class TestRMWebServicesSchedulerActivities
extends TestRMWebServicesCapacitySched {
public class TestRMWebServicesSchedulerActivities extends JerseyTestBase {

private MockRM rm;

private static final Logger LOG = LoggerFactory.getLogger(
TestRMWebServicesSchedulerActivities.class);
public TestRMWebServicesSchedulerActivities() {
super(createWebAppDescriptor());
}

@Before
@Override
public void setUp() throws Exception {
super.setUp();
rm = createMockRM(new CapacitySchedulerConfiguration(
new Configuration(false)));
GuiceServletConfig.setInjector(
Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm)));
}

@Test
public void testAssignMultipleContainersPerNodeHeartbeat()
Expand Down