Skip to content

Commit eecc588

Browse files
author
slfan1989
committed
YARN-11290. Add Mysql sp_getApplicationsHomeSubCluster Script And Fix Junit Test.
1 parent 354b8b3 commit eecc588

File tree

3 files changed

+32
-16
lines changed

3 files changed

+32
-16
lines changed

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,14 +122,18 @@ BEGIN
122122
WHERE applicationId = applicationID_IN;
123123
END //
124124

125-
CREATE PROCEDURE sp_getApplicationsHomeSubCluster(IN limit_IN int)
125+
CREATE PROCEDURE sp_getApplicationsHomeSubCluster(IN limit_IN int, IN homeSubCluster_IN varchar(256))
126126
BEGIN
127127
SELECT
128128
applicationId, homeSubCluster
129129
FROM
130130
(SELECT
131131
applicationId, homeSubCluster, ROW_NUMBER() OVER (ORDER BY applicationId) AS row_num
132-
FROM applicationsHomeSubCluster) WHERE row_num <= limit_IN
132+
FROM applicationsHomeSubCluster)
133+
WHERE row_num <= limit_IN
134+
AND CASE WHEN homeSubCluster_IN IS NULL THEN 1 = 1
135+
WHEN homeSubCluster_IN IS NOT NULL THEN homeSubCluster = homeSubCluster_IN
136+
END
133137
END //
134138

135139
CREATE PROCEDURE sp_deleteApplicationHomeSubCluster(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,12 @@
2828
import java.util.Map;
2929
import java.util.HashMap;
3030
import java.util.Set;
31+
import java.util.function.Supplier;
3132
import java.util.stream.Collectors;
3233
import java.util.Arrays;
3334

3435
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
36+
import org.apache.hadoop.test.GenericTestUtils;
3537
import org.apache.hadoop.test.LambdaTestUtils;
3638
import org.apache.hadoop.util.Time;
3739
import org.apache.hadoop.yarn.MockApps;
@@ -132,6 +134,7 @@
132134
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
133135
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
134136
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
137+
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
135138
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
136139
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
137140
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -1308,13 +1311,6 @@ public void testSubmitReservation() throws Exception {
13081311
GetNewReservationResponse response = interceptor.getNewReservation(request);
13091312
Assert.assertNotNull(response);
13101313

1311-
// allow plan follower to synchronize, manually trigger an assignment
1312-
Map<SubClusterId, MockRM> mockRMs = interceptor.getMockRMs();
1313-
for (MockRM mockRM : mockRMs.values()) {
1314-
ReservationSystem reservationSystem = mockRM.getReservationSystem();
1315-
reservationSystem.synchronizePlan("root.decided", true);
1316-
}
1317-
13181314
// Submit Reservation
13191315
ReservationId reservationId = response.getReservationId();
13201316
ReservationDefinition rDefinition = createReservationDefinition(1024, 1);
@@ -1384,13 +1380,6 @@ public void testSubmitReservationMultipleSubmission() throws Exception {
13841380
GetNewReservationResponse response = interceptor.getNewReservation(request);
13851381
Assert.assertNotNull(response);
13861382

1387-
// allow plan follower to synchronize, manually trigger an assignment
1388-
Map<SubClusterId, MockRM> mockRMs = interceptor.getMockRMs();
1389-
for (MockRM mockRM : mockRMs.values()) {
1390-
ReservationSystem reservationSystem = mockRM.getReservationSystem();
1391-
reservationSystem.synchronizePlan("root.decided", true);
1392-
}
1393-
13941383
// First Submit Reservation
13951384
ReservationId reservationId = response.getReservationId();
13961385
ReservationDefinition rDefinition = createReservationDefinition(1024, 1);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,16 @@
2828
import java.util.Map;
2929
import java.util.HashMap;
3030
import java.util.concurrent.ConcurrentHashMap;
31+
import java.util.concurrent.TimeoutException;
3132

33+
import org.apache.hadoop.test.GenericTestUtils;
3234
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
3335
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
3436
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
3537
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
3638
import org.apache.hadoop.yarn.api.records.NodeAttribute;
3739
import org.apache.hadoop.yarn.api.records.NodeAttributeType;
40+
import org.apache.hadoop.yarn.api.records.Resource;
3841
import org.apache.hadoop.yarn.exceptions.YarnException;
3942
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
4043
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
@@ -43,6 +46,8 @@
4346
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
4447
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManager;
4548
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
49+
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
50+
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
4651
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
4752
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
4853
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
@@ -90,6 +95,7 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster(
9095
mockRMs.put(subClusterId, mockRM);
9196
}
9297
initNodeAttributes(subClusterId, mockRM);
98+
initReservationSystem(mockRM);
9399
return mockRM.getClientRMService();
94100
}
95101
}
@@ -161,4 +167,21 @@ private void initNodeAttributes(SubClusterId subClusterId, MockRM mockRM) {
161167
throw new RuntimeException(e);
162168
}
163169
}
170+
171+
private void initReservationSystem(MockRM mockRM) throws YarnException {
172+
try {
173+
// Ensure that the reserved resources of the RM#Reservation System are allocated
174+
String planName = "root.decided";
175+
ReservationSystem reservationSystem = mockRM.getReservationSystem();
176+
reservationSystem.synchronizePlan(planName, true);
177+
178+
GenericTestUtils.waitFor(() -> {
179+
Plan plan = reservationSystem.getPlan(planName);
180+
Resource resource = plan.getTotalCapacity();
181+
return (resource.getMemorySize() > 0 && resource.getVirtualCores() > 0);
182+
}, 100, 2000);
183+
} catch (TimeoutException | InterruptedException e) {
184+
throw new YarnException(e);
185+
}
186+
}
164187
}

0 commit comments

Comments
 (0)