From 68828951a523a75632637671dc9a8de6be9d6469 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Tue, 24 Sep 2024 12:18:32 +0200 Subject: [PATCH 1/2] YARN-11733. Fix the order of updating CPU controls with cgroup v1 Change-Id: I09429c878c124be9d6a09e8f027ab89d34606f2f --- .../resources/CGroupsCpuResourceHandlerImpl.java | 8 ++++---- .../TestCGroupsCpuResourceHandlerImpl.java | 16 ++++++++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java index 54686fddb2f52..04b084852b969 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java @@ -59,14 +59,14 @@ public class CGroupsCpuResourceHandlerImpl extends AbstractCGroupsCpuResourceHan @Override protected void updateCgroupMaxCpuLimit(String cgroupId, String quota, String period) throws ResourceHandlerException { - if (quota != null) { - cGroupsHandler - .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_QUOTA_US, quota); - } if (period != null) { cGroupsHandler .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_PERIOD_US, period); } + if (quota != null) { + cGroupsHandler + .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_QUOTA_US, quota); + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java index 842fc6b437a72..f44b2e5d696e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java @@ -31,6 +31,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.InOrder; import org.mockito.Mockito; import java.io.File; @@ -190,11 +191,12 @@ public void testPreStartStrictUsage() throws Exception { CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf(CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT)); // set quota and period - verify(mockCGroupsHandler, times(1)) + InOrder cpuLimitOrder = inOrder(mockCGroupsHandler); + cpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US)); - verify(mockCGroupsHandler, times(1)) + cpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf( (int) (CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US * share))); @@ -222,10 +224,11 @@ public void testPreStartRestrictedContainers() throws Exception { conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, cpuPerc); cGroupsCpuResourceHandler.bootstrap(plugin, conf); - verify(mockCGroupsHandler, times(1)) + InOrder rootCpuLimitOrder = inOrder(mockCGroupsHandler); + rootCpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf("333333")); - verify(mockCGroupsHandler, times(1)) + rootCpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US)); @@ -262,10 +265,11 @@ public void testPreStartRestrictedContainers() throws Exception { CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf( CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT * cVcores)); // set quota and period - verify(mockCGroupsHandler, times(1)) + InOrder cpuLimitOrder = inOrder(mockCGroupsHandler); + cpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(periodUS)); - verify(mockCGroupsHandler, times(1)) + cpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(quotaUS)); } From 4c87c3db99c44628b07ce123c8fa43be5dc18bde Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Tue, 24 Sep 2024 13:17:49 +0200 Subject: [PATCH 2/2] YARN-11733. Simplified the unit test changes Change-Id: I28b4a73fa7ec56ef8af413c5d76ceee8b2e9d7e4 --- .../linux/resources/TestCGroupsCpuResourceHandlerImpl.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java index f44b2e5d696e2..9fbf9f03d2bc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java @@ -224,11 +224,11 @@ public void testPreStartRestrictedContainers() throws Exception { conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, cpuPerc); cGroupsCpuResourceHandler.bootstrap(plugin, conf); - InOrder rootCpuLimitOrder = inOrder(mockCGroupsHandler); - rootCpuLimitOrder.verify(mockCGroupsHandler, times(1)) + InOrder cpuLimitOrder = inOrder(mockCGroupsHandler); + cpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf("333333")); - rootCpuLimitOrder.verify(mockCGroupsHandler, times(1)) + cpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US)); @@ -265,7 +265,6 @@ public void testPreStartRestrictedContainers() throws Exception { CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf( CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT * cVcores)); // set quota and period - InOrder cpuLimitOrder = inOrder(mockCGroupsHandler); cpuLimitOrder.verify(mockCGroupsHandler, times(1)) .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(periodUS));