From e42fef953fddcf6089805d55109d4d82b28df251 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Sun, 16 Mar 2025 15:17:55 +0100 Subject: [PATCH 1/2] YARN-11796. Accept extra JDK 17 options in tests --- .../TestLinuxContainerExecutorWithMocks.java | 7 +++++++ .../localizer/TestContainerLocalizer.java | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 58038d607a825..e2e3b533f9b03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -53,6 +53,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -311,6 +312,12 @@ public void testStartLocalizer() throws IOException { .build()); List result=readMockParams(); + + if (Shell.isJavaVersionAtLeast(17)) { + assertTrue(result.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); + assertTrue(result.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); + } + assertThat(result).hasSize(26); assertThat(result.get(0)).isEqualTo(YarnConfiguration. DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java index 13927219f928d..61d1f18cd5ea6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java @@ -751,6 +751,10 @@ public void testAdminOptionsPrecedeUserDefinedJavaOptions() throws Exception { " userOption1 userOption2"); List javaOpts = localizer.getJavaOpts(conf); + if (Shell.isJavaVersionAtLeast(17)) { + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); + } assertEquals(4, javaOpts.size()); assertTrue(javaOpts.get(0).equals("adminOption1")); assertTrue(javaOpts.get(1).equals("adminOption2")); @@ -768,6 +772,10 @@ public void testAdminOptionsPrecedeDefaultUserOptions() throws Exception { "adminOption1 adminOption2"); List javaOpts = localizer.getJavaOpts(conf); + if (Shell.isJavaVersionAtLeast(17)) { + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); + } assertEquals(3, javaOpts.size()); assertTrue(javaOpts.get(0).equals("adminOption1")); assertTrue(javaOpts.get(1).equals("adminOption2")); @@ -784,6 +792,10 @@ public void testUserOptionsWhenAdminOptionsAreNotDefined() throws Exception { "userOption1 userOption2"); List javaOpts = localizer.getJavaOpts(conf); + if (Shell.isJavaVersionAtLeast(17)) { + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); + } assertEquals(2, javaOpts.size()); assertTrue(javaOpts.get(0).equals("userOption1")); assertTrue(javaOpts.get(1).equals("userOption2")); @@ -797,6 +809,10 @@ public void testJavaOptionsWithoutDefinedAdminOrUserOptions() throws Exception { Configuration conf = new Configuration(); List javaOpts = localizer.getJavaOpts(conf); + if (Shell.isJavaVersionAtLeast(17)) { + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); + assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); + } assertEquals(1, javaOpts.size()); assertTrue(javaOpts.get(0).equals("-Xmx256m")); } From 697d5ebe62a2185aae94ef2fbfd81d24b81f1418 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Fri, 21 Mar 2025 12:50:43 +0100 Subject: [PATCH 2/2] add comments --- .../nodemanager/TestLinuxContainerExecutorWithMocks.java | 1 + .../containermanager/localizer/TestContainerLocalizer.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index e2e3b533f9b03..757a31c37fbf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -314,6 +314,7 @@ public void testStartLocalizer() throws IOException { List result=readMockParams(); if (Shell.isJavaVersionAtLeast(17)) { + // Added by ContainerLocalizer for JDK17+ (MAPREDUCE-7456) assertTrue(result.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); assertTrue(result.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java index 61d1f18cd5ea6..a30a13f0a132e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java @@ -715,6 +715,7 @@ public void testDefaultJavaOptionsWhenExtraJDK17OptionsAreConfigured() throws Ex List javaOpts = localizer.getJavaOpts(conf); if (Shell.isJavaVersionAtLeast(17)) { + // Added by ContainerLocalizer for JDK17+ (MAPREDUCE-7456) assertTrue(javaOpts.contains("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); assertTrue(javaOpts.contains("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); } @@ -733,6 +734,7 @@ public void testDefaultJavaOptionsWhenExtraJDK17OptionsAreNotConfigured() throws List javaOpts = localizer.getJavaOpts(conf); if (Shell.isJavaVersionAtLeast(17)) { + // Added by ContainerLocalizer for JDK17+ (MAPREDUCE-7456) assertFalse(javaOpts.contains("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); assertFalse(javaOpts.contains("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); } @@ -752,6 +754,7 @@ public void testAdminOptionsPrecedeUserDefinedJavaOptions() throws Exception { List javaOpts = localizer.getJavaOpts(conf); if (Shell.isJavaVersionAtLeast(17)) { + // Added by ContainerLocalizer for JDK17+ (MAPREDUCE-7456) assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); } @@ -773,6 +776,7 @@ public void testAdminOptionsPrecedeDefaultUserOptions() throws Exception { List javaOpts = localizer.getJavaOpts(conf); if (Shell.isJavaVersionAtLeast(17)) { + // Added by ContainerLocalizer for JDK17+ (MAPREDUCE-7456) assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); } @@ -793,6 +797,7 @@ public void testUserOptionsWhenAdminOptionsAreNotDefined() throws Exception { List javaOpts = localizer.getJavaOpts(conf); if (Shell.isJavaVersionAtLeast(17)) { + // Added by ContainerLocalizer for JDK17+ (MAPREDUCE-7456) assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); } @@ -810,6 +815,7 @@ public void testJavaOptionsWithoutDefinedAdminOrUserOptions() throws Exception { List javaOpts = localizer.getJavaOpts(conf); if (Shell.isJavaVersionAtLeast(17)) { + // Added by ContainerLocalizer for JDK17+ (MAPREDUCE-7456) assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); assertTrue(javaOpts.remove("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); }