From e4ef4283e45ac7197f9ead1275d9a02050df4854 Mon Sep 17 00:00:00 2001 From: sandeep katta Date: Sun, 22 Sep 2019 11:31:17 +0530 Subject: [PATCH 1/5] driver java options are not passed to driver process in Yarn client mode --- .../org/apache/spark/launcher/SparkSubmitCommandBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index f603e100fd2eb..7a7433865125b 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -392,7 +392,7 @@ boolean isClientMode(Map userProps) { // Default master is "local[*]", so assume client mode in that case return userMaster == null || "client".equals(userDeployMode) || - (!userMaster.equals("yarn") && userDeployMode == null); + userDeployMode == null; } /** From 32760a7430196cd72d97c61d86b80f86c012930a Mon Sep 17 00:00:00 2001 From: sandeep katta Date: Mon, 23 Sep 2019 07:47:20 +0530 Subject: [PATCH 2/5] added test case --- .../SparkSubmitCommandBuilderSuite.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 32a91b1789412..f84089c4b8976 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -250,6 +250,34 @@ public void testMissingAppResource() { new SparkSubmitCommandBuilder().buildSparkSubmitArgs(); } + @Test + public void testisClientMode() { + // Default master is "local[*]" + SparkSubmitCommandBuilder launcher = + newCommandBuilder(Collections.emptyList()); + assertTrue("By default application run in local mode", + launcher.isClientMode(Collections.EMPTY_MAP)); + // --master yarn or it can be any RM + List sparkSubmitArgs = Arrays.asList( + parser.MASTER, + "yarn"); + launcher = newCommandBuilder(sparkSubmitArgs); + assertTrue("By default deploy mode is client", + launcher.isClientMode(Collections.EMPTY_MAP)); + // --master yarn and set spark.submit.deployMode to client + Map userProps = new HashMap<>(); + userProps.put("spark.submit.deployMode", "client"); + assertTrue(launcher.isClientMode(userProps)); + // --master mesos --deploy-mode cluster + sparkSubmitArgs = Arrays.asList( + parser.MASTER, + "mesos", + parser.DEPLOY_MODE, + "cluster"); + launcher = newCommandBuilder(sparkSubmitArgs); + assertTrue(!launcher.isClientMode(Collections.EMPTY_MAP)); + } + private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception { final String DRIVER_DEFAULT_PARAM = "-Ddriver-default"; final String DRIVER_EXTRA_PARAM = "-Ddriver-extra"; From ac8c36d818d25a2be8503a205f8af50d41bd8578 Mon Sep 17 00:00:00 2001 From: sandeep katta Date: Tue, 24 Sep 2019 10:47:04 +0530 Subject: [PATCH 3/5] update review comments --- .../apache/spark/launcher/SparkSubmitCommandBuilderSuite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index f84089c4b8976..f26f5671f3c52 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -251,7 +251,7 @@ public void testMissingAppResource() { } @Test - public void testisClientMode() { + public void testIsClientMode() { // Default master is "local[*]" SparkSubmitCommandBuilder launcher = newCommandBuilder(Collections.emptyList()); From 7b2497a4f1723291ab9fc42e7d510f7b0b6b8b9a Mon Sep 17 00:00:00 2001 From: sandeep katta Date: Tue, 24 Sep 2019 14:26:18 +0530 Subject: [PATCH 4/5] review comments fix --- .../SparkSubmitCommandBuilderSuite.java | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index f26f5671f3c52..752e8d4c23f8b 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -253,29 +253,21 @@ public void testMissingAppResource() { @Test public void testIsClientMode() { // Default master is "local[*]" - SparkSubmitCommandBuilder launcher = - newCommandBuilder(Collections.emptyList()); + SparkSubmitCommandBuilder builder = newCommandBuilder(Collections.emptyList()); assertTrue("By default application run in local mode", - launcher.isClientMode(Collections.EMPTY_MAP)); + builder.isClientMode(Collections.emptyMap())); // --master yarn or it can be any RM - List sparkSubmitArgs = Arrays.asList( - parser.MASTER, - "yarn"); - launcher = newCommandBuilder(sparkSubmitArgs); - assertTrue("By default deploy mode is client", - launcher.isClientMode(Collections.EMPTY_MAP)); + List sparkSubmitArgs = Arrays.asList(parser.MASTER, "yarn"); + builder = newCommandBuilder(sparkSubmitArgs); + assertTrue("By default deploy mode is client", builder.isClientMode(Collections.emptyMap())); // --master yarn and set spark.submit.deployMode to client Map userProps = new HashMap<>(); userProps.put("spark.submit.deployMode", "client"); - assertTrue(launcher.isClientMode(userProps)); + assertTrue(builder.isClientMode(userProps)); // --master mesos --deploy-mode cluster - sparkSubmitArgs = Arrays.asList( - parser.MASTER, - "mesos", - parser.DEPLOY_MODE, - "cluster"); - launcher = newCommandBuilder(sparkSubmitArgs); - assertTrue(!launcher.isClientMode(Collections.EMPTY_MAP)); + sparkSubmitArgs = Arrays.asList(parser.MASTER, "mesos", parser.DEPLOY_MODE, "cluster"); + builder = newCommandBuilder(sparkSubmitArgs); + assertFalse(builder.isClientMode(Collections.emptyMap())); } private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception { From 6f1ba4324ba40f4c7b26afe38efd41f95297d0a4 Mon Sep 17 00:00:00 2001 From: sandeep katta Date: Thu, 26 Sep 2019 11:31:02 +0530 Subject: [PATCH 5/5] changed the condition to 1 liner --- .../org/apache/spark/launcher/SparkSubmitCommandBuilder.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 7a7433865125b..383c3f60a595b 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -390,9 +390,7 @@ boolean isClientMode(Map userProps) { String userMaster = firstNonEmpty(master, userProps.get(SparkLauncher.SPARK_MASTER)); String userDeployMode = firstNonEmpty(deployMode, userProps.get(SparkLauncher.DEPLOY_MODE)); // Default master is "local[*]", so assume client mode in that case - return userMaster == null || - "client".equals(userDeployMode) || - userDeployMode == null; + return userMaster == null || userDeployMode == null || "client".equals(userDeployMode); } /**