From a69850b6fdcbe2e234e70a597d9ad6beae6a6937 Mon Sep 17 00:00:00 2001 From: Gabor Somogyi Date: Tue, 29 May 2018 17:38:56 +0200 Subject: [PATCH 1/6] [SPARK-24319][SPARK SUBMIT] Fix spark-submit execution where no main class is required. --- .../launcher/SparkSubmitCommandBuilder.java | 10 ++++++- .../SparkSubmitCommandBuilderSuite.java | 26 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 5cb6457bf5c2..bffc706702a0 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -91,6 +91,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { final List userArgs; private final List parsedArgs; private final boolean requiresAppResource; + private final boolean requiresMainClass; private final boolean isExample; /** @@ -106,6 +107,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { */ SparkSubmitCommandBuilder() { this.requiresAppResource = true; + this.requiresMainClass = false; this.isExample = false; this.parsedArgs = new ArrayList<>(); this.userArgs = new ArrayList<>(); @@ -145,9 +147,11 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { OptionParser parser = new OptionParser(true); parser.parse(submitArgs); this.requiresAppResource = parser.requiresAppResource; + this.requiresMainClass = parser.requiresMainClass; } else { this.isExample = isExample; this.requiresAppResource = false; + this.requiresMainClass = false; } } @@ -229,7 +233,7 @@ List buildSparkSubmitArgs() { args.add(join(",", pyFiles)); } - if (isExample) { + if (isExample && requiresMainClass) { checkArgument(mainClass != null, "Missing example class name."); } @@ -422,6 +426,7 @@ private List findExamplesJars() { private class OptionParser extends SparkSubmitOptionParser { boolean requiresAppResource = true; + boolean requiresMainClass = true; private final boolean errorOnUnknownArgs; OptionParser(boolean errorOnUnknownArgs) { @@ -471,16 +476,19 @@ protected boolean handle(String opt, String value) { case KILL_SUBMISSION: case STATUS: requiresAppResource = false; + requiresMainClass = false; parsedArgs.add(opt); parsedArgs.add(value); break; case HELP: case USAGE_ERROR: requiresAppResource = false; + requiresMainClass = false; parsedArgs.add(opt); break; case VERSION: requiresAppResource = false; + requiresMainClass = false; parsedArgs.add(opt); break; default: diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 2e050f841307..948977ca0617 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -18,6 +18,7 @@ package org.apache.spark.launcher; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -74,8 +75,11 @@ public void testCliHelpAndNoArg() throws Exception { @Test public void testCliKillAndStatus() throws Exception { - testCLIOpts(parser.STATUS); - testCLIOpts(parser.KILL_SUBMISSION); + List params = Arrays.asList("driver-20160531171222-0000"); + testCLIOpts(null, parser.STATUS, params); + testCLIOpts(null, parser.KILL_SUBMISSION, params); + testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.STATUS, params); + testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.KILL_SUBMISSION, params); } @Test @@ -190,6 +194,13 @@ public void testSparkRShell() throws Exception { env.get("SPARKR_SUBMIT_ARGS")); } + @Test + public void testExamplesRunnerNoMainClass() throws Exception { + testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.HELP, null); + testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.USAGE_ERROR, null); + testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.VERSION, null); + } + @Test public void testExamplesRunner() throws Exception { List sparkSubmitArgs = Arrays.asList( @@ -344,8 +355,15 @@ private List buildCommand(List args, Map env) th return newCommandBuilder(args).buildCommand(env); } - private void testCLIOpts(String opt) throws Exception { - List helpArgs = Arrays.asList(opt, "driver-20160531171222-0000"); + private void testCLIOpts(String appResource, String opt, List params) throws Exception { + List helpArgs = new ArrayList<>(); + if (appResource != null) { + helpArgs.add(appResource); + } + helpArgs.add(opt); + if (params != null) { + helpArgs.addAll(params); + } Map env = new HashMap<>(); List cmd = buildCommand(helpArgs, env); assertTrue(opt + " should be contained in the final cmd.", From 9052dff88709a02f63cbaca4bece38075a9ab402 Mon Sep 17 00:00:00 2001 From: Gabor Somogyi Date: Wed, 30 May 2018 11:01:02 +0200 Subject: [PATCH 2/6] No argument run-example test added. --- .../spark/launcher/SparkSubmitCommandBuilder.java | 13 +++++++++---- .../launcher/SparkSubmitCommandBuilderSuite.java | 10 ++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index bffc706702a0..989ab5a01377 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -144,10 +144,15 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { } this.isExample = isExample; - OptionParser parser = new OptionParser(true); - parser.parse(submitArgs); - this.requiresAppResource = parser.requiresAppResource; - this.requiresMainClass = parser.requiresMainClass; + if (!submitArgs.isEmpty()) { + OptionParser parser = new OptionParser(true); + parser.parse(submitArgs); + this.requiresAppResource = parser.requiresAppResource; + this.requiresMainClass = parser.requiresMainClass; + } else { + this.requiresAppResource = false; + this.requiresMainClass = false; + } } else { this.isExample = isExample; this.requiresAppResource = false; diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 948977ca0617..05247e0ea8dd 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -194,6 +194,16 @@ public void testSparkRShell() throws Exception { env.get("SPARKR_SUBMIT_ARGS")); } + @Test + public void testExamplesRunnerNoArg() throws Exception { + List sparkSubmitArgs = Arrays.asList(SparkSubmitCommandBuilder.RUN_EXAMPLE); + Map env = new HashMap<>(); + List cmd = buildCommand(sparkSubmitArgs, env); + assertTrue( + "org.apache.spark.deploy.SparkSubmit should be contained in the final cmd of empty input.", + cmd.contains("org.apache.spark.deploy.SparkSubmit")); + } + @Test public void testExamplesRunnerNoMainClass() throws Exception { testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.HELP, null); From 12a5145464dc8429ec9b120afb86a0b984deed93 Mon Sep 17 00:00:00 2001 From: Gabor Somogyi Date: Fri, 1 Jun 2018 14:35:29 +0200 Subject: [PATCH 3/6] Fixed noargs error message, merged requires flags --- .../launcher/SparkSubmitCommandBuilder.java | 46 ++++++++----------- .../SparkSubmitCommandBuilderSuite.java | 17 +++---- 2 files changed, 26 insertions(+), 37 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 989ab5a01377..685fd3d2bf49 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -90,8 +90,8 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { final List userArgs; private final List parsedArgs; - private final boolean requiresAppResource; - private final boolean requiresMainClass; + // Special command means no appResource and no mainClass required + private final boolean isSpecialCommand; private final boolean isExample; /** @@ -106,8 +106,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { * spark-submit argument list to be modified after creation. */ SparkSubmitCommandBuilder() { - this.requiresAppResource = true; - this.requiresMainClass = false; + this.isSpecialCommand = false; this.isExample = false; this.parsedArgs = new ArrayList<>(); this.userArgs = new ArrayList<>(); @@ -141,31 +140,31 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { case RUN_EXAMPLE: isExample = true; submitArgs = args.subList(1, args.size()); + if (submitArgs.isEmpty()) { + throw new IllegalArgumentException("Missing argument"); + } } this.isExample = isExample; if (!submitArgs.isEmpty()) { OptionParser parser = new OptionParser(true); parser.parse(submitArgs); - this.requiresAppResource = parser.requiresAppResource; - this.requiresMainClass = parser.requiresMainClass; + this.isSpecialCommand = parser.isSpecialCommand; } else { - this.requiresAppResource = false; - this.requiresMainClass = false; + this.isSpecialCommand = true; } } else { this.isExample = isExample; - this.requiresAppResource = false; - this.requiresMainClass = false; + this.isSpecialCommand = true; } } @Override public List buildCommand(Map env) throws IOException, IllegalArgumentException { - if (PYSPARK_SHELL.equals(appResource) && requiresAppResource) { + if (PYSPARK_SHELL.equals(appResource) && !isSpecialCommand) { return buildPySparkShellCommand(env); - } else if (SPARKR_SHELL.equals(appResource) && requiresAppResource) { + } else if (SPARKR_SHELL.equals(appResource) && !isSpecialCommand) { return buildSparkRCommand(env); } else { return buildSparkSubmitCommand(env); @@ -175,18 +174,18 @@ public List buildCommand(Map env) List buildSparkSubmitArgs() { List args = new ArrayList<>(); OptionParser parser = new OptionParser(false); - final boolean requiresAppResource; + final boolean isSpecialCommand; // If the user args array is not empty, we need to parse it to detect exactly what // the user is trying to run, so that checks below are correct. if (!userArgs.isEmpty()) { parser.parse(userArgs); - requiresAppResource = parser.requiresAppResource; + isSpecialCommand = parser.isSpecialCommand; } else { - requiresAppResource = this.requiresAppResource; + isSpecialCommand = this.isSpecialCommand; } - if (!allowsMixedArguments && requiresAppResource) { + if (!allowsMixedArguments && !isSpecialCommand) { checkArgument(appResource != null, "Missing application resource."); } @@ -238,7 +237,7 @@ List buildSparkSubmitArgs() { args.add(join(",", pyFiles)); } - if (isExample && requiresMainClass) { + if (isExample && !isSpecialCommand) { checkArgument(mainClass != null, "Missing example class name."); } @@ -430,8 +429,7 @@ private List findExamplesJars() { private class OptionParser extends SparkSubmitOptionParser { - boolean requiresAppResource = true; - boolean requiresMainClass = true; + boolean isSpecialCommand = false; private final boolean errorOnUnknownArgs; OptionParser(boolean errorOnUnknownArgs) { @@ -480,20 +478,14 @@ protected boolean handle(String opt, String value) { break; case KILL_SUBMISSION: case STATUS: - requiresAppResource = false; - requiresMainClass = false; + isSpecialCommand = true; parsedArgs.add(opt); parsedArgs.add(value); break; case HELP: case USAGE_ERROR: - requiresAppResource = false; - requiresMainClass = false; - parsedArgs.add(opt); - break; case VERSION: - requiresAppResource = false; - requiresMainClass = false; + isSpecialCommand = true; parsedArgs.add(opt); break; default: diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 05247e0ea8dd..0612de9f6257 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -194,14 +194,11 @@ public void testSparkRShell() throws Exception { env.get("SPARKR_SUBMIT_ARGS")); } - @Test + @Test(expected = IllegalArgumentException.class) public void testExamplesRunnerNoArg() throws Exception { List sparkSubmitArgs = Arrays.asList(SparkSubmitCommandBuilder.RUN_EXAMPLE); Map env = new HashMap<>(); - List cmd = buildCommand(sparkSubmitArgs, env); - assertTrue( - "org.apache.spark.deploy.SparkSubmit should be contained in the final cmd of empty input.", - cmd.contains("org.apache.spark.deploy.SparkSubmit")); + buildCommand(sparkSubmitArgs, env); } @Test @@ -366,16 +363,16 @@ private List buildCommand(List args, Map env) th } private void testCLIOpts(String appResource, String opt, List params) throws Exception { - List helpArgs = new ArrayList<>(); + List args = new ArrayList<>(); if (appResource != null) { - helpArgs.add(appResource); + args.add(appResource); } - helpArgs.add(opt); + args.add(opt); if (params != null) { - helpArgs.addAll(params); + args.addAll(params); } Map env = new HashMap<>(); - List cmd = buildCommand(helpArgs, env); + List cmd = buildCommand(args, env); assertTrue(opt + " should be contained in the final cmd.", cmd.contains(opt)); } From b03e3de9b326a8cf9061125e0f22bde2a12bf30f Mon Sep 17 00:00:00 2001 From: Gabor Somogyi Date: Tue, 12 Jun 2018 10:48:45 +0200 Subject: [PATCH 4/6] Move buildCommand part under exception handling --- .../java/org/apache/spark/launcher/Main.java | 36 +++++++++++++------ .../launcher/SparkSubmitCommandBuilder.java | 6 ++-- .../SparkSubmitCommandBuilderSuite.java | 19 ++++++++++ 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/Main.java b/launcher/src/main/java/org/apache/spark/launcher/Main.java index 1e34bb8c7327..264b27ed7317 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/Main.java +++ b/launcher/src/main/java/org/apache/spark/launcher/Main.java @@ -17,6 +17,7 @@ package org.apache.spark.launcher; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -54,10 +55,12 @@ public static void main(String[] argsArray) throws Exception { String className = args.remove(0); boolean printLaunchCommand = !isEmpty(System.getenv("SPARK_PRINT_LAUNCH_COMMAND")); - AbstractCommandBuilder builder; + Map env = new HashMap<>(); + List cmd; if (className.equals("org.apache.spark.deploy.SparkSubmit")) { try { - builder = new SparkSubmitCommandBuilder(args); + AbstractCommandBuilder builder = new SparkSubmitCommandBuilder(args); + cmd = buildCommand(builder, env, printLaunchCommand); } catch (IllegalArgumentException e) { printLaunchCommand = false; System.err.println("Error: " + e.getMessage()); @@ -76,17 +79,12 @@ public static void main(String[] argsArray) throws Exception { help.add(parser.className); } help.add(parser.USAGE_ERROR); - builder = new SparkSubmitCommandBuilder(help); + AbstractCommandBuilder builder = new SparkSubmitCommandBuilder(help); + cmd = buildCommand(builder, env, printLaunchCommand); } } else { - builder = new SparkClassCommandBuilder(className, args); - } - - Map env = new HashMap<>(); - List cmd = builder.buildCommand(env); - if (printLaunchCommand) { - System.err.println("Spark Command: " + join(" ", cmd)); - System.err.println("========================================"); + AbstractCommandBuilder builder = new SparkClassCommandBuilder(className, args); + cmd = buildCommand(builder, env, printLaunchCommand); } if (isWindows()) { @@ -101,6 +99,22 @@ public static void main(String[] argsArray) throws Exception { } } + /** + * Prepare spark commands with the appropriate command builder. + * If printLaunchCommand is set then the commands will be printed to the stderr. + */ + private static List buildCommand(AbstractCommandBuilder builder, + Map env, + boolean printLaunchCommand) + throws IOException, IllegalArgumentException { + List cmd = builder.buildCommand(env); + if (printLaunchCommand) { + System.err.println("Spark Command: " + join(" ", cmd)); + System.err.println("========================================"); + } + return cmd; + } + /** * Prepare a command line for execution from a Windows batch script. * diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 685fd3d2bf49..992687299e48 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -139,10 +139,8 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { case RUN_EXAMPLE: isExample = true; + appResource = SparkLauncher.NO_RESOURCE; submitArgs = args.subList(1, args.size()); - if (submitArgs.isEmpty()) { - throw new IllegalArgumentException("Missing argument"); - } } this.isExample = isExample; @@ -151,7 +149,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { parser.parse(submitArgs); this.isSpecialCommand = parser.isSpecialCommand; } else { - this.isSpecialCommand = true; + this.isSpecialCommand = false; } } else { this.isExample = isExample; diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 0612de9f6257..e75b18927c81 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -28,7 +28,10 @@ import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; + import static org.junit.Assert.*; public class SparkSubmitCommandBuilderSuite extends BaseSuite { @@ -208,6 +211,22 @@ public void testExamplesRunnerNoMainClass() throws Exception { testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.VERSION, null); } + @Rule + public ExpectedException testExamplesRunnerWithMasterNoMainClassEx = ExpectedException.none(); + + @Test + public void testExamplesRunnerWithMasterNoMainClass() throws Exception { + testExamplesRunnerWithMasterNoMainClassEx.expect(IllegalArgumentException.class); + testExamplesRunnerWithMasterNoMainClassEx.expectMessage("Missing example class name."); + + List sparkSubmitArgs = Arrays.asList( + SparkSubmitCommandBuilder.RUN_EXAMPLE, + parser.MASTER + "=foo" + ); + Map env = new HashMap<>(); + buildCommand(sparkSubmitArgs, env); + } + @Test public void testExamplesRunner() throws Exception { List sparkSubmitArgs = Arrays.asList( From 8a8067ec89705fb8acaec82eecd94c2312aeea76 Mon Sep 17 00:00:00 2001 From: Gabor Somogyi Date: Wed, 13 Jun 2018 11:34:07 +0200 Subject: [PATCH 5/6] Minor fixes. --- .../src/main/java/org/apache/spark/launcher/Main.java | 7 ++++--- .../spark/launcher/SparkSubmitCommandBuilder.java | 10 +++------- .../spark/launcher/SparkSubmitCommandBuilderSuite.java | 10 +++++----- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/Main.java b/launcher/src/main/java/org/apache/spark/launcher/Main.java index 264b27ed7317..42b1f9dcd660 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/Main.java +++ b/launcher/src/main/java/org/apache/spark/launcher/Main.java @@ -103,9 +103,10 @@ public static void main(String[] argsArray) throws Exception { * Prepare spark commands with the appropriate command builder. * If printLaunchCommand is set then the commands will be printed to the stderr. */ - private static List buildCommand(AbstractCommandBuilder builder, - Map env, - boolean printLaunchCommand) + private static List buildCommand( + AbstractCommandBuilder builder, + Map env, + boolean printLaunchCommand) throws IOException, IllegalArgumentException { List cmd = builder.buildCommand(env); if (printLaunchCommand) { diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 992687299e48..cc65f78b45c3 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -144,13 +144,9 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { } this.isExample = isExample; - if (!submitArgs.isEmpty()) { - OptionParser parser = new OptionParser(true); - parser.parse(submitArgs); - this.isSpecialCommand = parser.isSpecialCommand; - } else { - this.isSpecialCommand = false; - } + OptionParser parser = new OptionParser(true); + parser.parse(submitArgs); + this.isSpecialCommand = parser.isSpecialCommand; } else { this.isExample = isExample; this.isSpecialCommand = true; diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index e75b18927c81..b343094b2e7b 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -39,6 +39,9 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { private static File dummyPropsFile; private static SparkSubmitOptionParser parser; + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @BeforeClass public static void setUp() throws Exception { dummyPropsFile = File.createTempFile("spark", "properties"); @@ -211,13 +214,10 @@ public void testExamplesRunnerNoMainClass() throws Exception { testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.VERSION, null); } - @Rule - public ExpectedException testExamplesRunnerWithMasterNoMainClassEx = ExpectedException.none(); - @Test public void testExamplesRunnerWithMasterNoMainClass() throws Exception { - testExamplesRunnerWithMasterNoMainClassEx.expect(IllegalArgumentException.class); - testExamplesRunnerWithMasterNoMainClassEx.expectMessage("Missing example class name."); + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Missing example class name."); List sparkSubmitArgs = Arrays.asList( SparkSubmitCommandBuilder.RUN_EXAMPLE, From 5a737d7d9dc11ee84744f57700d4c0a1b13fb73b Mon Sep 17 00:00:00 2001 From: Gabor Somogyi Date: Thu, 14 Jun 2018 17:11:05 +0200 Subject: [PATCH 6/6] Indent fix --- launcher/src/main/java/org/apache/spark/launcher/Main.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/Main.java b/launcher/src/main/java/org/apache/spark/launcher/Main.java index 42b1f9dcd660..d967aa39a482 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/Main.java +++ b/launcher/src/main/java/org/apache/spark/launcher/Main.java @@ -104,10 +104,9 @@ public static void main(String[] argsArray) throws Exception { * If printLaunchCommand is set then the commands will be printed to the stderr. */ private static List buildCommand( - AbstractCommandBuilder builder, - Map env, - boolean printLaunchCommand) - throws IOException, IllegalArgumentException { + AbstractCommandBuilder builder, + Map env, + boolean printLaunchCommand) throws IOException, IllegalArgumentException { List cmd = builder.buildCommand(env); if (printLaunchCommand) { System.err.println("Spark Command: " + join(" ", cmd));