[SPARK-24319][SPARK SUBMIT] Fix spark-submit execution where no main class is required.

Author: gaborgsomogyi

launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java

@@ -91,6 +91,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
   final List<String> userArgs;
   private final List<String> parsedArgs;
   private final boolean requiresAppResource;
+  private final boolean requiresMainClass;
   private final boolean isExample;
 
   /**
@@ -106,6 +107,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
    */
   SparkSubmitCommandBuilder() {
     this.requiresAppResource = true;
+    this.requiresMainClass = false;
     this.isExample = false;
     this.parsedArgs = new ArrayList<>();
     this.userArgs = new ArrayList<>();
@@ -145,9 +147,11 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
       OptionParser parser = new OptionParser(true);
       parser.parse(submitArgs);
       this.requiresAppResource = parser.requiresAppResource;
+      this.requiresMainClass = parser.requiresMainClass;
     } else {
       this.isExample = isExample;
       this.requiresAppResource = false;
+      this.requiresMainClass = false;
     }
   }
 
@@ -229,7 +233,7 @@ List<String> buildSparkSubmitArgs() {
       args.add(join(",", pyFiles));
     }
 
-    if (isExample) {
+    if (isExample && requiresMainClass) {
       checkArgument(mainClass != null, "Missing example class name.");
     }
 
@@ -422,6 +426,7 @@ private List<String> findExamplesJars() {
   private class OptionParser extends SparkSubmitOptionParser {
 
     boolean requiresAppResource = true;
+    boolean requiresMainClass = true;
     private final boolean errorOnUnknownArgs;
 
     OptionParser(boolean errorOnUnknownArgs) {
@@ -471,16 +476,19 @@ protected boolean handle(String opt, String value) {
         case KILL_SUBMISSION:
         case STATUS:
           requiresAppResource = false;
+          requiresMainClass = false;
           parsedArgs.add(opt);
           parsedArgs.add(value);
           break;
         case HELP:
         case USAGE_ERROR:
           requiresAppResource = false;
+          requiresMainClass = false;
           parsedArgs.add(opt);
           break;
         case VERSION:
           requiresAppResource = false;
+          requiresMainClass = false;
           parsedArgs.add(opt);
           break;
         default:

launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java

@@ -18,6 +18,7 @@
 package org.apache.spark.launcher;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -74,8 +75,11 @@ public void testCliHelpAndNoArg() throws Exception {
 
   @Test
   public void testCliKillAndStatus() throws Exception {
-    testCLIOpts(parser.STATUS);
-    testCLIOpts(parser.KILL_SUBMISSION);
+    List<String> params = Arrays.asList("driver-20160531171222-0000");
+    testCLIOpts(null, parser.STATUS, params);
+    testCLIOpts(null, parser.KILL_SUBMISSION, params);
+    testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.STATUS, params);
+    testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.KILL_SUBMISSION, params);
   }
 
   @Test
@@ -190,6 +194,13 @@ public void testSparkRShell() throws Exception {
       env.get("SPARKR_SUBMIT_ARGS"));
   }
 
+  @Test
+  public void testExamplesRunnerNoMainClass() throws Exception {
+    testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.HELP, null);
+    testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.USAGE_ERROR, null);
+    testCLIOpts(SparkSubmitCommandBuilder.RUN_EXAMPLE, parser.VERSION, null);
+  }
+
   @Test
   public void testExamplesRunner() throws Exception {
     List<String> sparkSubmitArgs = Arrays.asList(
@@ -344,8 +355,15 @@ private List<String> buildCommand(List<String> args, Map<String, String> env) th
     return newCommandBuilder(args).buildCommand(env);
   }
 
-  private void testCLIOpts(String opt) throws Exception {
-    List<String> helpArgs = Arrays.asList(opt, "driver-20160531171222-0000");
+  private void testCLIOpts(String appResource, String opt, List<String> params) throws Exception {
+    List<String> helpArgs = new ArrayList<>();
+    if (appResource != null) {
+      helpArgs.add(appResource);
+    }
+    helpArgs.add(opt);
+    if (params != null) {
+      helpArgs.addAll(params);
+    }
     Map<String, String> env = new HashMap<>();
     List<String> cmd = buildCommand(helpArgs, env);
     assertTrue(opt + " should be contained in the final cmd.",