Better usage messages.

Print the actual error message before the usual usage string.

Author: Marcelo Vanzin

bin/pyspark

@@ -23,9 +23,12 @@ export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
 source "$SPARK_HOME"/bin/load-spark-env.sh
 
 function usage() {
+  if [ -n "$1" ]; then
+    echo $1
+  fi
   echo "Usage: ./bin/pyspark [options]" 1>&2
   "$SPARK_HOME"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2
-  exit 0
+  exit $2
 }
 export -f usage
 

bin/spark-shell

@@ -31,9 +31,12 @@ set -o posix
 export FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 
 usage() {
+  if [ -n "$1" ]; then
+    echo $1
+  fi
   echo "Usage: ./bin/spark-shell [options]"
   "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2
-  exit $1
+  exit $2
 }
 export -f usage
 

bin/spark-shell2.cmd

@@ -39,13 +39,14 @@ set SPARK_SUBMIT_OPTS="%SPARK_SUBMIT_OPTS% -Dscala.usejavacp=true"
 :run_shell
 call %SPARK_HOME%\bin\spark-submit2.cmd --class org.apache.spark.repl.Main %*
 set SPARK_ERROR_LEVEL=%ERRORLEVEL%
-if "%SPARK_LAUNCHER_USAGE_ERROR%"=="1" (
+if not "x%SPARK_LAUNCHER_USAGE_ERROR%"=="x" (
   call :usage
   exit /b 1
 )
 exit /b %SPARK_ERROR_LEVEL%
 
 :usage
+echo %SPARK_LAUNCHER_USAGE_ERROR%
 echo "Usage: .\bin\spark-shell.cmd [options]" >&2
 call %SPARK_HOME%\bin\spark-submit2.cmd --help 2>&1 | findstr /V "Usage" 1>&2
 goto :eof

bin/spark-sql

@@ -31,6 +31,9 @@ export CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver"
 export FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 
 function usage {
+  if [ -n "$1" ]; then
+    echo $1
+  fi
   echo "Usage: ./bin/spark-sql [options] [cli option]"
   pattern="usage"
   pattern+="\|Spark assembly has been built with Hive"
@@ -43,12 +46,12 @@ function usage {
   echo
   echo "CLI options:"
   "$FWDIR"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
-  exit $1
+  exit $2
 }
 export -f usage
 
 if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
-  usage 0
+  usage "" 0
 fi
 
 exec "$FWDIR"/bin/spark-submit --class $CLASS "$@"

bin/spark-submit

@@ -19,10 +19,16 @@
 
 SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
 
-usage() {
-  "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit --help
-  exit $1
-}
-export -f usage
+# Only define a usage function if an upstream script hasn't done so.
+if ! type -t usage >/dev/null 2>&1; then
+  usage() {
+    if [ -n "$1" ]; then
+      echo $1
+    fi
+    "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit --help
+    exit $2
+  }
+  export -f usage
+fi
 
 exec "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"

bin/spark-submit.cmd

@@ -20,15 +20,4 @@ rem
 rem This is the entry point for running Spark submit. To avoid polluting the
 rem environment, it just launches a new cmd to do the real work.
 
-set CLASS=org.apache.spark.deploy.SparkSubmit
-call %~dp0spark-class2.cmd  %CLASS% %*
-set SPARK_ERROR_LEVEL=%ERRORLEVEL%
-if "%SPARK_LAUNCHER_USAGE_ERROR%"=="1" (
-  call :usage
-  exit /b 1
-)
-exit /b %SPARK_ERROR_LEVEL%
-
-:usage
-call %SPARK_HOME%\bin\spark-class2.cmd %CLASS% --help
-goto :eof
+cmd /V /E /C %~dp0spark-submit2.cmd %*

bin/spark-submit2.cmd

@@ -0,0 +1,35 @@
+@echo off
+
+rem
+rem Licensed to the Apache Software Foundation (ASF) under one or more
+rem contributor license agreements.  See the NOTICE file distributed with
+rem this work for additional information regarding copyright ownership.
+rem The ASF licenses this file to You under the Apache License, Version 2.0
+rem (the "License"); you may not use this file except in compliance with
+rem the License.  You may obtain a copy of the License at
+rem
+rem    http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+rem
+
+rem This is the entry point for running Spark submit. To avoid polluting the
+rem environment, it just launches a new cmd to do the real work.
+
+set CLASS=org.apache.spark.deploy.SparkSubmit
+call %~dp0spark-class2.cmd %CLASS% %*
+set SPARK_ERROR_LEVEL=%ERRORLEVEL%
+if not "x%SPARK_LAUNCHER_USAGE_ERROR%"=="x" (
+  call :usage
+  exit /b 1
+)
+exit /b %SPARK_ERROR_LEVEL%
+
+:usage
+echo %SPARK_LAUNCHER_USAGE_ERROR%
+call %SPARK_HOME%\bin\spark-class2.cmd %CLASS% --help
+goto :eof

launcher/src/main/java/org/apache/spark/launcher/Main.java

@@ -54,6 +54,7 @@ public static void main(String[] argsArray) throws Exception {
     String className = args.remove(0);
 
     boolean printLaunchCommand;
+    boolean printUsage;
     AbstractCommandBuilder builder;
     try {
       if (className.equals("org.apache.spark.deploy.SparkSubmit")) {
@@ -62,9 +63,11 @@ public static void main(String[] argsArray) throws Exception {
         builder = new SparkClassCommandBuilder(className, args);
       }
       printLaunchCommand = !isEmpty(System.getenv("SPARK_PRINT_LAUNCH_COMMAND"));
+      printUsage = false;
     } catch (IllegalArgumentException e) {
-      builder = new UsageLauncher();
+      builder = new UsageCommandBuilder(e.getMessage());
       printLaunchCommand = false;
+      printUsage = true;
     }
 
     Map<String, String> env = new HashMap<String, String>();
@@ -75,7 +78,13 @@ public static void main(String[] argsArray) throws Exception {
     }
 
     if (isWindows()) {
-      System.out.println(prepareWindowsCommand(cmd, env));
+      // When printing the usage message, we can't use "cmd /v" since that prevents the env
+      // variable from being seen in the caller script. So do not call prepareWindowsCommand().
+      if (printUsage) {
+        System.out.println(join(" ", cmd));
+      } else {
+        System.out.println(prepareWindowsCommand(cmd, env));
+      }
     } else {
       // In bash, use NULL as the arg separator since it cannot be used in an argument.
       List<String> bashCmd = prepareBashCommand(cmd, env);
@@ -133,23 +142,29 @@ private static List<String> prepareBashCommand(List<String> cmd, Map<String, Str
    * Internal launcher used when command line parsing fails. This will behave differently depending
    * on the platform:
    *
-   * - On Unix-like systems, it will print a call to the "usage" function with argument "1". The
-   *   function is expected to print the command's usage and exit with the provided exit code.
-   *   The script should use "export -f usage" after declaring a function called "usage", so that
-   *   the function is available to downstream scripts.
+   * - On Unix-like systems, it will print a call to the "usage" function with two arguments: the
+   *   the error string, and the exit code to use. The function is expected to print the command's
+   *   usage and exit with the provided exit code. The script should use "export -f usage" after
+   *   declaring a function called "usage", so that the function is available to downstream scripts.
    *
-   * - On Windows it will set the variable "SPARK_LAUNCHER_USAGE_ERROR" to "1". The batch script
-   *   should check for this variable and print its usage, since batch scripts don't really support
-   *   the "export -f" functionality used in bash.
+   * - On Windows it will set the variable "SPARK_LAUNCHER_USAGE_ERROR" to the usage error message.
+   *   The batch script should check for this variable and print its usage, since batch scripts
+   *   don't really support the "export -f" functionality used in bash.
    */
-  private static class UsageLauncher extends AbstractCommandBuilder {
+  private static class UsageCommandBuilder extends AbstractCommandBuilder {
+
+    private final String message;
+
+    UsageCommandBuilder(String message) {
+      this.message = message;
+    }
 
     @Override
     public List<String> buildCommand(Map<String, String> env) {
       if (isWindows()) {
-        return Arrays.asList("set", "SPARK_LAUNCHER_USAGE_ERROR=1");
+        return Arrays.asList("set", "SPARK_LAUNCHER_USAGE_ERROR=" + message);
       } else {
-        return Arrays.asList("usage", "1");
+        return Arrays.asList("usage", message, "1");
       }
     }