From 27741380631e4e293d040d8fc206343814467b37 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Fri, 1 Aug 2014 18:27:41 +0800 Subject: [PATCH 1/4] Added "--" to prevent spark-submit from shadowing application options --- bin/beeline | 29 +++++-------------- bin/spark-sql | 13 ++++++--- .../spark/deploy/SparkSubmitArguments.scala | 16 ++++++++-- .../spark/deploy/SparkSubmitSuite.scala | 16 ++++++++++ sbin/start-thriftserver.sh | 13 ++++++--- .../hive/thriftserver/HiveThriftServer2.scala | 1 - 6 files changed, 55 insertions(+), 33 deletions(-) diff --git a/bin/beeline b/bin/beeline index 09fe366c609f..1bda4dba5060 100755 --- a/bin/beeline +++ b/bin/beeline @@ -17,29 +17,14 @@ # limitations under the License. # -# Figure out where Spark is installed -FWDIR="$(cd `dirname $0`/..; pwd)" +# +# Shell script for starting BeeLine -# Find the java binary -if [ -n "${JAVA_HOME}" ]; then - RUNNER="${JAVA_HOME}/bin/java" -else - if [ `command -v java` ]; then - RUNNER="java" - else - echo "JAVA_HOME is not set" >&2 - exit 1 - fi -fi +# Enter posix mode for bash +set -o posix -# Compute classpath using external script -classpath_output=$($FWDIR/bin/compute-classpath.sh) -if [[ "$?" != "0" ]]; then - echo "$classpath_output" - exit 1 -else - CLASSPATH=$classpath_output -fi +# Figure out where Spark is installed +FWDIR="$(cd `dirname $0`/..; pwd)" CLASS="org.apache.hive.beeline.BeeLine" -exec "$RUNNER" -cp "$CLASSPATH" $CLASS "$@" +exec "$FWDIR/bin/spark-class" $CLASS "$@" diff --git a/bin/spark-sql b/bin/spark-sql index bba7f897b19b..5527f0dc7967 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -26,11 +26,16 @@ set -o posix # Figure out where Spark is installed FWDIR="$(cd `dirname $0`/..; pwd)" -if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then - echo "Usage: ./sbin/spark-sql [options]" - $FWDIR/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 +CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" + +if [[ "$@" = --help ]] || [[ "$@" = -h ]]; then + echo "Usage: ./sbin/spark-sql [options] [--] [CLI options]" + exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 + echo + echo "CLI options:" + exec "$FWDIR"/bin/spark-submit spark-internal --class $CLASS -- -H 2>&1 | tail -n +3 + echo exit 0 fi -CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" exec "$FWDIR"/bin/spark-submit --class $CLASS spark-internal $@ diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index dd044e629876..16892989ed94 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -311,6 +311,15 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { verbose = true parse(tail) + case "--" :: tail => + if (inSparkOpts) { + SparkSubmit.printErrorAndExit( + "Application option separator \"--\" must be after the primary resource " + + "(i.e., application jar file or Python file).") + } else { + childArgs ++= tail.filter(_.nonEmpty) + } + case value :: tail => if (inSparkOpts) { value match { @@ -377,8 +386,11 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { | | --executor-memory MEM Memory per executor (e.g. 1000M, 2G) (Default: 1G). | - | --help, -h Show this help message and exit - | --verbose, -v Print additional debug output + | --help, -h Show this help message and exit. + | --verbose, -v Print additional debug output. + | + | -- A "--" signals the end of spark-submit options, all command + | line arguments after "--" are passed to the application. | | Spark standalone with cluster deploy mode only: | --driver-cores NUM Cores for driver (Default: 1). diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 9190b05e2dba..f3e8917abf42 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -106,6 +106,22 @@ class SparkSubmitSuite extends FunSuite with Matchers { appArgs.childArgs should be (Seq("some", "--weird", "args")) } + test("handles arguments with \"--\"") { + val clArgs = + """--name myApp + |--class Foo + |userjar.jar + |--master local + |some + |-- + |--weird args + """.stripMargin.split("\\s+").toSeq + val appArgs = new SparkSubmitArguments(clArgs) + appArgs.master should be ("local") + appArgs.mainClass should be ("Foo") + appArgs.childArgs should be (Seq("some", "--weird", "args")) + } + test("handles YARN cluster mode") { val clArgs = Seq( "--deploy-mode", "cluster", diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 8398e6f19b51..78ae951d9454 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -26,11 +26,16 @@ set -o posix # Figure out where Spark is installed FWDIR="$(cd `dirname $0`/..; pwd)" -if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then - echo "Usage: ./sbin/start-thriftserver [options]" - $FWDIR/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 +CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2" + +if [[ "$@" = --help ]] || [[ "$@" = -h ]]; then + echo "Usage: ./sbin/start-thriftserver.sh [options] [--] [thrift server options]" + exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 + echo + echo "Thrift server options:" + exec "$FWDIR"/bin/spark-submit spark-internal --class $CLASS -- -H 2>&1 | tail -n +3 + echo exit 0 fi -CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2" exec "$FWDIR"/bin/spark-submit --class $CLASS spark-internal $@ diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala index ddbc2a79fb51..2361ec51ed4a 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala @@ -40,7 +40,6 @@ private[hive] object HiveThriftServer2 extends Logging { val optionsProcessor = new ServerOptionsProcessor("HiveThriftServer2") if (!optionsProcessor.process(args)) { - logger.warn("Error starting HiveThriftServer2 with given arguments") System.exit(-1) } From e2463673da671afa230192a3d919378c2fc5f9cc Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Sat, 2 Aug 2014 21:22:28 +0800 Subject: [PATCH 2/4] Added --primary to retained full downward compatibility --- bin/spark-sql | 8 ++-- .../spark/deploy/SparkSubmitArguments.scala | 39 +++++++++++++++---- .../spark/deploy/SparkSubmitSuite.scala | 22 ++++++++--- docs/submitting-applications.md | 22 ++++++++--- sbin/start-thriftserver.sh | 8 ++-- 5 files changed, 75 insertions(+), 24 deletions(-) diff --git a/bin/spark-sql b/bin/spark-sql index 5527f0dc7967..3d900e4e2866 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -30,12 +30,14 @@ CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" if [[ "$@" = --help ]] || [[ "$@" = -h ]]; then echo "Usage: ./sbin/spark-sql [options] [--] [CLI options]" - exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 + exec "$FWDIR"/bin/spark-submit --help 2>&1 | tail -n +5 1>&2 echo echo "CLI options:" - exec "$FWDIR"/bin/spark-submit spark-internal --class $CLASS -- -H 2>&1 | tail -n +3 + exec "$FWDIR"/bin/spark-submit\ + --primary spark-internal --class $CLASS\ + -- --help 2>&1 | grep -v -- "-H,--help" | tail -n +3 1>&2 echo exit 0 fi -exec "$FWDIR"/bin/spark-submit --class $CLASS spark-internal $@ +exec "$FWDIR"/bin/spark-submit --class $CLASS --primary spark-internal $@ diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 16892989ed94..5b8adfbc4698 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -311,13 +311,25 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { verbose = true parse(tail) + case ("--primary" | "-p") :: value :: tail => + primaryResource = if (!SparkSubmit.isShell(value) && !SparkSubmit.isInternal(value)) { + Utils.resolveURI(value).toString + } else { + value + } + isPython = SparkSubmit.isPython(value) + parse(tail) + case "--" :: tail => if (inSparkOpts) { - SparkSubmit.printErrorAndExit( - "Application option separator \"--\" must be after the primary resource " + - "(i.e., application jar file or Python file).") - } else { + // Primary resource is specified with "--primary", "--" is considered as the separator of + // spark-submit options and user application options. childArgs ++= tail.filter(_.nonEmpty) + } else { + // Primary resource is specified as a positional argument, "--" is passed to the + // application as a normal argument. + childArgs += "--" + parse(tail) } case value :: tail => @@ -331,6 +343,14 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { val errMessage = s"Unrecognized option '$value'." SparkSubmit.printErrorAndExit(errMessage) case v => + if (primaryResource != null) { + // Primary resource has already been specified by --primary. It's more likely that + // user forgot using -- to separate application options from spark-submit options. + SparkSubmit.printErrorAndExit( + s"Unrecognized option '$value', " + + "note that application options must appear after \"--\".") + } + primaryResource = if (!SparkSubmit.isShell(v) && !SparkSubmit.isInternal(v)) { Utils.resolveURI(v).toString @@ -358,7 +378,10 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { outStream.println("Unknown/unsupported param " + unknownParam) } outStream.println( - """Usage: spark-submit [options] [app options] + """Usage: + | spark-submit [options] [app options] + | spark-submit [options] --primary -- [app options] + | |Options: | --master MASTER_URL spark://host:port, mesos://host:port, yarn, or local. | --deploy-mode DEPLOY_MODE Whether to launch the driver program locally ("client") or @@ -389,8 +412,10 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { | --help, -h Show this help message and exit. | --verbose, -v Print additional debug output. | - | -- A "--" signals the end of spark-submit options, all command - | line arguments after "--" are passed to the application. + | --primary The primary jar file or Python file of the application. + | -- Used together with "--primary". A "--" signals the end of + | spark-submit options, everything after "--" are passed as + | command line arguments to the application. | | Spark standalone with cluster deploy mode only: | --driver-cores NUM Cores for driver (Default: 1). diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index f3e8917abf42..72b54285e212 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -70,11 +70,11 @@ class SparkSubmitSuite extends FunSuite with Matchers { } test("prints usage on empty input") { - testPrematureExit(Array[String](), "Usage: spark-submit") + testPrematureExit(Array[String](), "Usage:\n spark-submit") } test("prints usage with only --help") { - testPrematureExit(Array("--help"), "Usage: spark-submit") + testPrematureExit(Array("--help"), "Usage:\n spark-submit") } test("prints error with unrecognized options") { @@ -106,14 +106,14 @@ class SparkSubmitSuite extends FunSuite with Matchers { appArgs.childArgs should be (Seq("some", "--weird", "args")) } - test("handles arguments with \"--\"") { + test("handles arguments to user program with --primary and --") { val clArgs = """--name myApp |--class Foo - |userjar.jar + |--primary userjar.jar |--master local - |some |-- + |some |--weird args """.stripMargin.split("\\s+").toSeq val appArgs = new SparkSubmitArguments(clArgs) @@ -122,6 +122,18 @@ class SparkSubmitSuite extends FunSuite with Matchers { appArgs.childArgs should be (Seq("some", "--weird", "args")) } + test("handles arguments to user program with --primary but no --") { + val clArgs = + """--name myApp + |--class Foo + |--primary userjar.jar + |--master local + |some + |--weird args + """.stripMargin.split("\\s+") + testPrematureExit(clArgs, "Unrecognized option 'some'") + } + test("handles YARN cluster mode") { val clArgs = Seq( "--deploy-mode", "cluster", diff --git a/docs/submitting-applications.md b/docs/submitting-applications.md index 45b70b1a5457..116012177c7e 100644 --- a/docs/submitting-applications.md +++ b/docs/submitting-applications.md @@ -35,17 +35,23 @@ dependencies, and can support different cluster managers and deploy modes that S --deploy-mode \ --conf = \ ... # other options - \ + --primary \ + -- \ [application-arguments] {% endhighlight %} +(**NOTE** Prior to Spark 1.1, `--primary` and `--` are not required. The old application option +passing mode is still supported. Start from Spark 1.1, `--` is used as user application option +separator, to let user application receive arbitrary command line option, including those are once +swallowed by `spark-submit`, like `--master`.) + Some of the commonly used options are: * `--class`: The entry point for your application (e.g. `org.apache.spark.examples.SparkPi`) * `--master`: The [master URL](#master-urls) for the cluster (e.g. `spark://23.195.26.187:7077`) * `--deploy-mode`: Whether to deploy your driver on the worker nodes (`cluster`) or locally as an external client (`client`) (default: `client`)* * `--conf`: Arbitrary Spark configuration property in key=value format. For values that contain spaces wrap "key=value" in quotes (as shown). -* `application-jar`: Path to a bundled jar including your application and all dependencies. The URL must be globally visible inside of your cluster, for instance, an `hdfs://` path or a `file://` path that is present on all nodes. +* `--primary application-jar`: Path to a bundled jar including your application and all dependencies. The URL must be globally visible inside of your cluster, for instance, an `hdfs://` path or a `file://` path that is present on all nodes. * `application-arguments`: Arguments passed to the main method of your main class, if any *A common deployment strategy is to submit your application from a gateway machine that is @@ -71,7 +77,8 @@ examples of common options: ./bin/spark-submit \ --class org.apache.spark.examples.SparkPi \ --master local[8] \ - /path/to/examples.jar \ + --primary /path/to/examples.jar \ + -- \ 100 # Run on a Spark standalone cluster @@ -80,7 +87,8 @@ examples of common options: --master spark://207.184.161.138:7077 \ --executor-memory 20G \ --total-executor-cores 100 \ - /path/to/examples.jar \ + --primary /path/to/examples.jar \ + -- \ 1000 # Run on a YARN cluster @@ -90,13 +98,15 @@ export HADOOP_CONF_DIR=XXX --master yarn-cluster \ # can also be `yarn-client` for client mode --executor-memory 20G \ --num-executors 50 \ - /path/to/examples.jar \ + --primary /path/to/examples.jar \ + -- \ 1000 # Run a Python application on a cluster ./bin/spark-submit \ --master spark://207.184.161.138:7077 \ - examples/src/main/python/pi.py \ + --primary examples/src/main/python/pi.py \ + -- \ 1000 {% endhighlight %} diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 78ae951d9454..90827958bfbe 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -30,12 +30,14 @@ CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2" if [[ "$@" = --help ]] || [[ "$@" = -h ]]; then echo "Usage: ./sbin/start-thriftserver.sh [options] [--] [thrift server options]" - exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 + exec "$FWDIR"/bin/spark-submit --help 2>&1 | tail -n +5 1>&2 echo echo "Thrift server options:" - exec "$FWDIR"/bin/spark-submit spark-internal --class $CLASS -- -H 2>&1 | tail -n +3 + exec "$FWDIR"/bin/spark-submit\ + --primary spark-internal --class $CLASS\ + -- --help 2>&1 | grep -v usage | tail -n +3 1>&2 echo exit 0 fi -exec "$FWDIR"/bin/spark-submit --class $CLASS spark-internal $@ +exec "$FWDIR"/bin/spark-submit --class $CLASS --primary spark-internal $@ From 6c23c521f40504c76127fa7bc167ead65c880011 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Sat, 2 Aug 2014 21:35:51 +0800 Subject: [PATCH 3/4] Allowed empty string as user application argument --- .../spark/deploy/SparkSubmitArguments.scala | 2 +- .../apache/spark/deploy/SparkSubmitSuite.scala | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 5b8adfbc4698..18c70d400c67 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -324,7 +324,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { if (inSparkOpts) { // Primary resource is specified with "--primary", "--" is considered as the separator of // spark-submit options and user application options. - childArgs ++= tail.filter(_.nonEmpty) + childArgs ++= tail } else { // Primary resource is specified as a positional argument, "--" is passed to the // application as a normal argument. diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 72b54285e212..2813ab20d761 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -134,6 +134,22 @@ class SparkSubmitSuite extends FunSuite with Matchers { testPrematureExit(clArgs, "Unrecognized option 'some'") } + test("handles arguments to user program with empty string") { + val clArgs = + """--name myApp + |--class Foo + |--primary userjar.jar + |--master local + |-- + """.stripMargin.split("\\s+").toSeq :+ "" + val appArgs = new SparkSubmitArguments(clArgs) + val xx = clArgs.map(arg => s"[$arg]") + println(s"### $xx") + appArgs.master should be ("local") + appArgs.mainClass should be ("Foo") + appArgs.childArgs should be (Seq("")) + } + test("handles YARN cluster mode") { val clArgs = Seq( "--deploy-mode", "cluster", From b8059c04b817168779e6b717cfe44edf8db47d3d Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Sun, 3 Aug 2014 03:38:50 +0800 Subject: [PATCH 4/4] Addressed @andrewor14's comments --- bin/pyspark | 2 +- bin/spark-shell | 2 +- bin/spark-sql | 2 +- .../org/apache/spark/deploy/SparkSubmitArguments.scala | 10 ++++++---- docs/submitting-applications.md | 9 +++++---- sbin/start-thriftserver.sh | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/bin/pyspark b/bin/pyspark index 39a20e2a24a3..02a709ab71e1 100755 --- a/bin/pyspark +++ b/bin/pyspark @@ -27,7 +27,7 @@ SCALA_VERSION=2.10 if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then echo "Usage: ./bin/pyspark [options]" 1>&2 - $FWDIR/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 + exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v "^Usage\|\( spark-submit\)" 1>&2 exit 0 fi diff --git a/bin/spark-shell b/bin/spark-shell index 756c8179d12b..300ab87aff13 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -33,7 +33,7 @@ FWDIR="$(cd `dirname $0`/..; pwd)" if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then echo "Usage: ./bin/spark-shell [options]" - $FWDIR/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 + exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v "^Usage\|\( spark-submit\)" 1>&2 exit 0 fi diff --git a/bin/spark-sql b/bin/spark-sql index 3d900e4e2866..f74eb455ea49 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -30,7 +30,7 @@ CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" if [[ "$@" = --help ]] || [[ "$@" = -h ]]; then echo "Usage: ./sbin/spark-sql [options] [--] [CLI options]" - exec "$FWDIR"/bin/spark-submit --help 2>&1 | tail -n +5 1>&2 + exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v "^Usage\|\( spark-submit\)" 1>&2 echo echo "CLI options:" exec "$FWDIR"/bin/spark-submit\ diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 18c70d400c67..486a5378d847 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -412,10 +412,12 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { | --help, -h Show this help message and exit. | --verbose, -v Print additional debug output. | - | --primary The primary jar file or Python file of the application. - | -- Used together with "--primary". A "--" signals the end of - | spark-submit options, everything after "--" are passed as - | command line arguments to the application. + | --primary The primary jar file or Python file of the application. Used + | in conjunction with "--" to pass arbitrary arguments to the + | application if any. + | -- A "--" signals the end of spark-submit options, everything + | after "--" are passed as command line arguments to the + | application. Only used in conjunction with "--primary". | | Spark standalone with cluster deploy mode only: | --driver-cores NUM Cores for driver (Default: 1). diff --git a/docs/submitting-applications.md b/docs/submitting-applications.md index 116012177c7e..d5efd907ef36 100644 --- a/docs/submitting-applications.md +++ b/docs/submitting-applications.md @@ -40,10 +40,11 @@ dependencies, and can support different cluster managers and deploy modes that S [application-arguments] {% endhighlight %} -(**NOTE** Prior to Spark 1.1, `--primary` and `--` are not required. The old application option -passing mode is still supported. Start from Spark 1.1, `--` is used as user application option -separator, to let user application receive arbitrary command line option, including those are once -swallowed by `spark-submit`, like `--master`.) +(**NOTE** As of Spark 1.1, `--` is used as a separator for user application options, such that +anything that follows will be passed to the application as command line arguments. This includes +those that were once swallowed by `spark-submit`, e.g. `--help` and `--conf`. Note that this must be +used in conjunction with `--primary` to specify the main application jar. The old way of setting +application jars and passing arguments to applications is still supported.) Some of the commonly used options are: diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 90827958bfbe..7a2d224dc294 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -30,7 +30,7 @@ CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2" if [[ "$@" = --help ]] || [[ "$@" = -h ]]; then echo "Usage: ./sbin/start-thriftserver.sh [options] [--] [thrift server options]" - exec "$FWDIR"/bin/spark-submit --help 2>&1 | tail -n +5 1>&2 + exec "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v "^Usage\|\( spark-submit\)" 1>&2 echo echo "Thrift server options:" exec "$FWDIR"/bin/spark-submit\