Change format to --conf PROP=VALUE

Author: sryza

core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala

@@ -291,6 +291,14 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
         jars = Utils.resolveURIs(value)
         parse(tail)
 
+      case ("--conf") :: value :: tail =>
+        val equalsIndex = value.indexOf('=')
+        if (equalsIndex == -1) {
+          SparkSubmit.printErrorAndExit(s"Spark config without '=': $value")
+        }
+        sparkProperties(value.substring(0, equalsIndex)) = value.substring(equalsIndex+1)
+        parse(tail)
+
       case ("--help" | "-h") :: tail =>
         printUsageAndExit(0)
 
@@ -305,14 +313,6 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
             case v if v.startsWith("--") && v.contains("=") && v.split("=").size == 2 =>
               val parts = v.split("=")
               parse(Seq(parts(0), parts(1)) ++ tail)
-            // spark config property
-            case v if v.startsWith("--spark.") =>
-              if (tail.isEmpty) {
-                val errMessage = s"Spark config without value: $v"
-                SparkSubmit.printErrorAndExit(errMessage)
-              }
-              sparkProperties(v.substring(2)) = tail.head
-              parse(tail.tail)
             case v if v.startsWith("-") =>
               val errMessage = s"Unrecognized option '$value'."
               SparkSubmit.printErrorAndExit(errMessage)
@@ -372,6 +372,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
         |
         |  --help, -h                  Show this help message and exit
         |  --verbose, -v               Print additional debug output
+        |  --conf PROP=VALUE           Arbitrary Spark configuration property.
         |
         | Spark standalone with cluster deploy mode only:
         |  --driver-cores NUM          Cores for driver (Default: 1).

core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala

@@ -120,7 +120,7 @@ class SparkSubmitSuite extends FunSuite with Matchers {
       "--archives", "archive1.txt,archive2.txt",
       "--num-executors", "6",
       "--name", "beauty",
-      "--spark.shuffle.spill", "false",
+      "--conf", "spark.shuffle.spill=false",
       "thejar.jar",
       "arg1", "arg2")
     val appArgs = new SparkSubmitArguments(clArgs)
@@ -158,7 +158,7 @@ class SparkSubmitSuite extends FunSuite with Matchers {
       "--archives", "archive1.txt,archive2.txt",
       "--num-executors", "6",
       "--name", "trill",
-      "--spark.shuffle.spill", "false",
+      "--conf", "spark.shuffle.spill=false",
       "thejar.jar",
       "arg1", "arg2")
     val appArgs = new SparkSubmitArguments(clArgs)
@@ -190,7 +190,7 @@ class SparkSubmitSuite extends FunSuite with Matchers {
       "--supervise",
       "--driver-memory", "4g",
       "--driver-cores", "5",
-      "--spark.shuffle.spill", "false",
+      "--conf", "spark.shuffle.spill=false",
       "thejar.jar",
       "arg1", "arg2")
     val appArgs = new SparkSubmitArguments(clArgs)
@@ -214,7 +214,7 @@ class SparkSubmitSuite extends FunSuite with Matchers {
       "--total-executor-cores", "5",
       "--class", "org.SomeClass",
       "--driver-memory", "4g",
-      "--spark.shuffle.spill", "false",
+      "--conf", "spark.shuffle.spill=false",
       "thejar.jar",
       "arg1", "arg2")
     val appArgs = new SparkSubmitArguments(clArgs)
@@ -236,7 +236,7 @@ class SparkSubmitSuite extends FunSuite with Matchers {
       "--total-executor-cores", "5",
       "--class", "org.SomeClass",
       "--driver-memory", "4g",
-      "--spark.shuffle.spill", "false",
+      "--conf", "spark.shuffle.spill=false",
       "thejar.jar",
       "arg1", "arg2")
     val appArgs = new SparkSubmitArguments(clArgs)

docs/configuration.md

@@ -42,7 +42,7 @@ val sc = new SparkContext(new SparkConf())
 
 Then, you can supply configuration values at runtime:
 {% highlight bash %}
-./bin/spark-submit --name "My fancy app" --master local[4] myApp.jar --spark.shuffle.spill false
+./bin/spark-submit --name "My app" --master local[4] myApp.jar --conf spark.shuffle.spill=false
 {% endhighlight %}
 
 The Spark shell and [`spark-submit`](cluster-overview.html#launching-applications-with-spark-submit)