Responses to Tom's review

Author: pwendell

.rat-excludes

@@ -11,6 +11,7 @@ RELEASE
 control
 docs
 fairscheduler.xml.template
+spark-defaults.properties.template
 log4j.properties
 log4j.properties.template
 metrics.properties.template

conf/spark-defaults.properties.template

@@ -0,0 +1,7 @@
+# Default system properties included when running spark-submit.
+# This is useful for setting default environmental settings.
+
+# Example:
+# spark.master 	          spark://master:7077
+# spark.eventLog.enabled  true
+# spark.eventLog.dir      hdfs://namenode:8021/directory

conf/spark-env.sh.template

@@ -17,7 +17,6 @@
 # - MESOS_NATIVE_LIBRARY, to point to your libmesos.so if you use Mesos
 
 # Options read in YARN client mode
-# - SPARK_YARN_APP_JAR, Path to your application’s JAR file (required)
 # - SPARK_EXECUTOR_INSTANCES, Number of workers to start (Default: 2)
 # - SPARK_EXECUTOR_CORES, Number of cores for the workers (Default: 1).
 # - SPARK_EXECUTOR_MEMORY, Memory per Worker (e.g. 1000M, 2G) (Default: 1G)

core/src/main/scala/org/apache/spark/SparkConf.scala

@@ -244,7 +244,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
           |This has undefined behavior when running on a cluster and is deprecated in Spark 1.0+.
           |
           |Please instead use:
-          | - ./spark-submit with conf/spark-defaults.conf to set properties for an application
+          | - ./spark-submit with conf/spark-defaults.properties to set defaults for an application
           | - ./spark-submit with --driver-java-options to set -X options for a driver
           | - spark.executor.extraJavaOptions to set -X options for executors
           | - SPARK_DAEMON_OPTS to set java options for standalone daemons (i.e. master, worker)

core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala

@@ -145,7 +145,7 @@ private[spark] class SparkSubmitArguments(args: Array[String]) {
       driverExtraClassPath = value
       parseOpts(tail)
 
-    case ("--driver-java-opts") :: value :: tail =>
+    case ("--driver-java-options") :: value :: tail =>
       driverExtraJavaOptions = value
       parseOpts(tail)
 
@@ -217,8 +217,11 @@ private[spark] class SparkSubmitArguments(args: Array[String]) {
         |  --jars JARS                 A comma-separated list of local jars to include on the
         |                              driver classpath and that SparkContext.addJar will work
         |                              with. Doesn't work on standalone with 'cluster' deploy mode.
+        |  --properties-file FILE      Path to a file from which to load extra properties. If not
+        |                              specified, this will look for conf/spark-defaults.properties.
+        |
         |  --driver-memory MEM         Memory for driver (e.g. 1000M, 2G) (Default: 512M).
-        |  --driver-java-opts          Extra Java options to pass to the driver
+        |  --driver-java-options       Extra Java options to pass to the driver
         |  --driver-library-path       Extra library path entries to pass to the driver
         |  --driver-class-path         Extra class path entries to pass to the driver
         |

docs/configuration.md

@@ -659,7 +659,7 @@ Apart from these, the following properties are also available, and may be useful
     A string of extra JVM options to pass to executors. For instance, GC settings or other
     logging. Note that it is illegal to set Spark properties or heap size settings with this 
     option. Spark properties should be set using a SparkConf object or the 
-    spark-defaults.conf file used with the spark-submit script. Heap size settings can be set
+    spark-defaults.properties file used with the spark-submit script. Heap size settings can be set
     with spark.executor.memory.
   </td>
 </tr>