addressed comments

Author: ArtRand

docs/running-on-mesos.md

@@ -33,7 +33,8 @@ To get started, follow the steps below to install Mesos and deploy Spark jobs vi
 # Installing Mesos
 
 Spark {{site.SPARK_VERSION}} is designed for use with Mesos {{site.MESOS_VERSION}} or newer and does not
-require any special patches of Mesos.
+require any special patches of Mesos. File and environment-based secrets support requires Mesos 1.3.0 or
+newer.
 
 If you already have a Mesos cluster running, you can skip this Mesos installation step.
 
@@ -426,7 +427,8 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>spark.mesos.secret</code></td>
   <td>(none)</td>
   <td>
-    Set the secret with which Spark framework will use to authenticate with Mesos.
+    Set the secret with which Spark framework will use to authenticate with Mesos. Used, for example, when
+    authenticating with the registry.
   </td>
 </tr>
 <tr>
@@ -482,7 +484,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>spark.mesos.driver.secret.envkey</code></td>
   <td><code>(none)</code></td>
   <td>
-    A comma-seperated list that, if set, the contents of the secret referenced 
+    A comma-separated list that, if set, the contents of the secret referenced
     by spark.mesos.driver.secret.name or spark.mesos.driver.value will be 
     written to the provided environment variable in the driver's process.
   </td>
@@ -491,7 +493,7 @@ See the [configuration page](configuration.html) for information on Spark config
 <td><code>spark.mesos.driver.secret.filename</code></td>
   <td><code>(none)</code></td>
   <td>
-    A comma-seperated list that, if set, the contents of the secret referenced by
+    A comma-separated list that, if set, the contents of the secret referenced by
     spark.mesos.driver.secret.name or spark.mesos.driver.secret.value will be 
     written to the provided file.  Relative paths are relative to the container's work
     directory.  Absolute paths must already exist.  Consult the Mesos Secret
@@ -502,15 +504,15 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>spark.mesos.driver.secret.name</code></td>
   <td><code>(none)</code></td>
   <td>
-    A comma-seperated list of secret references.  Consult the Mesos Secret
+    A comma-separated list of secret references.  Consult the Mesos Secret
     protobuf for more information.
   </td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.secret.value</code></td>
   <td><code>(none)</code></td>
   <td>
-    A comma-seperated list of secret values.  Consult the Mesos Secret
+    A comma-separated list of secret values.  Consult the Mesos Secret
     protobuf for more information.
   </td>
 </tr>

docs/security.md

@@ -73,6 +73,9 @@ For long-running apps like Spark Streaming apps to be able to write to HDFS, it
 ### Standalone mode
 The user needs to provide key-stores and configuration options for master and workers. They have to be set by attaching appropriate Java system properties in `SPARK_MASTER_OPTS` and in `SPARK_WORKER_OPTS` environment variables, or just in `SPARK_DAEMON_JAVA_OPTS`. In this mode, the user may allow the executors to use the SSL settings inherited from the worker which spawned that executor. It can be accomplished by setting `spark.ssl.useNodeLocalConf` to `true`. If that parameter is set, the settings provided by user on the client side, are not used by the executors.
 
+### Mesos mode
+Mesos 1.3.0 and newer supports `Secrets` primitives as both file-based and environment based secrets. Spark allows the specification of file-based and environment variable based secrets with the `spark.mesos.driver.secret.filename` and `spark.mesos.driver.secret.envkey`, respectively. Depending on the secret store backend secrets can be passed by reference or by value with the `spark.mesos.driver.secret.name` and `spark.mesos.driver.secret.value` configuration properties, respectively. Reference type secrets are served by the secret store and referred to by name, for example `/mysecret`. Value type secrets are passed on the command line and translated into their appropriate files or environment variables. 
+
 ### Preparing the key-stores
 Key-stores can be generated by `keytool` program. The reference documentation for this tool is
 [here](https://docs.oracle.com/javase/7/docs/technotes/tools/solaris/keytool.html). The most basic

resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala

@@ -58,28 +58,28 @@ package object config {
 
   private [spark] val DRIVER_LABELS =
     ConfigBuilder("spark.mesos.driver.labels")
-      .doc("Mesos labels to add to the driver.  Labels are free-form key-value pairs.  Key-value" +
+      .doc("Mesos labels to add to the driver.  Labels are free-form key-value pairs. Key-value " +
         "pairs should be separated by a colon, and commas used to list more than one." +
         "Ex. key:value,key2:value2")
       .stringConf
       .createOptional
 
   private[spark] val SECRET_NAME =
     ConfigBuilder("spark.mesos.driver.secret.name")
-      .doc("A comma-seperated list of secret references. Consult the Mesos Secret protobuf for " +
+      .doc("A comma-separated list of secret references. Consult the Mesos Secret protobuf for " +
         "more information.")
       .stringConf
       .createOptional
 
   private[spark] val SECRET_VALUE =
     ConfigBuilder("spark.mesos.driver.secret.value")
-      .doc("A comma-seperated list of secret values.")
+      .doc("A comma-separated list of secret values.")
       .stringConf
       .createOptional
 
   private[spark] val SECRET_ENVKEY =
     ConfigBuilder("spark.mesos.driver.secret.envkey")
-      .doc("A comma-seperated list of the environment variables to contain the secrets." +
+      .doc("A comma-separated list of the environment variables to contain the secrets." +
         "The environment variable will be set on the driver.")
       .stringConf
       .createOptional

resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala

@@ -397,9 +397,12 @@ private[spark] class MesosClusterScheduler(
 
     // add secret environment variables
     getSecretEnvVar(desc).foreach { variable =>
-      logInfo(s"Setting secret name=${variable.getSecret.getReference.getName} " +
-        s"on environment variable name=${variable.getName}")
-
+      if (variable.getSecret.getReference.isInitialized) {
+        logInfo(s"Setting reference secret ${variable.getSecret.getReference.getName}" +
+          s"on file ${variable.getName}")
+      } else {
+        logInfo(s"Setting secret on environment variable name=${variable.getName}")
+      }
       envBuilder.addVariables(variable)
     }
 
@@ -589,9 +592,12 @@ private[spark] class MesosClusterScheduler(
     val containerInfo = MesosSchedulerBackendUtil.containerInfo(desc.conf)
 
     getSecretVolume(desc).foreach { volume =>
-      logInfo(s"Setting secret name=${volume.getSource.getSecret.getReference.getName} " +
-        s"on file name=${volume.getContainerPath}")
-
+      if (volume.getSource.getSecret.getReference.isInitialized) {
+        logInfo(s"Setting reference secret ${volume.getSource.getSecret.getReference.getName}" +
+          s"on file ${volume.getContainerPath}")
+      } else {
+        logInfo(s"Setting secret on file name=${volume.getContainerPath}")
+      }
       containerInfo.addVolumes(volume)
     }
 
@@ -639,8 +645,6 @@ private[spark] class MesosClusterScheduler(
     if (referenceSecrets.nonEmpty) referenceSecrets else valueSecrets
   }
 
-
-
   private def illegalSecretInput(dest: Seq[String], s: Seq[Secret]): Boolean = {
     if (dest.isEmpty) {  // no destination set (ie not using secrets of this type
       return false