Review feedback and import clean-ups

Author: pwendell

docs/configuration.md

@@ -650,9 +650,9 @@ Apart from these, the following properties are also available, and may be useful
   <td>spark.executor.extraJavaOptions</td>
   <td>(none)</td>
   <td>
-    A string of extra JVM options to pass to executors. For instance, GC settings or custom
-    paths for native code. Note that it is illegal to set Spark properties or heap size 
-    settings with this option.
+    A string of extra JVM options to pass to executors. For instance, GC settings or other
+    logging. Note that it is illegal to set Spark properties or heap size settings with this 
+    option.
   </td>
 </tr>
 

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala

@@ -17,36 +17,25 @@
 
 package org.apache.spark.deploy.yarn
 
-import java.net.{InetAddress, UnknownHostException, URI}
-import java.nio.ByteBuffer
+import java.net.{InetAddress, URI, UnknownHostException}
 
 import scala.collection.JavaConversions._
-import scala.collection.mutable.HashMap
-import scala.collection.mutable.Map
+import scala.collection.mutable.{HashMap, Map}
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs._
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.DataOutputBuffer
+import org.apache.hadoop.fs.permission.FsPermission
 import org.apache.hadoop.mapred.Master
 import org.apache.hadoop.mapreduce.MRJobConfig
-import org.apache.hadoop.net.NetUtils
 import org.apache.hadoop.security.UserGroupInformation
 import org.apache.hadoop.util.StringUtils
 import org.apache.hadoop.yarn.api._
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
 import org.apache.hadoop.yarn.api.protocolrecords._
 import org.apache.hadoop.yarn.api.records._
 import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.ipc.YarnRPC
-import org.apache.hadoop.yarn.util.{Records, Apps}
-
+import org.apache.hadoop.yarn.util.{Apps, Records}
 import org.apache.spark.{Logging, SparkConf}
-import org.apache.spark.util.Utils
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.deploy.ExecutorLauncher
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
-
 
 /**
  * The entry point (starting in Client#main() and Client#run()) for launching Spark on YARN. The
@@ -355,8 +344,8 @@ trait ClientBase extends Logging {
         JAVA_OPTS += s"-D$k=$v"
       }
       // TODO: honor driver classpath here: sys.props.get("spark.driver.classPath")
-      sys.props.get("spark.driver.javaOpts").map(opts => JAVA_OPTS += opts)
-      sys.props.get("spark.driver.libraryPath").map(p => JAVA_OPTS + s"-Djava.library.path=$p")
+      sys.props.get("spark.driver.javaOpts").foreach(opts => JAVA_OPTS += opts)
+      sys.props.get("spark.driver.libraryPath").foreach(p => JAVA_OPTS += s"-Djava.library.path=$p")
     }
 
     if (!localResources.contains(ClientBase.LOG4J_PROP)) {

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala

@@ -18,26 +18,18 @@
 package org.apache.spark.deploy.yarn
 
 import java.net.URI
-import java.nio.ByteBuffer
-import java.security.PrivilegedExceptionAction
 
 import scala.collection.JavaConversions._
 import scala.collection.mutable.HashMap
 
-import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.DataOutputBuffer
-import org.apache.hadoop.net.NetUtils
-import org.apache.hadoop.security.UserGroupInformation
 import org.apache.hadoop.yarn.api._
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
 import org.apache.hadoop.yarn.api.records._
-import org.apache.hadoop.yarn.api.protocolrecords._
-import org.apache.hadoop.yarn.util.{Apps, ConverterUtils, Records}
-
-import org.apache.spark.{SparkConf, Logging}
 import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.util.{Apps, ConverterUtils, Records}
 
+import org.apache.spark.{Logging, SparkConf}
 
 trait ExecutorRunnableUtil extends Logging {
 
@@ -58,9 +50,10 @@ trait ExecutorRunnableUtil extends Logging {
     val executorMemoryString = executorMemory + "m"
     JAVA_OPTS += "-Xms" + executorMemoryString + " -Xmx" + executorMemoryString + " "
 
-    // Set extra Java options for the executor
-    val executorOpts = sys.props.find(_._1.contains("spark.executor.extraJavaOptions"))
-    JAVA_OPTS += executorOpts
+    // Set extra Java options for the executor, if defined
+    sys.props.get("spark.executor.extraJavaOptions").foreach { opts =>
+      JAVA_OPTS += opts
+    }
 
     JAVA_OPTS += " -Djava.io.tmpdir=" +
       new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR) + " "