Parse spark.driver.extra* in bash

Author: andrewor14

bin/spark-submit

@@ -25,6 +25,8 @@ while (($#)); do
     DEPLOY_MODE=$2
   elif [ "$1" = "--driver-memory" ]; then
     DRIVER_MEMORY=$2
+  elif [ "$1" = "--properties-file" ]; then
+    PROPERTIES_FILE=$2
   elif [ "$1" = "--driver-library-path" ]; then
     export SPARK_SUBMIT_LIBRARY_PATH=$2
   elif [ "$1" = "--driver-class-path" ]; then
@@ -36,9 +38,35 @@ while (($#)); do
 done
 
 DEPLOY_MODE=${DEPLOY_MODE:-"client"}
+PROPERTIES_FILE=${PROPERTIES_FILE:-"$SPARK_HOME/conf/spark-defaults.conf"}
 
-if [ -n "$DRIVER_MEMORY" ] && [ $DEPLOY_MODE == "client" ]; then
-  export SPARK_DRIVER_MEMORY=$DRIVER_MEMORY
+# For client mode, the driver will be launched in the JVM that launches
+# SparkSubmit, so we need to handle the class paths, java options, and
+# memory pre-emptively in bash. Otherwise, it will be too late by the
+# time the JVM has started.
+
+if [ $DEPLOY_MODE == "client" ]; then
+  if [ -n "$DRIVER_MEMORY" ]; then
+    export SPARK_DRIVER_MEMORY=$DRIVER_MEMORY
+  fi
+  # We parse the default properties file here, assuming each line is
+  # a key value pair delimited either by white space or "=" sign. All
+  # spark.driver.* configs must be processed now before it's too late.
+  if [ -f "$PROPERTIES_FILE" ]; then
+    DRIVER_EXTRA_JAVA_OPTS="spark.driver.extraJavaOptions"
+    DRIVER_EXTRA_CLASSPATH="spark.driver.extraClassPath"
+    DRIVER_EXTRA_LIBRARY_PATH="spark.driver.extraLibraryPath"
+    # Remove "=" sign and double quotes around the value, if any
+    DRIVER_EXTRA_JAVA_OPTS=$(sed "/^#/ d" "$PROPERTIES_FILE" | grep "$DRIVER_EXTRA_JAVA_OPTS" | \
+      sed "s/$DRIVER_EXTRA_JAVA_OPTS//g" | sed "s/^=//g" | sed "s/^\"\(.*\)\"$/\1/g")
+    DRIVER_EXTRA_CLASSPATH=$(sed "/^#/ d" "$PROPERTIES_FILE" | grep "$DRIVER_EXTRA_CLASSPATH" | \
+      sed "s/$DRIVER_EXTRA_CLASSPATH//g" | sed "s/^=//g" | sed "s/^\"\(.*\)\"$/\1/g")
+    DRIVER_EXTRA_LIBRARY_PATH=$(sed "/^#/ d" "$PROPERTIES_FILE" | grep "$DRIVER_EXTRA_LIBRARY_PATH" | \
+      sed "s/$DRIVER_EXTRA_LIBRARY_PATH//g" | sed "s/^=//g" | sed "s/^\"\(.*\)\"$/\1/g")
+    export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS $DRIVER_EXTRA_JAVA_OPTS"
+    export SPARK_SUBMIT_CLASSPATH="$SPARK_SUBMIT_CLASSPATH:$DRIVER_EXTRA_CLASSPATH"
+    export SPARK_SUBMIT_LIBRARY_PATH="$SPARK_SUBMIT_LIBRARY_PATH:$DRIVER_EXTRA_LIBRARY_PATH"
+  fi
 fi
 
 exec $SPARK_HOME/bin/spark-class org.apache.spark.deploy.SparkSubmit "${ORIG_ARGS[@]}"

bin/spark-submit.new

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+SPARK_HOME="$(cd `dirname $0`/..; pwd)"
+ORIG_ARGS=("$@")
+
+while (($#)); do
+  if [ "$1" = "--deploy-mode" ]; then
+    DEPLOY_MODE=$2
+  elif [ "$1" = "--driver-memory" ]; then
+    DRIVER_MEMORY=$2
+  elif [ "$1" = "--driver-library-path" ]; then
+    SPARK_SUBMIT_LIBRARY_PATH=$2
+  elif [ "$1" = "--driver-class-path" ]; then
+    SPARK_SUBMIT_CLASSPATH=$2
+  elif [ "$1" = "--driver-java-options" ]; then
+    SPARK_SUBMIT_OPTS=$2
+  elif [ "$1" = "--properties-file" ]; then
+    PROPERTIES_FILE=$2
+  fi
+  shift
+done
+
+DEPLOY_MODE=${DEPLOY_MODE:-"client"}
+
+if [ -n "$DRIVER_MEMORY" ] && [ $DEPLOY_MODE == "client" ]; then
+  SPARK_DRIVER_MEMORY=$DRIVER_MEMORY
+fi
+
+PROPERTIES_FILE=${PROPERTIES_FILE:-"$SPARK_HOME/conf/spark-defaults.conf"}
+if [ -f $PROPERTIES_FILE ]; then
+  DRIVER_EXTRA_JAVA_OPTIONS="spark.driver.extraJavaOptions"
+  DRIVER_EXTRA_CLASSPATH="spark.driver.extraClassPath"
+  DRIVER_EXTRA_LIBRARY_PATH="spark.driver.extraLibraryPath"
+  DRIVER_EXTRA_JAVA_OPTIONS=$(sed "/^#/ d" "$PROPERTIES_FILE" | grep "$DRIVER_EXTRA_JAVA_OPTIONS" | sed "s/$DRIVER_EXTRA_JAVA_OPTIONS//g" | sed "s/^=//g")
+  DRIVER_EXTRA_CLASSPATH=$(sed "/^#/ d" "$PROPERTIES_FILE" | grep "$DRIVER_EXTRA_CLASSPATH" | sed "s/$DRIVER_EXTRA_CLASSPATH//g" | sed "s/^=//g")
+  DRIVER_EXTRA_LIBRARY_PATH=$(sed "/^#/ d" "$PROPERTIES_FILE" | grep "$DRIVER_EXTRA_LIBRARY_PATH" | sed "s/$DRIVER_EXTRA_LIBRARY_PATH//g" | sed "s/^=//g")
+fi
+
+echo "DEPLOY_MODE = $DEPLOY_MODE"
+echo "DRIVER_MEMORY = $DRIVER_MEMORY"
+echo "SPARK_DRIVER_MEMORY = $SPARK_DRIVER_MEMORY"
+echo "SPARK_SUBMIT_LIBRARY_PATH = $SPARK_SUBMIT_LIBRARY_PATH"
+echo "SPARK_SUBMIT_CLASSPATH = $SPARK_SUBMIT_CLASSPATH"
+echo "SPARK_SUBMIT_OPTS = $SPARK_SUBMIT_OPTS"
+echo "DRIVER_EXTRA_JAVA_OPTIONS = $DRIVER_EXTRA_JAVA_OPTIONS"
+echo "DRIVER_EXTRA_CLASSPATH = $DRIVER_EXTRA_CLASSPATH"
+echo "DRIVER_EXTRA_LIBRARY_PATH = $DRIVER_EXTRA_LIBRARY_PATH"
+
+echo "exec $SPARK_HOME/bin/spark-class org.apache.spark.deploy.SparkSubmit "${ORIG_ARGS[@]}""
+