Escape Java options + parse java properties files properly

Author: andrewor14

bin/spark-class

@@ -146,10 +146,41 @@ if $cygwin; then
 fi
 export CLASSPATH
 
+# Properly escape java options, dealing with whitespace, double quotes and backslashes
+# This accepts a string, and returns the escaped list through ESCAPED_JAVA_OPTS
+escape_java_options() {
+  ESCAPED_JAVA_OPTS=() # return value
+  option_buffer=""     # buffer for collecting parts of an option
+  opened_quotes=0      # whether we are expecting a closing double quotes
+  for word in $1; do
+    contains_quote=$(echo "$word" | grep \" | grep -v \\\\\")
+    if [ -n "$contains_quote" ]; then
+      # Flip the bit
+      opened_quotes=$(((opened_quotes + 1) % 2))
+    fi
+    if [[ $opened_quotes == 0 ]]; then
+      ESCAPED_JAVA_OPTS+=("$(echo "$option_buffer $word" | sed "s/^[[:space:]]*//" | sed "s/\([^\\]\)\"/\1/g")")
+      option_buffer=""
+    else
+      option_buffer="$option_buffer $word"
+    fi
+  done
+  # Something is wrong if we ended with open double quotes
+  if [[ $opened_quotes == 1 ]]; then
+    echo "Java options parse error! Expecting closing double quotes." 1>&2
+    exit 1
+  fi
+}
+
+escape_java_options "$JAVA_OPTS"
+for option in "${ESCAPED_JAVA_OPTS[@]}"; do
+  echo "$option"
+done
+
 if [ "$SPARK_PRINT_LAUNCH_COMMAND" == "1" ]; then
   echo -n "Spark Command: " 1>&2
-  echo "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@" 1>&2
+  echo "$RUNNER" -cp "$CLASSPATH" "${ESCAPED_JAVA_OPTS[@]}" "$@" 1>&2
   echo -e "========================================\n" 1>&2
 fi
 
-exec "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@"
+exec "$RUNNER" -cp "$CLASSPATH" "${ESCAPED_JAVA_OPTS[@]}" "$@"

bin/spark-submit

@@ -52,15 +52,14 @@ if [ $DEPLOY_MODE == "client" ]; then
   if [ -f "$PROPERTIES_FILE" ]; then
     echo "Using properties file $PROPERTIES_FILE." 1>&2
 
-    # Parse the value of the given config
-    # This removes the "=" sign, whitespace, and double quotes around the value (if any)
+    # Parse the value of the given config according to the specifications outlined in
+    # http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader)
     parse_config() {
       result=$( \
-        sed "/^#/ d" "$PROPERTIES_FILE" | \
+        sed "/^[#!]/ d" "conf/spark-defaults.conf" | \
         grep "$1" | \
-        sed "s/$1//g" | \
-        sed "s/^[[:space:]]*=//g" | \
-        sed "s/^[[:space:]]*\"\(.*\)\"[[:space:]]*$/\1/g" | \
+        sed "s/$1//" | \
+        sed "s/^[[:space:]]*[:=]\{0,1\}//" | \
         sed "s/^[[:space:]]*\(.*\)[[:space:]]*$/\1/g" \
       )
     }