Move parsing and escaping logic to utils.sh

This commit also fixes a deadly typo.

bin/spark-class

@@ -30,6 +30,9 @@ FWDIR="$(cd `dirname $0`/..; pwd)"
 # Export this as SPARK_HOME
 export SPARK_HOME="$FWDIR"
 
+# Load utility functions
+. "$SPARK_HOME/bin/utils.sh"
+
 . $FWDIR/bin/load-spark-env.sh
 
 if [ -z "$1" ]; then
@@ -77,7 +80,7 @@ case "$1" in
     'org.apache.spark.deploy.SparkSubmit')
     OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_SUBMIT_OPTS"
     if [ -n "$SPARK_SUBMIT_LIBRARY_PATH" ]; then
-      OUR_JAVA_OPTS="$OUT_JAVA_OPTS -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH"
+      OUR_JAVA_OPTS="$OUR_JAVA_OPTS -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH"
     fi
     OUR_JAVA_MEM=${SPARK_DRIVER_MEMORY:-$DEFAULT_MEM}
     ;;
@@ -103,11 +106,16 @@ fi
 # Set JAVA_OPTS to be able to load native libraries and to set heap size
 JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
 JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"
+
 # Load extra JAVA_OPTS from conf/java-opts, if it exists
 if [ -e "$FWDIR/conf/java-opts" ] ; then
   JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`"
 fi
-export JAVA_OPTS
+
+# Escape JAVA_OPTS properly to handle whitespace, double quotes and backslashes
+# This exports the escaped java options into ESCAPED_JAVA_OPTS
+escape_java_options "$JAVA_OPTS"
+
 # Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in CommandUtils.scala!
 
 TOOLS_DIR="$FWDIR"/tools
@@ -148,37 +156,6 @@ if $cygwin; then
 fi
 export CLASSPATH
 
-# Properly escape java options, dealing with whitespace, double quotes and backslashes
-# This accepts a string, and returns the escaped list through ESCAPED_JAVA_OPTS
-escape_java_options() {
-  ESCAPED_JAVA_OPTS=() # return value
-  option_buffer=""     # buffer for collecting parts of an option
-  opened_quotes=0      # whether we are expecting a closing double quotes
-  for word in $1; do
-    contains_quote=$(echo "$word" | grep \" | grep -v \\\\\")
-    if [ -n "$contains_quote" ]; then
-      # Flip the bit
-      opened_quotes=$(((opened_quotes + 1) % 2))
-    fi
-    if [[ $opened_quotes == 0 ]]; then
-      ESCAPED_JAVA_OPTS+=("$(echo "$option_buffer $word" | sed "s/^[[:space:]]*//" | sed "s/\([^\\]\)\"/\1/g")")
-      option_buffer=""
-    else
-      option_buffer="$option_buffer $word"
-    fi
-  done
-  # Something is wrong if we ended with open double quotes
-  if [[ $opened_quotes == 1 ]]; then
-    echo "Java options parse error! Expecting closing double quotes." 1>&2
-    exit 1
-  fi
-}
-
-escape_java_options "$JAVA_OPTS"
-for option in "${ESCAPED_JAVA_OPTS[@]}"; do
-  echo "$option"
-done
-
 if [ "$SPARK_PRINT_LAUNCH_COMMAND" == "1" ]; then
   echo -n "Spark Command: " 1>&2
   echo "$RUNNER" -cp "$CLASSPATH" "${ESCAPED_JAVA_OPTS[@]}" "$@" 1>&2

bin/spark-submit

@@ -20,6 +20,14 @@
 export SPARK_HOME="$(cd `dirname $0`/..; pwd)"
 ORIG_ARGS=("$@")
 
+# Load utility functions
+. "$SPARK_HOME/bin/utils.sh"
+
+# For client mode, the driver will be launched in the JVM that launches
+# SparkSubmit, so we need to handle the class paths, java options, and
+# memory pre-emptively in bash. Otherwise, it will be too late by the
+# time the JVM has started.
+
 while (($#)); do
   if [ "$1" = "--deploy-mode" ]; then
     DEPLOY_MODE=$2
@@ -40,34 +48,14 @@ done
 DEPLOY_MODE=${DEPLOY_MODE:-"client"}
 PROPERTIES_FILE=${PROPERTIES_FILE:-"$SPARK_HOME/conf/spark-defaults.conf"}
 
-# For client mode, the driver will be launched in the JVM that launches
-# SparkSubmit, so we need to handle the class paths, java options, and
-# memory pre-emptively in bash. Otherwise, it will be too late by the
-# time the JVM has started.
-
 if [ $DEPLOY_MODE == "client" ]; then
-  # We parse the default properties file here, assuming each line is
-  # a key value pair delimited either by white space or "=" sign. All
-  # spark.driver.* configs must be processed now before it's too late.
+  # Parse the default properties file here for spark.driver.* configs
   if [ -f "$PROPERTIES_FILE" ]; then
     echo "Using properties file $PROPERTIES_FILE." 1>&2
-
-    # Parse the value of the given config according to the specifications outlined in
-    # http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader)
-    parse_config() {
-      result=$( \
-        sed "/^[#!]/ d" "conf/spark-defaults.conf" | \
-        grep "$1" | \
-        sed "s/$1//" | \
-        sed "s/^[[:space:]]*[:=]\{0,1\}//" | \
-        sed "s/^[[:space:]]*\(.*\)[[:space:]]*$/\1/g" \
-      )
-    }
-    parse_config "spark.driver.memory";            DRIVER_MEMORY_CONF="$result"
-    parse_config "spark.driver.extraJavaOptions";  DRIVER_EXTRA_JAVA_OPTS="$result"
-    parse_config "spark.driver.extraClassPath";    DRIVER_EXTRA_CLASSPATH="$result"
-    parse_config "spark.driver.extraLibraryPath";  DRIVER_EXTRA_LIBRARY_PATH="$result"
-
+    parse_java_property "spark.driver.memory";            DRIVER_MEMORY_CONF="$JAVA_PROPERTY_VALUE"
+    parse_java_property "spark.driver.extraJavaOptions";  DRIVER_EXTRA_JAVA_OPTS="$JAVA_PROPERTY_VALUE"
+    parse_java_property "spark.driver.extraClassPath";    DRIVER_EXTRA_CLASSPATH="$JAVA_PROPERTY_VALUE"
+    parse_java_property "spark.driver.extraLibraryPath";  DRIVER_EXTRA_LIBRARY_PATH="$JAVA_PROPERTY_VALUE"
     if [ -n "$DRIVER_EXTRA_JAVA_OPTS" ]; then
       export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS $DRIVER_EXTRA_JAVA_OPTS"
     fi
@@ -80,7 +68,6 @@ if [ $DEPLOY_MODE == "client" ]; then
   else
     echo "Warning: properties file $PROPERTIES_FILE does not exist!" 1>&2
   fi
-
   # Favor command line memory over config memory
   DRIVER_MEMORY=${DRIVER_MEMORY:-"$DRIVER_MEMORY_CONF"}
   if [ -n "$DRIVER_MEMORY" ]; then

bin/utils.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Parse the value of a config from a java properties file according to the specifications in
+# http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader).
+# This accepts the name of the config and returns the value through JAVA_PROPERTY_VALUE.
+parse_java_property() {
+  JAVA_PROPERTY_VALUE=$( \
+    sed "/^[#!]/ d" "conf/spark-defaults.conf" | \
+    grep "$1" | \
+    sed "s/$1//" | \
+    sed "s/^[[:space:]]*[:=]\{0,1\}//" | \
+    sed "s/^[[:space:]]*\(.*\)[[:space:]]*$/\1/g" \
+  )
+  export JAVA_PROPERTY_VALUE
+}
+
+# Properly escape java options, dealing with whitespace, double quotes and backslashes
+# This accepts a string, and returns the escaped list through ESCAPED_JAVA_OPTS.
+escape_java_options() {
+  ESCAPED_JAVA_OPTS=() # return value
+  option_buffer=""     # buffer for collecting parts of an option
+  opened_quotes=0      # whether we are expecting a closing double quotes
+  for word in $1; do
+    contains_quote=$(echo "$word" | grep \" | grep -v \\\\\")
+    if [ -n "$contains_quote" ]; then
+      # Flip the bit
+      opened_quotes=$(((opened_quotes + 1) % 2))
+    fi
+    if [[ $opened_quotes == 0 ]]; then
+      ESCAPED_JAVA_OPTS+=("$(echo "$option_buffer $word" | sed "s/^[[:space:]]*//" | sed "s/\([^\\]\)\"/\1/g")")
+      option_buffer=""
+    else
+      option_buffer="$option_buffer $word"
+    fi
+  done
+  # Something is wrong if we ended with open double quotes
+  if [[ $opened_quotes == 1 ]]; then
+    echo "Java options parse error! Expecting closing double quotes." 1>&2
+    exit 1
+  fi
+  export ESCAPED_JAVA_OPTS
+}
+