From 948395d34b42d7ac8c24bfc3fc128b181d324b84 Mon Sep 17 00:00:00 2001 From: wangfei Date: Fri, 8 Aug 2014 15:35:29 +0800 Subject: [PATCH 1/4] fix spark-sql error when set --driver-java-options --- bin/spark-sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/spark-sql b/bin/spark-sql index 7813ccc36141..22f86ae79a18 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -84,6 +84,11 @@ while (($#)); do CLI_ARGS+=($1) SUBMISSION_ARGS+=($1); shift ;; + + --driver-java-options) + shift; + export SPARK_SUBMIT_OPTS=$1; shift + ;; *) SUBMISSION_ARGS+=($1); shift From e6d79e306194ef4b2a40801866bf497098247caa Mon Sep 17 00:00:00 2001 From: wangfei Date: Fri, 8 Aug 2014 15:45:20 +0800 Subject: [PATCH 2/4] fix start-thriftserver bug when set driver-java-options --- sbin/start-thriftserver.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 603f50ae1324..a1985e975704 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -68,6 +68,11 @@ while (($#)); do THRIFT_SERVER_ARGS+=($1); shift THRIFT_SERVER_ARGS+=($1); shift ;; + + --driver-java-options) + shift; + export SPARK_SUBMIT_OPTS=$1; shift + ;; *) SUBMISSION_ARGS+=($1); shift From 8bd40f2bada3d02eecd35622ce240505b1362516 Mon Sep 17 00:00:00 2001 From: wangfei Date: Tue, 12 Aug 2014 10:22:07 +0800 Subject: [PATCH 3/4] quote variables to fix this problem --- bin/spark-sql | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/bin/spark-sql b/bin/spark-sql index 22f86ae79a18..564f1f419060 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -65,35 +65,30 @@ while (($#)); do case $1 in -d | --define | --database | -f | -h | --hiveconf | --hivevar | -i | -p) ensure_arg_number $# 2 - CLI_ARGS+=($1); shift - CLI_ARGS+=($1); shift + CLI_ARGS+=("$1"); shift + CLI_ARGS+=("$1"); shift ;; -e) ensure_arg_number $# 2 - CLI_ARGS+=($1); shift - CLI_ARGS+=(\"$1\"); shift + CLI_ARGS+=("$1"); shift + CLI_ARGS+=("$1"); shift ;; -s | --silent) - CLI_ARGS+=($1); shift + CLI_ARGS+=("$1"); shift ;; -v | --verbose) # Both SparkSubmit and SparkSQLCLIDriver recognizes -v | --verbose - CLI_ARGS+=($1) - SUBMISSION_ARGS+=($1); shift - ;; - - --driver-java-options) - shift; - export SPARK_SUBMIT_OPTS=$1; shift + CLI_ARGS+=("$1") + SUBMISSION_ARGS+=("$1"); shift ;; *) - SUBMISSION_ARGS+=($1); shift + SUBMISSION_ARGS+=("$1"); shift ;; esac done -eval exec "$FWDIR"/bin/spark-submit --class $CLASS ${SUBMISSION_ARGS[*]} spark-internal ${CLI_ARGS[*]} +exec "$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_ARGS[@]}" spark-internal "${CLI_ARGS[@]}" From 516554df73b1055ef14f6aebabd794cbf6ccdb90 Mon Sep 17 00:00:00 2001 From: wangfei Date: Tue, 12 Aug 2014 10:23:16 +0800 Subject: [PATCH 4/4] quote variables to fix this issue --- sbin/start-thriftserver.sh | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index a1985e975704..2c4452473ccb 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -65,19 +65,14 @@ while (($#)); do case $1 in --hiveconf) ensure_arg_number $# 2 - THRIFT_SERVER_ARGS+=($1); shift - THRIFT_SERVER_ARGS+=($1); shift - ;; - - --driver-java-options) - shift; - export SPARK_SUBMIT_OPTS=$1; shift + THRIFT_SERVER_ARGS+=("$1"); shift + THRIFT_SERVER_ARGS+=("$1"); shift ;; *) - SUBMISSION_ARGS+=($1); shift + SUBMISSION_ARGS+=("$1"); shift ;; esac done -eval exec "$FWDIR"/bin/spark-submit --class $CLASS ${SUBMISSION_ARGS[*]} spark-internal ${THRIFT_SERVER_ARGS[*]} +exec "$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_ARGS[@]}" spark-internal "${THRIFT_SERVER_ARGS[@]}"