From f5081465658ee9035a498ac1fb8a532b89e895cc Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sat, 10 May 2014 23:06:12 -0700 Subject: [PATCH 1/3] SPARK-1652: Set driver memory correctly in spark-submit. The previous check didn't account for the fact that the default deploy mode is "client" unless otherwise specified. Also, this sets the more narrowly defined SPARK_DRIVER_MEM instead of setting SPARK_MEM. --- bin/spark-submit | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/spark-submit b/bin/spark-submit index 49bc26252cadf..821454745b6f3 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -35,8 +35,8 @@ while (($#)); do shift done -if [ ! -z $DRIVER_MEMORY ] && [ ! -z $DEPLOY_MODE ] && [ $DEPLOY_MODE = "client" ]; then - export SPARK_MEM=$DRIVER_MEMORY +if [ ! -z $DRIVER_MEMORY ] && [ ! "$DEPLOY_MODE" == "cluster" ]; then + export SPARK_DRIVER_MEMORY=$DRIVER_MEMORY fi $SPARK_HOME/bin/spark-class org.apache.spark.deploy.SparkSubmit "${ORIG_ARGS[@]}" From e788edfe4dfbddf741000265262b642c125e4c6c Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 11 May 2014 14:03:41 -0700 Subject: [PATCH 2/3] Changes based on Aaron's feedback --- bin/spark-submit | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/spark-submit b/bin/spark-submit index 821454745b6f3..4e8bdde56f4f4 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -35,7 +35,9 @@ while (($#)); do shift done -if [ ! -z $DRIVER_MEMORY ] && [ ! "$DEPLOY_MODE" == "cluster" ]; then +DEPLOY_MODE=${DEPLOY_MODE:-"client"} + +if [ ! -z $DRIVER_MEMORY ] && [ ! $DEPLOY_MODE == "cluster" ]; then export SPARK_DRIVER_MEMORY=$DRIVER_MEMORY fi From 430b98f368901ae653d37c2a1d728db1f59c5f05 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 11 May 2014 16:55:59 -0700 Subject: [PATCH 3/3] Feedback from Aaron --- bin/spark-submit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/spark-submit b/bin/spark-submit index 4e8bdde56f4f4..63903b17a2902 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -37,7 +37,7 @@ done DEPLOY_MODE=${DEPLOY_MODE:-"client"} -if [ ! -z $DRIVER_MEMORY ] && [ ! $DEPLOY_MODE == "cluster" ]; then +if [ -n "$DRIVER_MEMORY" ] && [ $DEPLOY_MODE == "client" ]; then export SPARK_DRIVER_MEMORY=$DRIVER_MEMORY fi