From 0647c6b71db398fa10b8d151ddceaa23769e4d85 Mon Sep 17 00:00:00 2001 From: Chris Heller Date: Mon, 29 Dec 2014 14:28:53 -0500 Subject: [PATCH 1/2] Allow spark-daemon.sh to support foreground operation --- sbin/spark-daemon.sh | 80 ++++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index 5e812a1d91c6b..d92836cf963fa 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -29,7 +29,7 @@ # SPARK_NICENESS The scheduling priority for daemons. Defaults to 0. ## -usage="Usage: spark-daemon.sh [--config ] (start|stop) " +usage="Usage: spark-daemon.sh [--foreground] [--config ] (start|stop) " # if no args specified, show usage if [ $# -le 1 ]; then @@ -44,6 +44,14 @@ sbin="`cd "$sbin"; pwd`" # get arguments +RUN_IN_FOREGROUND=0 +# Check if --foreground is passed as an argument. It is an optional parameter. +if [ "$1" == "--foreground" ] +then + shift + RUN_IN_FOREGROUND="1" +fi + # Check if --config is passed as an argument. It is an optional parameter. # Exit if the argument is not a directory. @@ -95,17 +103,19 @@ fi export SPARK_PRINT_LAUNCH_COMMAND="1" -# get log directory -if [ "$SPARK_LOG_DIR" = "" ]; then - export SPARK_LOG_DIR="$SPARK_HOME/logs" -fi -mkdir -p "$SPARK_LOG_DIR" -touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 -TEST_LOG_DIR=$? -if [ "${TEST_LOG_DIR}" = "0" ]; then - rm -f "$SPARK_LOG_DIR"/.spark_test -else - chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR" +if [ "$RUN_IN_FOREGROUND" = "0" ]; then + # get log directory + if [ "$SPARK_LOG_DIR" = "" ]; then + export SPARK_LOG_DIR="$SPARK_HOME/logs" + fi + mkdir -p "$SPARK_LOG_DIR" + touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 + TEST_LOG_DIR=$? + if [ "${TEST_LOG_DIR}" = "0" ]; then + rm -f "$SPARK_LOG_DIR"/.spark_test + else + chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR" + fi fi if [ "$SPARK_PID_DIR" = "" ]; then @@ -141,24 +151,36 @@ case $option in rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $SPARK_MASTER/ "$SPARK_HOME" fi - spark_rotate_log "$log" - echo "starting $command, logging to $log" - if [ $option == spark-submit ]; then - source "$SPARK_HOME"/bin/utils.sh - gatherSparkSubmitOpts "$@" - nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \ - "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" >> "$log" 2>&1 < /dev/null & + if [ "$RUN_IN_FOREGROUND" = "0" ]; then + spark_rotate_log "$log" + echo starting $command, logging to $log + if [ $option == spark-submit ]; then + source "$SPARK_HOME"/bin/utils.sh + gatherSparkSubmitOpts "$@" + nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \ + "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" >> "$log" 2>&1 < /dev/null & + else + nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & + fi + newpid=$! + echo $newpid > $pid + sleep 2 + # Check if the process has died; in that case we'll tail the log so the user can see + if ! kill -0 $newpid >/dev/null 2>&1; then + echo "failed to launch $command:" + tail -2 "$log" | sed 's/^/ /' + echo "full log in $log" + fi else - nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & - fi - newpid=$! - echo $newpid > $pid - sleep 2 - # Check if the process has died; in that case we'll tail the log so the user can see - if [[ ! $(ps -p "$newpid" -o args=) =~ $command ]]; then - echo "failed to launch $command:" - tail -2 "$log" | sed 's/^/ /' - echo "full log in $log" + echo starting $command, logging to stdout + if [ $option == spark-submit ]; then + source "$SPARK_HOME"/bin/utils.sh + gatherSparkSubmitOpts "$@" + nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \ + "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" 2>&1 < /dev/null + else + nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" 2>&1 < /dev/null + fi fi ;; From 0995ee1e590d9372f9064bf57558b5424fcd8d64 Mon Sep 17 00:00:00 2001 From: Chris Heller Date: Mon, 23 Feb 2015 22:59:59 -0500 Subject: [PATCH 2/2] Adjust indentation on new code --- sbin/spark-daemon.sh | 86 ++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index d92836cf963fa..a07b428ddebed 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -48,8 +48,8 @@ RUN_IN_FOREGROUND=0 # Check if --foreground is passed as an argument. It is an optional parameter. if [ "$1" == "--foreground" ] then - shift - RUN_IN_FOREGROUND="1" + shift + RUN_IN_FOREGROUND="1" fi # Check if --config is passed as an argument. It is an optional parameter. @@ -104,18 +104,18 @@ fi export SPARK_PRINT_LAUNCH_COMMAND="1" if [ "$RUN_IN_FOREGROUND" = "0" ]; then - # get log directory - if [ "$SPARK_LOG_DIR" = "" ]; then - export SPARK_LOG_DIR="$SPARK_HOME/logs" - fi - mkdir -p "$SPARK_LOG_DIR" - touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 - TEST_LOG_DIR=$? - if [ "${TEST_LOG_DIR}" = "0" ]; then - rm -f "$SPARK_LOG_DIR"/.spark_test - else - chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR" - fi + # get log directory + if [ "$SPARK_LOG_DIR" = "" ]; then + export SPARK_LOG_DIR="$SPARK_HOME/logs" + fi + mkdir -p "$SPARK_LOG_DIR" + touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 + TEST_LOG_DIR=$? + if [ "${TEST_LOG_DIR}" = "0" ]; then + rm -f "$SPARK_LOG_DIR"/.spark_test + else + chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR" + fi fi if [ "$SPARK_PID_DIR" = "" ]; then @@ -152,35 +152,35 @@ case $option in fi if [ "$RUN_IN_FOREGROUND" = "0" ]; then - spark_rotate_log "$log" - echo starting $command, logging to $log - if [ $option == spark-submit ]; then - source "$SPARK_HOME"/bin/utils.sh - gatherSparkSubmitOpts "$@" - nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \ - "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" >> "$log" 2>&1 < /dev/null & - else - nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & - fi - newpid=$! - echo $newpid > $pid - sleep 2 - # Check if the process has died; in that case we'll tail the log so the user can see - if ! kill -0 $newpid >/dev/null 2>&1; then - echo "failed to launch $command:" - tail -2 "$log" | sed 's/^/ /' - echo "full log in $log" - fi - else - echo starting $command, logging to stdout - if [ $option == spark-submit ]; then - source "$SPARK_HOME"/bin/utils.sh - gatherSparkSubmitOpts "$@" - nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \ - "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" 2>&1 < /dev/null - else - nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" 2>&1 < /dev/null - fi + spark_rotate_log "$log" + echo starting $command, logging to $log + if [ $option == spark-submit ]; then + source "$SPARK_HOME"/bin/utils.sh + gatherSparkSubmitOpts "$@" + nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \ + "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" >> "$log" 2>&1 < /dev/null & + else + nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & + fi + newpid=$! + echo $newpid > $pid + sleep 2 + # Check if the process has died; in that case we'll tail the log so the user can see + if ! kill -0 $newpid >/dev/null 2>&1; then + echo "failed to launch $command:" + tail -2 "$log" | sed 's/^/ /' + echo "full log in $log" + fi + else # run in foreground + echo starting $command, logging to stdout + if [ $option == spark-submit ]; then + source "$SPARK_HOME"/bin/utils.sh + gatherSparkSubmitOpts "$@" + nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \ + "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" 2>&1 < /dev/null + else + nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" 2>&1 < /dev/null + fi fi ;;