From 0a1a20c4cffe0bb64e56bac7875f1bfd7e4c64f6 Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Fri, 21 Feb 2014 17:26:55 -0800 Subject: [PATCH 1/7] Add scripts using tachyon tarball - currently won't work as we're waiting on the new bin/tachyon script from tachyon --- make-distribution.sh | 31 +++++++++++++++++++++++++++++++ sbin/start-all.sh | 15 +++++++++++++-- sbin/start-master.sh | 17 +++++++++++++++++ sbin/start-slaves.sh | 21 ++++++++++++++++++++- sbin/stop-master.sh | 2 ++ sbin/stop-slaves.sh | 3 +++ 6 files changed, 86 insertions(+), 3 deletions(-) diff --git a/make-distribution.sh b/make-distribution.sh index e6b5956d1e7e2..ecb5823458842 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -58,6 +58,7 @@ echo "Version is ${VERSION}" # Initialize defaults SPARK_HADOOP_VERSION=1.0.4 SPARK_YARN=false +SPARK_TACHYON=false MAKE_TGZ=false # Parse arguments @@ -70,6 +71,9 @@ while (( "$#" )); do --with-yarn) SPARK_YARN=true ;; + --with-tachyon) + SPARK_TACHYON=true + ;; --tgz) MAKE_TGZ=true ;; @@ -90,6 +94,12 @@ else echo "YARN disabled" fi +if [ "$SPARK_TACHYON" == "true" ]; then + echo "Tachyon Enabled" +else + echo "Tachyon Disabled" +fi + # Build fat JAR export SPARK_HADOOP_VERSION export SPARK_YARN @@ -113,6 +123,27 @@ cp -r "$FWDIR/python" "$DISTDIR" cp -r "$FWDIR/sbin" "$DISTDIR" +# Download and copy in tachyon, if requested +if [ "$SPARK_TACHYON" == "true" ]; then + TACHYON_VERSION="0.4.0" + TACHYON_URL="https://github.com/amplab/tachyon/releases/download/v${TACHYON_VERSION}/tachyon-${TACHYON_VERSION}-bin.tar.gz" + + TMPD=`mktemp -d` + + pushd $TMPD > /dev/null + echo "Fetchting tachyon tgz" + wget "$TACHYON_URL" + + tar xf "tachyon-${TACHYON_VERSION}-bin.tar.gz" + cp "tachyon-${TACHYON_VERSION}/target/tachyon-${TACHYON_VERSION}-jar-with-dependencies.jar" "$DISTDIR/jars" + mkdir "$DISTDIR/sbin/tachyon" + cp -r "tachyon-${TACHYON_VERSION}"/{bin,conf,libexec} "$DISTDIR/sbin/tachyon" + sed -i "s|export TACHYON_JAR=\$TACHYON_HOME/target/\(.*\)|# This is set for spark's make-distribution\n export TACHYON_JAR=\$TACHYON_HOME/../../jars/\1|" "$DISTDIR/sbin/tachyon/libexec/tachyon-config.sh" + + popd > /dev/null + rm -rf $TMPD +fi + if [ "$MAKE_TGZ" == "true" ]; then TARDIR="$FWDIR/spark-$VERSION" cp -r "$DISTDIR" "$TARDIR" diff --git a/sbin/start-all.sh b/sbin/start-all.sh index 2daf49db359df..5c89ab4d86b3a 100755 --- a/sbin/start-all.sh +++ b/sbin/start-all.sh @@ -24,11 +24,22 @@ sbin=`dirname "$0"` sbin=`cd "$sbin"; pwd` +TACHYON_STR="" + +while (( "$#" )); do +case $1 in + --with-tachyon) + TACHYON_STR="--with-tachyon" + ;; + esac +shift +done + # Load the Spark configuration . "$sbin/spark-config.sh" # Start Master -"$sbin"/start-master.sh +"$sbin"/start-master.sh $TACHYON_STR # Start Workers -"$sbin"/start-slaves.sh +"$sbin"/start-slaves.sh $TACHYON_STR diff --git a/sbin/start-master.sh b/sbin/start-master.sh index ec3dfdb4197ec..fec14a659e1e6 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -22,6 +22,17 @@ sbin=`dirname "$0"` sbin=`cd "$sbin"; pwd` +START_TACHYON=false + +while (( "$#" )); do +case $1 in + --with-tachyon) + START_TACHYON=true + ;; + esac +shift +done + . "$sbin/spark-config.sh" if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then @@ -41,3 +52,9 @@ if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then fi "$sbin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT + +if [ "$START_TACHYON" == "true" ]; then + "$sbin"/tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP + "$sbin"/tachyon/bin/tachyon format -s + "$sbin"/tachyon/bin/tachyon-start.sh master +fi diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index fd5cdeb1e6788..c63fef81afd4d 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -20,6 +20,18 @@ sbin=`dirname "$0"` sbin=`cd "$sbin"; pwd` + +START_TACHYON=false + +while (( "$#" )); do +case $1 in + --with-tachyon) + START_TACHYON=true + ;; + esac +shift +done + . "$sbin/spark-config.sh" if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then @@ -37,7 +49,7 @@ fi # Launch the slaves if [ "$SPARK_WORKER_INSTANCES" = "" ]; then - exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT else if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then SPARK_WORKER_WEBUI_PORT=8081 @@ -46,3 +58,10 @@ else "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" $(( $i + 1 )) spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i )) done fi + +if [ "$START_TACHYON" == "true" ]; then + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP + + # set -t so we can call sudo + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 +fi diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index 2adabd426563c..2d31f8de5da63 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -25,3 +25,5 @@ sbin=`cd "$sbin"; pwd` . "$sbin/spark-config.sh" "$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 + +"$sbin"/tachyon/bin/tachyon killAll tachyon.master.Master diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index eb803b4900347..5f7a3f60ce7b2 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -26,6 +26,9 @@ if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then . "${SPARK_CONF_DIR}/spark-env.sh" fi +# do before the below calls as they exec +"$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/tachyon/bin/tachyon killAll tachyon.worker.Worker + if [ "$SPARK_WORKER_INSTANCES" = "" ]; then "$sbin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker 1 else From 4dc9809a66311dc99ab00c379f2e04857e40fee8 Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Fri, 28 Feb 2014 13:35:47 -0800 Subject: [PATCH 2/7] Update to tachyon 0.4.1 - This should now work (tested on ec2) --- make-distribution.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make-distribution.sh b/make-distribution.sh index ecb5823458842..3207194791cb1 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -125,7 +125,7 @@ cp -r "$FWDIR/sbin" "$DISTDIR" # Download and copy in tachyon, if requested if [ "$SPARK_TACHYON" == "true" ]; then - TACHYON_VERSION="0.4.0" + TACHYON_VERSION="0.4.1" TACHYON_URL="https://github.com/amplab/tachyon/releases/download/v${TACHYON_VERSION}/tachyon-${TACHYON_VERSION}-bin.tar.gz" TMPD=`mktemp -d` From 0561574fe71600cfdabb54f6e131c2ca2eca8325 Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Fri, 28 Feb 2014 14:50:12 -0800 Subject: [PATCH 3/7] Copy over web resources so web interface can run --- make-distribution.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/make-distribution.sh b/make-distribution.sh index 3207194791cb1..e7247ec38661d 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -136,8 +136,9 @@ if [ "$SPARK_TACHYON" == "true" ]; then tar xf "tachyon-${TACHYON_VERSION}-bin.tar.gz" cp "tachyon-${TACHYON_VERSION}/target/tachyon-${TACHYON_VERSION}-jar-with-dependencies.jar" "$DISTDIR/jars" - mkdir "$DISTDIR/sbin/tachyon" + mkdir -p "$DISTDIR/sbin/tachyon/src/main/java/tachyon/web" cp -r "tachyon-${TACHYON_VERSION}"/{bin,conf,libexec} "$DISTDIR/sbin/tachyon" + cp -r "tachyon-${TACHYON_VERSION}"/src/main/java/tachyon/web/resources "$DISTDIR/sbin/tachyon/src/main/java/tachyon/web" sed -i "s|export TACHYON_JAR=\$TACHYON_HOME/target/\(.*\)|# This is set for spark's make-distribution\n export TACHYON_JAR=\$TACHYON_HOME/../../jars/\1|" "$DISTDIR/sbin/tachyon/libexec/tachyon-config.sh" popd > /dev/null From 111e8e18ede86861cc620bfc4ddf7d989750cdef Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Thu, 13 Mar 2014 13:35:45 -0700 Subject: [PATCH 4/7] Only try tachyon operations if tachyon script exists --- sbin/start-master.sh | 4 ++++ sbin/stop-slaves.sh | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sbin/start-master.sh b/sbin/start-master.sh index fec14a659e1e6..9049b9fd930db 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -27,6 +27,10 @@ START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) + if [ ! -e "$sbin"/tachyon/bin/tachyon ]; then + echo "Error: --with-tachyon specified, but tachyon not found." + exit -1 + fi START_TACHYON=true ;; esac diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index 5f7a3f60ce7b2..c5e9b3a269dd9 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -27,7 +27,9 @@ if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then fi # do before the below calls as they exec -"$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/tachyon/bin/tachyon killAll tachyon.worker.Worker +if [ -e "$sbin"/tachyon/bin/tachyon ]; then + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/tachyon/bin/tachyon killAll tachyon.worker.Worker +fi if [ "$SPARK_WORKER_INSTANCES" = "" ]; then "$sbin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker 1 From f2f9bc67e129be48fb6c018832e11dc47a3b219f Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Thu, 13 Mar 2014 19:40:40 -0700 Subject: [PATCH 5/7] More checks for tachyon script --- sbin/start-slaves.sh | 4 ++++ sbin/stop-master.sh | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index c63fef81afd4d..71b9860564e7e 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -26,6 +26,10 @@ START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) + if [ ! -e "$sbin"/tachyon/bin/tachyon ]; then + echo "Error: --with-tachyon specified, but tachyon not found." + exit -1 + fi START_TACHYON=true ;; esac diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index 2d31f8de5da63..fb615df1a6eac 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -26,4 +26,6 @@ sbin=`cd "$sbin"; pwd` "$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 -"$sbin"/tachyon/bin/tachyon killAll tachyon.master.Master +if [ -e "$sbin"/tachyon/bin/tachyon ]; then + "$sbin"/tachyon/bin/tachyon killAll tachyon.master.Master +fi From 738ba2320c327da860f3e4bae0307da716146d78 Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Tue, 18 Mar 2014 13:15:00 -0700 Subject: [PATCH 6/7] Move tachyon out of sbin --- make-distribution.sh | 8 ++++---- sbin/start-master.sh | 8 ++++---- sbin/start-slaves.sh | 6 +++--- sbin/stop-master.sh | 4 ++-- sbin/stop-slaves.sh | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/make-distribution.sh b/make-distribution.sh index e7247ec38661d..6bc6819d8da92 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -136,10 +136,10 @@ if [ "$SPARK_TACHYON" == "true" ]; then tar xf "tachyon-${TACHYON_VERSION}-bin.tar.gz" cp "tachyon-${TACHYON_VERSION}/target/tachyon-${TACHYON_VERSION}-jar-with-dependencies.jar" "$DISTDIR/jars" - mkdir -p "$DISTDIR/sbin/tachyon/src/main/java/tachyon/web" - cp -r "tachyon-${TACHYON_VERSION}"/{bin,conf,libexec} "$DISTDIR/sbin/tachyon" - cp -r "tachyon-${TACHYON_VERSION}"/src/main/java/tachyon/web/resources "$DISTDIR/sbin/tachyon/src/main/java/tachyon/web" - sed -i "s|export TACHYON_JAR=\$TACHYON_HOME/target/\(.*\)|# This is set for spark's make-distribution\n export TACHYON_JAR=\$TACHYON_HOME/../../jars/\1|" "$DISTDIR/sbin/tachyon/libexec/tachyon-config.sh" + mkdir -p "$DISTDIR/tachyon/src/main/java/tachyon/web" + cp -r "tachyon-${TACHYON_VERSION}"/{bin,conf,libexec} "$DISTDIR/tachyon" + cp -r "tachyon-${TACHYON_VERSION}"/src/main/java/tachyon/web/resources "$DISTDIR/tachyon/src/main/java/tachyon/web" + sed -i "s|export TACHYON_JAR=\$TACHYON_HOME/target/\(.*\)|# This is set for spark's make-distribution\n export TACHYON_JAR=\$TACHYON_HOME/../../jars/\1|" "$DISTDIR/tachyon/libexec/tachyon-config.sh" popd > /dev/null rm -rf $TMPD diff --git a/sbin/start-master.sh b/sbin/start-master.sh index 9049b9fd930db..03a3428aea9f1 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -27,7 +27,7 @@ START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) - if [ ! -e "$sbin"/tachyon/bin/tachyon ]; then + if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then echo "Error: --with-tachyon specified, but tachyon not found." exit -1 fi @@ -58,7 +58,7 @@ fi "$sbin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT if [ "$START_TACHYON" == "true" ]; then - "$sbin"/tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP - "$sbin"/tachyon/bin/tachyon format -s - "$sbin"/tachyon/bin/tachyon-start.sh master + "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP + "$sbin"/../tachyon/bin/tachyon format -s + "$sbin"/../tachyon/bin/tachyon-start.sh master fi diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 71b9860564e7e..9ded6c25625bd 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -26,7 +26,7 @@ START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) - if [ ! -e "$sbin"/tachyon/bin/tachyon ]; then + if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then echo "Error: --with-tachyon specified, but tachyon not found." exit -1 fi @@ -64,8 +64,8 @@ else fi if [ "$START_TACHYON" == "true" ]; then - "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP # set -t so we can call sudo - SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/../tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 fi diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index fb615df1a6eac..b6bdaa4db373c 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -26,6 +26,6 @@ sbin=`cd "$sbin"; pwd` "$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 -if [ -e "$sbin"/tachyon/bin/tachyon ]; then - "$sbin"/tachyon/bin/tachyon killAll tachyon.master.Master +if [ -e "$sbin"/../tachyon/bin/tachyon ]; then + "$sbin"/../tachyon/bin/tachyon killAll tachyon.master.Master fi diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index c5e9b3a269dd9..6bf393ccd4b09 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -27,8 +27,8 @@ if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then fi # do before the below calls as they exec -if [ -e "$sbin"/tachyon/bin/tachyon ]; then - "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/tachyon/bin/tachyon killAll tachyon.worker.Worker +if [ -e "$sbin"/../tachyon/bin/tachyon ]; then + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon killAll tachyon.worker.Worker fi if [ "$SPARK_WORKER_INSTANCES" = "" ]; then From 2eee15b725cb7425882084052001522f66edc9be Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Tue, 18 Mar 2014 21:26:16 -0700 Subject: [PATCH 7/7] Put back in exec, start tachyon first --- sbin/start-slaves.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 9ded6c25625bd..da641cfe3c6fa 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -51,9 +51,16 @@ if [ "$SPARK_MASTER_IP" = "" ]; then SPARK_MASTER_IP=`hostname` fi +if [ "$START_TACHYON" == "true" ]; then + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP + + # set -t so we can call sudo + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/../tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 +fi + # Launch the slaves if [ "$SPARK_WORKER_INSTANCES" = "" ]; then - "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT + exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT else if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then SPARK_WORKER_WEBUI_PORT=8081 @@ -62,10 +69,3 @@ else "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" $(( $i + 1 )) spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i )) done fi - -if [ "$START_TACHYON" == "true" ]; then - "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP - - # set -t so we can call sudo - SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/../tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 -fi