From 4b1e6f58bb91cb8e882a73537276115d977955f1 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 19 Sep 2017 22:22:02 -0700 Subject: [PATCH 1/5] Check the java version --- dev/create-release/release-build.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index ee2407a4ddf0e..1ddb4ee60e33f 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -95,6 +95,23 @@ if [ -z "$SPARK_VERSION" ]; then | grep -v INFO | grep -v WARNING | grep -v Download) fi +# Verify we have the right java version set +java_version=$("${JAVA_HOME}"/bin/javac -version 2>&1 | cut -d " " -f 2) + +if [[ ! $SPARK_VERSION < "v2.2" ]]; then + if [[ $java_version < "1.8." ]]; then + echo "Java version $java_version is less than required 1.8 for 2.2+" + echo "Please set JAVA_HOME correctly." + exit 1 + fi +else + if [[ $java_version > "1.7." ]]; then + echo "Java version $java_version is higher than required 1.7 for pre-2.2" + echo "Please set JAVA_HOME correctly." + exit 1 + fi +fi + if [ -z "$SPARK_PACKAGE_VERSION" ]; then SPARK_PACKAGE_VERSION="${SPARK_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" fi From c9583359174ad43b97ff438cb67159a570137f8b Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 19 Sep 2017 22:22:14 -0700 Subject: [PATCH 2/5] Drop the leading v for publish-release --- dev/create-release/release-build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 1ddb4ee60e33f..fc67c89939650 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -116,6 +116,13 @@ if [ -z "$SPARK_PACKAGE_VERSION" ]; then SPARK_PACKAGE_VERSION="${SPARK_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" fi +# Drop the v from the start of the version string in publish-release mode for nexus. +if [[ "$@" == *"publish-release"* ]]; then + # shellcheck disable=SC2001 + VERSION_WITH_RC=$(echo "$RELEASE_TAG" | sed -e "s/v//") + SPARK_VERSION=$VERSION_WITH_RC +fi + DEST_DIR_NAME="spark-$SPARK_PACKAGE_VERSION" function LFTP { From ebeab002012794a55f63b8bef5e97d1c7659ecd0 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 19 Sep 2017 22:25:02 -0700 Subject: [PATCH 3/5] We should do it for both nexus actions --- dev/create-release/release-build.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index fc67c89939650..0d6dc9aab8742 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -116,13 +116,6 @@ if [ -z "$SPARK_PACKAGE_VERSION" ]; then SPARK_PACKAGE_VERSION="${SPARK_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" fi -# Drop the v from the start of the version string in publish-release mode for nexus. -if [[ "$@" == *"publish-release"* ]]; then - # shellcheck disable=SC2001 - VERSION_WITH_RC=$(echo "$RELEASE_TAG" | sed -e "s/v//") - SPARK_VERSION=$VERSION_WITH_RC -fi - DEST_DIR_NAME="spark-$SPARK_PACKAGE_VERSION" function LFTP { @@ -308,6 +301,11 @@ if [[ "$1" == "docs" ]]; then fi if [[ "$1" == "publish-snapshot" ]]; then + # Drop the v from the start of the version string for nexus. + # shellcheck disable=SC2001 + VERSION_WITH_RC=$(echo "$RELEASE_TAG" | sed -e "s/v//") + SPARK_VERSION=$VERSION_WITH_RC + cd spark # Publish Spark to Maven release repo echo "Deploying Spark SNAPSHOT at '$GIT_REF' ($git_hash)" @@ -342,6 +340,11 @@ if [[ "$1" == "publish-snapshot" ]]; then fi if [[ "$1" == "publish-release" ]]; then + # Drop the v from the start of the version string for nexus. + # shellcheck disable=SC2001 + VERSION_WITH_RC=$(echo "$RELEASE_TAG" | sed -e "s/v//") + SPARK_VERSION=$VERSION_WITH_RC + cd spark # Publish Spark to Maven release repo echo "Publishing Spark checkout at '$GIT_REF' ($git_hash)" From aa4cbf69b080435bc836dc98dddd20307fba6588 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 20 Sep 2017 09:44:06 -0700 Subject: [PATCH 4/5] Use package version for publish-release, auto matically switch to JAVA_7_HOME if its available --- dev/create-release/release-build.sh | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 0d6dc9aab8742..4f0922f971d8d 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -31,8 +31,8 @@ Top level targets are All other inputs are environment variables GIT_REF - Release tag or commit to build from -SPARK_VERSION - Release identifier used when publishing -SPARK_PACKAGE_VERSION - Release identifier in top level package directory +SPARK_VERSION - Version of Spark being built (e.g. 2.1.2) +SPARK_PACKAGE_VERSION - Release identifier in top level package directory (e.g. 2.1.2-rc1) REMOTE_PARENT_DIR - Parent in which to create doc or release builds. REMOTE_PARENT_MAX_LENGTH - If set, parent directory will be cleaned to only have this number of subdirectories (by deleting old ones). WARNING: This deletes data. @@ -98,7 +98,7 @@ fi # Verify we have the right java version set java_version=$("${JAVA_HOME}"/bin/javac -version 2>&1 | cut -d " " -f 2) -if [[ ! $SPARK_VERSION < "v2.2" ]]; then +if [[ ! $SPARK_VERSION < "2.2." ]]; then if [[ $java_version < "1.8." ]]; then echo "Java version $java_version is less than required 1.8 for 2.2+" echo "Please set JAVA_HOME correctly." @@ -106,12 +106,17 @@ if [[ ! $SPARK_VERSION < "v2.2" ]]; then fi else if [[ $java_version > "1.7." ]]; then - echo "Java version $java_version is higher than required 1.7 for pre-2.2" - echo "Please set JAVA_HOME correctly." - exit 1 + if [ -z "$JAVA_7_HOME" ]; then + echo "Java version $java_version is higher than required 1.7 for pre-2.2" + echo "Please set JAVA_HOME correctly." + exit 1 + else + JAVA_HOME="$JAVA_7_HOME" + fi fi fi + if [ -z "$SPARK_PACKAGE_VERSION" ]; then SPARK_PACKAGE_VERSION="${SPARK_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" fi @@ -301,11 +306,6 @@ if [[ "$1" == "docs" ]]; then fi if [[ "$1" == "publish-snapshot" ]]; then - # Drop the v from the start of the version string for nexus. - # shellcheck disable=SC2001 - VERSION_WITH_RC=$(echo "$RELEASE_TAG" | sed -e "s/v//") - SPARK_VERSION=$VERSION_WITH_RC - cd spark # Publish Spark to Maven release repo echo "Deploying Spark SNAPSHOT at '$GIT_REF' ($git_hash)" @@ -340,10 +340,7 @@ if [[ "$1" == "publish-snapshot" ]]; then fi if [[ "$1" == "publish-release" ]]; then - # Drop the v from the start of the version string for nexus. - # shellcheck disable=SC2001 - VERSION_WITH_RC=$(echo "$RELEASE_TAG" | sed -e "s/v//") - SPARK_VERSION=$VERSION_WITH_RC + SPARK_VERSION=$SPARK_PACKAGE_VERSION cd spark # Publish Spark to Maven release repo From 94d7659e95fc8967336e9e86f8d7e0a3d021ed43 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 22 Sep 2017 00:13:06 -0700 Subject: [PATCH 5/5] Require JAVA_HOME be set --- dev/create-release/release-build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 4f0922f971d8d..97c3d8b3a0026 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -96,6 +96,11 @@ if [ -z "$SPARK_VERSION" ]; then fi # Verify we have the right java version set +if [ -z "$JAVA_HOME" ]; then + echo "Please set JAVA_HOME." + exit 1 +fi + java_version=$("${JAVA_HOME}"/bin/javac -version 2>&1 | cut -d " " -f 2) if [[ ! $SPARK_VERSION < "2.2." ]]; then