Skip to content

[SPARK-52584][BUILD] Make build script to support preview releases in finalize step #51291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SPARK-52584][BUILD] Make build script to support preview releases in finalize step #51291

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
210 changes: 147 additions & 63 deletions dev/create-release/release-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,12 @@ if [[ "$1" == "finalize" ]]; then
import re

release_version = "${RELEASE_VERSION}"
newline = f' <li><a href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark {release_version}</a></li>'
is_preview = bool(re.search(r'-preview\d*$', release_version))
base_version = re.sub(r'-preview\d*$', '', release_version)

stable_newline = f' <li><a href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark {release_version}</a></li>'
preview_newline = f' <li><a href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark {release_version} preview</a></li>'

inserted = False

def parse_version(v):
Expand All @@ -183,29 +188,63 @@ with open("documentation.md") as f:
lines = f.readlines()

with open("documentation.md", "w") as f:
for line in lines:
match = re.search(r'docs/(\d+\.\d+\.\d+)/', line)
if not inserted and match:
existing_version = match.group(1)
if vercmp(release_version, existing_version) >= 0:
f.write(newline + "\n")
if is_preview:
in_preview_section = False
for i, line in enumerate(lines):
if '<p>Documentation for preview releases:</p>' in line:
in_preview_section = True
f.write(line)
continue

if in_preview_section and re.search(r'docs/\d+\.\d+\.\d+-preview\d*/', line):
existing_version = re.search(r'docs/(\d+\.\d+\.\d+-preview\d*)/', line).group(1)

if existing_version == release_version:
inserted = True # Already exists, don't add
elif not inserted:
base_existing = re.sub(r'-preview\d*$', '', existing_version)
preview_num_existing = int(re.search(r'preview(\d*)', existing_version).group(1) or "0")
preview_num_new = int(re.search(r'preview(\d*)', release_version).group(1) or "0")

if (vercmp(base_version, base_existing) > 0) or \
(vercmp(base_version, base_existing) == 0 and preview_num_new >= preview_num_existing):
f.write(preview_newline + "\n")
inserted = True

f.write(line)
continue

if in_preview_section and "</ul>" in line and not inserted:
f.write(preview_newline + "\n")
inserted = True
f.write(line)
if not inserted:
f.write(newline + "\n")
f.write(line)
else:
for line in lines:
match = re.search(r'docs/(\d+\.\d+\.\d+)/', line)
if not inserted and match:
existing_version = match.group(1)
if vercmp(release_version, existing_version) >= 0:
f.write(stable_newline + "\n")
inserted = True
f.write(line)
if not inserted:
f.write(stable_newline + "\n")
EOF

echo "Edited documentation.md"

# 2. Add download link to js/downloads.js
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%m/%d/%Y")
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
NEW_PACKAGES="packagesV14"
if [[ "$rel_maj" -ge 4 ]]; then
NEW_PACKAGES="packagesV15"
fi
if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
echo "Skipping js/downloads.js for preview release: $RELEASE_VERSION"
else
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%m/%d/%Y")
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
NEW_PACKAGES="packagesV14"
if [[ "$rel_maj" -ge 4 ]]; then
NEW_PACKAGES="packagesV15"
fi

python3 <<EOF
python3 <<EOF
import re

release_version = "${RELEASE_VERSION}"
Expand Down Expand Up @@ -254,13 +293,44 @@ with open("js/downloads.js", "w") as f:
f.write(newline + "\n")
EOF

echo "Edited js/downloads.js"
echo "Edited js/downloads.js"
fi

# 3. Add news post
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
FILENAME="news/_posts/${RELEASE_DATE}-spark-${RELEASE_VERSION//./-}-released.md"
mkdir -p news/_posts
cat > "$FILENAME" <<EOF

if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
BASE_VERSION="${RELEASE_VERSION%%-preview*}"
cat > "$FILENAME" <<EOF
---
layout: post
title: Preview release of Spark ${BASE_VERSION}
categories:
- News
tags: []
status: publish
type: post
published: true
meta:
_edit_last: '4'
_wpas_done_all: '1'
---
To enable wide-scale community testing of the upcoming Spark ${BASE_VERSION} release, the Apache Spark community has posted a
<a href="https://archive.apache.org/dist/spark/spark-${RELEASE_VERSION}/">Spark ${RELEASE_VERSION} release</a>.
This preview is not a stable release in terms of either API or functionality, but it is meant to give the community early
access to try the code that will become Spark ${BASE_VERSION}. If you would like to test the release,
please <a href="https://archive.apache.org/dist/spark/spark-${RELEASE_VERSION}/">download</a> it, and send feedback using either
<a href="https://spark.apache.org/community.html">mailing lists</a> or
<a href="https://issues.apache.org/jira/browse/SPARK/?selectedTab=com.atlassian.jira.jira-projects-plugin:summary-panel">JIRA</a>.
The documentation is available at the <a href="https://spark.apache.org/docs/${RELEASE_VERSION}/">link</a>.

We'd like to thank our contributors and users for their contributions and early feedback to this release. This release would not have been possible without you.
EOF

else
cat > "$FILENAME" <<EOF
---
layout: post
title: Spark ${RELEASE_VERSION} released
Expand All @@ -276,16 +346,20 @@ meta:
---
We are happy to announce the availability of <a href="{{site.baseurl}}/releases/spark-release-${RELEASE_VERSION}.html" title="Spark Release ${RELEASE_VERSION}">Apache Spark ${RELEASE_VERSION}</a>! Visit the <a href="{{site.baseurl}}/releases/spark-release-${RELEASE_VERSION}.html" title="Spark Release ${RELEASE_VERSION}">release notes</a> to read about the new features, or <a href="{{site.baseurl}}/downloads.html">download</a> the release today.
EOF
fi

echo "Created $FILENAME"

# 4. Add release notes with Python to extract JIRA version ID
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
JIRA_PROJECT_ID=12315420
JIRA_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions"
JSON=$(curl -s "$JIRA_URL")
if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
echo "Skipping JIRA release notes for preview release: $RELEASE_VERSION"
else
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
JIRA_PROJECT_ID=12315420
JIRA_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions"
JSON=$(curl -s "$JIRA_URL")

VERSION_ID=$(python3 - <<EOF
VERSION_ID=$(python3 - <<EOF
import sys, json

release_version = "${RELEASE_VERSION}"
Expand All @@ -305,32 +379,32 @@ for v in versions:

print(version_id)
EOF
)
)

if [[ -z "$VERSION_ID" ]]; then
echo "Error: Couldn't find JIRA version ID for $RELEASE_VERSION" >&2
fi
if [[ -z "$VERSION_ID" ]]; then
echo "Error: Couldn't find JIRA version ID for $RELEASE_VERSION" >&2
fi

JIRA_LINK="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=${JIRA_PROJECT_ID}&version=${VERSION_ID}"
JIRA_LINK="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=${JIRA_PROJECT_ID}&version=${VERSION_ID}"

IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
if [[ "$rel_patch" -eq 0 ]]; then
ACKNOWLEDGE="patches and features to this release."
BODY="Apache Spark ${RELEASE_VERSION} is a new feature release. It introduces new functionality and improvements. We encourage users to try it and provide feedback."
else
ACKNOWLEDGE="patches to this release."
BODY="Apache Spark ${RELEASE_VERSION} is a maintenance release containing security and correctness fixes. This release is based on the branch-${rel_maj}.${rel_min} maintenance branch of Spark. We strongly recommend all ${rel_maj}.${rel_min} users to upgrade to this stable release."
fi
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
if [[ "$rel_patch" -eq 0 ]]; then
ACKNOWLEDGE="patches and features to this release."
BODY="Apache Spark ${RELEASE_VERSION} is a new feature release. It introduces new functionality and improvements. We encourage users to try it and provide feedback."
else
ACKNOWLEDGE="patches to this release."
BODY="Apache Spark ${RELEASE_VERSION} is a maintenance release containing security and correctness fixes. This release is based on the branch-${rel_maj}.${rel_min} maintenance branch of Spark. We strongly recommend all ${rel_maj}.${rel_min} users to upgrade to this stable release."
fi

BODY+="
BODY+="

You can find the list of resolved issues and detailed changes in the [JIRA release notes](${JIRA_LINK}).

We would like to acknowledge all community members for contributing ${ACKNOWLEDGE}"

FILENAME="releases/_posts/${RELEASE_DATE}-spark-release-${RELEASE_VERSION}.md"
mkdir -p releases/_posts
cat > "$FILENAME" <<EOF
FILENAME="releases/_posts/${RELEASE_DATE}-spark-release-${RELEASE_VERSION}.md"
mkdir -p releases/_posts
cat > "$FILENAME" <<EOF
---
layout: post
title: Spark Release ${RELEASE_VERSION}
Expand All @@ -347,40 +421,50 @@ meta:
${BODY}
EOF

echo "Created $FILENAME"
echo "Created $FILENAME"
fi

# 5. Build the website
bundle install
bundle exec jekyll build

# 6. Update latest symlink if minor/major release
LINK_PATH="site/docs/latest"
TARGET_DIR="site/docs/$RELEASE_VERSION"
# 6. Update latest or preview symlink
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
if [[ "$rel_patch" -eq 0 ]]; then
if [[ -L "$LINK_PATH" ]]; then
CURRENT_TARGET=$(readlink "$LINK_PATH")
else
CURRENT_TARGET=""
fi

if [[ "$CURRENT_TARGET" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
IFS='.' read -r cur_maj cur_min cur_patch <<< "$CURRENT_TARGET"
if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
LINK_PATH="site/docs/preview"

if [[ "$rel_maj" -gt "$cur_maj" ]]; then
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (major version increased)"
elif [[ "$rel_maj" -eq "$cur_maj" && "$rel_min" -gt "$cur_min" ]]; then
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (minor version increased)"
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (preview release)"

else
LINK_PATH="site/docs/latest"

if [[ "$rel_patch" -eq 0 ]]; then
if [[ -L "$LINK_PATH" ]]; then
CURRENT_TARGET=$(readlink "$LINK_PATH")
else
echo "Symlink $LINK_PATH points to $CURRENT_TARGET with equal or newer major.minor, no change"
CURRENT_TARGET=""
fi

if [[ "$CURRENT_TARGET" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
IFS='.' read -r cur_maj cur_min cur_patch <<< "$CURRENT_TARGET"

if [[ "$rel_maj" -gt "$cur_maj" ]]; then
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (major version increased)"
elif [[ "$rel_maj" -eq "$cur_maj" && "$rel_min" -gt "$cur_min" ]]; then
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (minor version increased)"
else
echo "Symlink $LINK_PATH points to $CURRENT_TARGET with equal or newer major.minor, no change"
fi
else
echo "No valid existing version target."
fi
else
echo "No valid existing version target."
echo "Patch release detected ($RELEASE_VERSION), not updating symlink"
fi
else
echo "Patch release detected ($RELEASE_VERSION), not updating symlink"
fi

git add .
Expand Down