HBASE-26645 Upgrade create-release to support both JDK8 and JDK11

dev-support/create-release/do-release-docker.sh

@@ -62,52 +62,121 @@ function usage {
 Usage: $NAME [OPTIONS]
 Runs release scripts inside a docker image.
 Options:
-  -d [path]    Required. Working directory. Output will be written to "output" in here.
-  -f           "force" -- actually publish this release. Unless you specify '-f', it will
-               default to dry run mode, which checks and does local builds, but does not
-               upload anything.
-  -t [tag]     Tag for the hbase-rm docker image to use for building (default: "latest").
-  -j [path]    Path to local JDK installation to use building. By default the script will
-               use openjdk8 installed in the docker image.
-  -p [project] Project to build: e.g. 'hbase' or 'hbase-thirdparty'; defaults to PROJECT env var
-  -r [repo]    Git repo to use for remote git operations. defaults to ASF gitbox for project.
-  -s [step]    Runs a single step of the process; valid steps: tag|publish-dist|publish-release.
-               If none specified, runs tag, then publish-dist, and then publish-release.
-               'publish-snapshot' is also an allowed, less used, option.
-  -x           Debug. Does less clean up (env file, gpg forwarding on mac)
+  -d [path], --directory=[path]
+     Required. Working directory. Output will be written to "output" in here.
+  -f, --force
+     Optional. Perform any associated permanent action. Without this flag, the
+     release defaults to 'dry run' mode, which checks and does local builds, but
+     does not upload anything.
+  -j [path], --java=[path]
+     Optional. Path to JDK installation on the host OS, which is mounted into
+     the container runtime. Mutually exclusive with the --jdk8 and --jdk11
+     options.
+  --jdk8
+     Optional. Override the default JDK selection and use the OpenJDK 8
+     installation provided by the container runtime. Mutually exclusive with the
+     -j and --jdk11 options.
+  --jdk11
+     Optional. Override the default JDK selection and use the OpenJDK 11
+     installation provided by the container runtime. Mutually exclusive with the
+     -j and --jdk8 options. When none of -j, --jdk8, --jdk11 are specified, the
+     default behavior is to behave as if this flag had been specified.
+  -p [project], --project=[project]
+     Optional. The name of the project to build: e.g. 'hbase' or
+     'hbase-thirdparty'; defaults to \`PROJECT\` environment variable.
+  -r [repo], --repo=[repo]
+     Optional. Git repo to use for remote git operations. defaults to ASF gitbox
+     for project.
+  -s [step], --step=[step]
+     Optional. Runs a single step of the process; valid steps: \`tag\`,
+     \`publish-dist\`, \`publish-snapshot\`, \`publish-release\`. If not
+     specified, runs \`tag\`, then \`publish-dist\`, and then \`publish-release\`.
+  -t [tag], --tag=[tag]
+     Optional. Tag for the hbase-rm docker image to use for building (default:
+     "latest").
+  -x, --debug
+     Optional. Enable debug mode, which produces more verbose output and
+     performs less clean up (env file, gpg forwarding on mac).
 EOF
   exit 1
 }
 
+# For long option names, use GNU getopt. This script assumes the presence of docker anyway, so use
+# the getopt provided out-of-the-box by the linux distro. Use the same base image as is used by
+# hbase-rm/Dockerfile.
+GETOPT_OPTS="$(docker container run \
+  ubuntu:22.04 \
+  /usr/bin/getopt \
+  -o 'd:fhj:p:r:s:t:x' \
+  --long 'directory:,force,help,java:,jdk8,jdk11,project:,repo:,step:,tag:,debug' \
+  -n 'do-release-docker.sh' \
+  -- \
+  "$@")"
+eval set -- "$GETOPT_OPTS"
+
+if [[ "$1" == "--" ]] ; then
+  usage
+fi
+
 WORKDIR=
-IMGTAG=latest
-JAVA=
+IMGTAG='latest'
 RELEASE_STEP=
 GIT_REPO=
-while getopts "d:fhj:p:r:s:t:x" opt; do
-  case $opt in
-    d) WORKDIR="$OPTARG" ;;
-    f) DRY_RUN=0 ;;
-    t) IMGTAG="$OPTARG" ;;
-    j) JAVA="$OPTARG" ;;
-    p) PROJECT="$OPTARG" ;;
-    r) GIT_REPO="$OPTARG" ;;
-    s) RELEASE_STEP="$OPTARG" ;;
-    x) DEBUG=1 ;;
-    h) usage ;;
-    ?) error "Invalid option. Run with -h for help." ;;
+while true ; do
+  case "$1" in
+    -d | --directory )
+      case "$2" in
+        "" ) shift 2 ;;
+        *  ) WORKDIR="$2" ; shift 2 ;;
+      esac ;;
+    -f | --force ) DRY_RUN=0 ; shift ;;
+    -h | --help ) usage ;;
+    -j | --java )
+      case "$2" in
+        "" ) shift 2 ;;
+        *  ) JAVA="$2" ; shift 2 ;;
+      esac ;;
+    --jdk8 ) JDK8=1 ; shift ;;
+    --jdk11 ) JDK11=1 ; shift ;;
+    -p | --project )
+      case "$2" in
+        "" ) shift 2 ;;
+        *  ) PROJECT="$2" ; shift 2 ;;
+      esac ;;
+    -r | --repo )
+      case "$2" in
+        "" ) shift 2 ;;
+        *  ) GIT_REPO="$2" ; shift 2 ;;
+      esac ;;
+    -s | --step )
+      case "$2" in
+        "" ) shift 2 ;;
+        *  ) RELEASE_STEP="$2" ; shift 2 ;;
+      esac ;;
+    -t | --tag )
+      case "$2" in
+        "" ) shift 2 ;;
+        *  ) IMGTAG="$2" ; shift 2 ;;
+      esac ;;
+    -x | --debug ) DEBUG=1 ; shift ;;
+    -- ) shift ; break ;;
+    * ) error "Invalid option '$1'. Run with -h for help." ;;
   esac
 done
-shift $((OPTIND-1))
-if (( $# > 0 )); then
-  error "Arguments can only be provided with option flags, invalid args: $*"
-fi
+
 export DEBUG
 
 if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
   error "Work directory (-d) must be defined and exist. Run with -h for help."
 fi
 
+JDK_ONE_OR_NONE="${JAVA:+x}${JDK8:+x}${JDK11:+x}"
+case "$JDK_ONE_OR_NONE" in
+  xx | xxx )
+    error "Options --java, --jdk8, --jdk11 are mutually exclusive. Run with -h for help."
+    ;;
+esac
+
 if [ -d "$WORKDIR/output" ]; then
   read -r -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER
   if [ "$ANSWER" != "y" ]; then
@@ -229,6 +298,14 @@ if [ -n "$JAVA" ]; then
   JAVA_MOUNT=(--mount "type=bind,src=${JAVA},dst=/opt/hbase-java,readonly")
 fi
 
+if [ -n "$JDK8" ] ; then
+  echo "JDK8=${JDK8}" >> "$ENVFILE"
+fi
+
+if [ -n "$JDK11" ] ; then
+  echo "JDK11=${JDK11}" >> "$ENVFILE"
+fi
+
 #TODO some debug output would be good here
 GIT_REPO_MOUNT=()
 if [ -n "${GIT_REPO}" ]; then

dev-support/create-release/do-release.sh

@@ -81,9 +81,17 @@ if [ "$RUNNING_IN_DOCKER" = "1" ]; then
   if [ -n "$JAVA_HOME" ]; then
     echo "Using JAVA_HOME from host."
     export PATH="$JAVA_HOME/bin:$PATH"
+  elif [ -n "$JDK8" ] ; then
+    jenv global 1.8
+    # TODO: use jenv export plugin instead?
+    JAVA_HOME=$(jenv javahome)
+    export JAVA_HOME
   else
-    # JAVA_HOME for the openjdk package.
-    export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+    # default to the openjdk11 package.
+    jenv global 11
+    # TODO: use jenv export plugin instead?
+    JAVA_HOME=$(jenv javahome)
+    export JAVA_HOME
   fi
 else
   # Outside docker, need to ask for information about the release.

dev-support/create-release/hbase-rm/Dockerfile

@@ -16,12 +16,8 @@
 #
 
 # Image for building HBase releases. Based on Ubuntu 22.04.
-#
-# Includes:
-# * Java 8
 FROM ubuntu:22.04
 
-
 # Install extra needed repos and refresh.
 #
 # This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch
@@ -36,16 +32,29 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
     libxml2-utils='2.9.13+dfsg-*' \
     lsof='4.93.2+dfsg-*' \
     openjdk-8-jdk='8u*' \
+    openjdk-11-jdk-headless='11*' \
     python3-pip='22.0.2+dfsg-*' \
     subversion='1.14.1-*' \
     wget='1.21.2-*' \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
-  && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \
   && pip3 install --no-cache-dir python-dateutil==2.8.2
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+# Install JEnv
+# update{-java}-alternatives is a global decision requiring sudo, but specifying a JDK at runtime
+# requires an update for the local user/shell. Use JEnv for this purpose.
+ENV JENV_PRODUCT_VERSION=0.5.5
+ARG JENV_URL="https://github.com/jenv/jenv/archive/${JENV_PRODUCT_VERSION}.tar.gz"
+ARG JENV_SHA512="2e312cfb4c0b2033fdc0ef4f3babfcdff2f58151247b3524589a70a0df3613d0af6abec1d044628ba321c1c0df5c5ba19d7a7bc93f41adbf4d5c0f29a3dcf9f8"
+RUN mkdir -p /opt/jenv \
+  && curl -fsSL -o /tmp/jenv.tar.gz "${JENV_URL}" \
+  && echo "${JENV_SHA512}  /tmp/jenv.tar.gz" | sha512sum -c - \
+  && tar -zxf /tmp/jenv.tar.gz -C /opt/jenv --strip-components=1 \
+  && rm -f /tmp/jenv.tar.gz \
+  && ln -s /opt/jenv/bin/jenv /usr/bin/jenv
+
 # Install maven
 ENV MAVEN_VERSION=3.8.6
 ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
@@ -70,13 +79,18 @@ ENV YETUS_HOME /opt/yetus
 
 ARG UID
 ARG RM_USER
-RUN groupadd hbase-rm && \
-    useradd --create-home -l --shell /bin/bash -p hbase-rm -u $UID $RM_USER && \
-    mkdir /home/$RM_USER/.gnupg && \
-    chown -R $RM_USER:hbase-rm /home/$RM_USER && \
-    chmod -R 700 /home/$RM_USER
+RUN groupadd hbase-rm \
+  && useradd --create-home --no-log-init --shell /bin/bash --password hbase-rm --uid $UID $RM_USER \
+  && mkdir /home/$RM_USER/.gnupg \
+  && chown -R $RM_USER:hbase-rm /home/$RM_USER \
+  && chmod -R 700 /home/$RM_USER
 
 USER $RM_USER:hbase-rm
+RUN mkdir -p "${HOME}/.jenv/versions" \
+  && bash -c 'echo eval "$(jenv init -)" >> ${HOME}/.bashrc' \
+  && { update-java-alternatives --list || true ; } | cut -d/ -f2- | xargs -I{} jenv add /{}
+ENV PATH="/home/${RM_USER}/.jenv/shims:${PATH}"
+
 WORKDIR /home/$RM_USER/hbase-rm/
 
 ENTRYPOINT [ "./do-release.sh" ]