diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index 804661945e09..2c231f4449bc 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -62,52 +62,121 @@ function usage { Usage: $NAME [OPTIONS] Runs release scripts inside a docker image. Options: - -d [path] Required. Working directory. Output will be written to "output" in here. - -f "force" -- actually publish this release. Unless you specify '-f', it will - default to dry run mode, which checks and does local builds, but does not - upload anything. - -t [tag] Tag for the hbase-rm docker image to use for building (default: "latest"). - -j [path] Path to local JDK installation to use building. By default the script will - use openjdk8 installed in the docker image. - -p [project] Project to build: e.g. 'hbase' or 'hbase-thirdparty'; defaults to PROJECT env var - -r [repo] Git repo to use for remote git operations. defaults to ASF gitbox for project. - -s [step] Runs a single step of the process; valid steps: tag|publish-dist|publish-release. - If none specified, runs tag, then publish-dist, and then publish-release. - 'publish-snapshot' is also an allowed, less used, option. - -x Debug. Does less clean up (env file, gpg forwarding on mac) + -d [path], --directory=[path] + Required. Working directory. Output will be written to "output" in here. + -f, --force + Optional. Perform any associated permanent action. Without this flag, the + release defaults to 'dry run' mode, which checks and does local builds, but + does not upload anything. + -j [path], --java=[path] + Optional. Path to JDK installation on the host OS, which is mounted into + the container runtime. Mutually exclusive with the --jdk8 and --jdk11 + options. + --jdk8 + Optional. Override the default JDK selection and use the OpenJDK 8 + installation provided by the container runtime. Mutually exclusive with the + -j and --jdk11 options. + --jdk11 + Optional. Override the default JDK selection and use the OpenJDK 11 + installation provided by the container runtime. Mutually exclusive with the + -j and --jdk8 options. When none of -j, --jdk8, --jdk11 are specified, the + default behavior is to behave as if this flag had been specified. + -p [project], --project=[project] + Optional. The name of the project to build: e.g. 'hbase' or + 'hbase-thirdparty'; defaults to \`PROJECT\` environment variable. + -r [repo], --repo=[repo] + Optional. Git repo to use for remote git operations. defaults to ASF gitbox + for project. + -s [step], --step=[step] + Optional. Runs a single step of the process; valid steps: \`tag\`, + \`publish-dist\`, \`publish-snapshot\`, \`publish-release\`. If not + specified, runs \`tag\`, then \`publish-dist\`, and then \`publish-release\`. + -t [tag], --tag=[tag] + Optional. Tag for the hbase-rm docker image to use for building (default: + "latest"). + -x, --debug + Optional. Enable debug mode, which produces more verbose output and + performs less clean up (env file, gpg forwarding on mac). EOF exit 1 } +# For long option names, use GNU getopt. This script assumes the presence of docker anyway, so use +# the getopt provided out-of-the-box by the linux distro. Use the same base image as is used by +# hbase-rm/Dockerfile. +GETOPT_OPTS="$(docker container run \ + ubuntu:22.04 \ + /usr/bin/getopt \ + -o 'd:fhj:p:r:s:t:x' \ + --long 'directory:,force,help,java:,jdk8,jdk11,project:,repo:,step:,tag:,debug' \ + -n 'do-release-docker.sh' \ + -- \ + "$@")" +eval set -- "$GETOPT_OPTS" + +if [[ "$1" == "--" ]] ; then + usage +fi + WORKDIR= -IMGTAG=latest -JAVA= +IMGTAG='latest' RELEASE_STEP= GIT_REPO= -while getopts "d:fhj:p:r:s:t:x" opt; do - case $opt in - d) WORKDIR="$OPTARG" ;; - f) DRY_RUN=0 ;; - t) IMGTAG="$OPTARG" ;; - j) JAVA="$OPTARG" ;; - p) PROJECT="$OPTARG" ;; - r) GIT_REPO="$OPTARG" ;; - s) RELEASE_STEP="$OPTARG" ;; - x) DEBUG=1 ;; - h) usage ;; - ?) error "Invalid option. Run with -h for help." ;; +while true ; do + case "$1" in + -d | --directory ) + case "$2" in + "" ) shift 2 ;; + * ) WORKDIR="$2" ; shift 2 ;; + esac ;; + -f | --force ) DRY_RUN=0 ; shift ;; + -h | --help ) usage ;; + -j | --java ) + case "$2" in + "" ) shift 2 ;; + * ) JAVA="$2" ; shift 2 ;; + esac ;; + --jdk8 ) JDK8=1 ; shift ;; + --jdk11 ) JDK11=1 ; shift ;; + -p | --project ) + case "$2" in + "" ) shift 2 ;; + * ) PROJECT="$2" ; shift 2 ;; + esac ;; + -r | --repo ) + case "$2" in + "" ) shift 2 ;; + * ) GIT_REPO="$2" ; shift 2 ;; + esac ;; + -s | --step ) + case "$2" in + "" ) shift 2 ;; + * ) RELEASE_STEP="$2" ; shift 2 ;; + esac ;; + -t | --tag ) + case "$2" in + "" ) shift 2 ;; + * ) IMGTAG="$2" ; shift 2 ;; + esac ;; + -x | --debug ) DEBUG=1 ; shift ;; + -- ) shift ; break ;; + * ) error "Invalid option '$1'. Run with -h for help." ;; esac done -shift $((OPTIND-1)) -if (( $# > 0 )); then - error "Arguments can only be provided with option flags, invalid args: $*" -fi + export DEBUG if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then error "Work directory (-d) must be defined and exist. Run with -h for help." fi +JDK_ONE_OR_NONE="${JAVA:+x}${JDK8:+x}${JDK11:+x}" +case "$JDK_ONE_OR_NONE" in + xx | xxx ) + error "Options --java, --jdk8, --jdk11 are mutually exclusive. Run with -h for help." + ;; +esac + if [ -d "$WORKDIR/output" ]; then read -r -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER if [ "$ANSWER" != "y" ]; then @@ -229,6 +298,14 @@ if [ -n "$JAVA" ]; then JAVA_MOUNT=(--mount "type=bind,src=${JAVA},dst=/opt/hbase-java,readonly") fi +if [ -n "$JDK8" ] ; then + echo "JDK8=${JDK8}" >> "$ENVFILE" +fi + +if [ -n "$JDK11" ] ; then + echo "JDK11=${JDK11}" >> "$ENVFILE" +fi + #TODO some debug output would be good here GIT_REPO_MOUNT=() if [ -n "${GIT_REPO}" ]; then diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 904d813fc3c6..8a801c667a93 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -81,9 +81,17 @@ if [ "$RUNNING_IN_DOCKER" = "1" ]; then if [ -n "$JAVA_HOME" ]; then echo "Using JAVA_HOME from host." export PATH="$JAVA_HOME/bin:$PATH" + elif [ -n "$JDK8" ] ; then + jenv global 1.8 + # TODO: use jenv export plugin instead? + JAVA_HOME=$(jenv javahome) + export JAVA_HOME else - # JAVA_HOME for the openjdk package. - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ + # default to the openjdk11 package. + jenv global 11 + # TODO: use jenv export plugin instead? + JAVA_HOME=$(jenv javahome) + export JAVA_HOME fi else # Outside docker, need to ask for information about the release. diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile index 5cc322b46f6f..6c39683a931d 100644 --- a/dev-support/create-release/hbase-rm/Dockerfile +++ b/dev-support/create-release/hbase-rm/Dockerfile @@ -16,12 +16,8 @@ # # Image for building HBase releases. Based on Ubuntu 22.04. -# -# Includes: -# * Java 8 FROM ubuntu:22.04 - # Install extra needed repos and refresh. # # This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch @@ -36,16 +32,29 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \ libxml2-utils='2.9.13+dfsg-*' \ lsof='4.93.2+dfsg-*' \ openjdk-8-jdk='8u*' \ + openjdk-11-jdk-headless='11*' \ python3-pip='22.0.2+dfsg-*' \ subversion='1.14.1-*' \ wget='1.21.2-*' \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \ && pip3 install --no-cache-dir python-dateutil==2.8.2 SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# Install JEnv +# update{-java}-alternatives is a global decision requiring sudo, but specifying a JDK at runtime +# requires an update for the local user/shell. Use JEnv for this purpose. +ENV JENV_PRODUCT_VERSION=0.5.5 +ARG JENV_URL="https://github.com/jenv/jenv/archive/${JENV_PRODUCT_VERSION}.tar.gz" +ARG JENV_SHA512="2e312cfb4c0b2033fdc0ef4f3babfcdff2f58151247b3524589a70a0df3613d0af6abec1d044628ba321c1c0df5c5ba19d7a7bc93f41adbf4d5c0f29a3dcf9f8" +RUN mkdir -p /opt/jenv \ + && curl -fsSL -o /tmp/jenv.tar.gz "${JENV_URL}" \ + && echo "${JENV_SHA512} /tmp/jenv.tar.gz" | sha512sum -c - \ + && tar -zxf /tmp/jenv.tar.gz -C /opt/jenv --strip-components=1 \ + && rm -f /tmp/jenv.tar.gz \ + && ln -s /opt/jenv/bin/jenv /usr/bin/jenv + # Install maven ENV MAVEN_VERSION=3.8.6 ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" @@ -70,13 +79,18 @@ ENV YETUS_HOME /opt/yetus ARG UID ARG RM_USER -RUN groupadd hbase-rm && \ - useradd --create-home -l --shell /bin/bash -p hbase-rm -u $UID $RM_USER && \ - mkdir /home/$RM_USER/.gnupg && \ - chown -R $RM_USER:hbase-rm /home/$RM_USER && \ - chmod -R 700 /home/$RM_USER +RUN groupadd hbase-rm \ + && useradd --create-home --no-log-init --shell /bin/bash --password hbase-rm --uid $UID $RM_USER \ + && mkdir /home/$RM_USER/.gnupg \ + && chown -R $RM_USER:hbase-rm /home/$RM_USER \ + && chmod -R 700 /home/$RM_USER USER $RM_USER:hbase-rm +RUN mkdir -p "${HOME}/.jenv/versions" \ + && bash -c 'echo eval "$(jenv init -)" >> ${HOME}/.bashrc' \ + && { update-java-alternatives --list || true ; } | cut -d/ -f2- | xargs -I{} jenv add /{} +ENV PATH="/home/${RM_USER}/.jenv/shims:${PATH}" + WORKDIR /home/$RM_USER/hbase-rm/ ENTRYPOINT [ "./do-release.sh" ]