HBASE-23339 release scripts should use host gpg-agent when running in container.

* put together a docker container that can use host gpg-agent forwarded over ssh.
* use gpg-agent forwarding container on OS X and directly forward the agent on Linux
* clean up the release container on exit
* use docker mounts instead of the deprecated volume syntax
* use image names within our project namespace

Author: busbey

dev-support/create-release/README.txt

@@ -17,6 +17,16 @@ anomalies are explained up in JIRA.
 
 See http://hbase.apache.org/book.html#maven.release
 
+Before starting an RC build, make sure your local gpg-agent has configs
+to properly handle your credentials, especially if you want to avoid
+typing the passphrase to your secret key.
+
+e.g. if you are going to run and step away, best to increase the TTL
+on caching the unlocked secret via ~/.gnupg/gpg-agent.conf
+  # in seconds, e.g. a day
+  default-cache-ttl 86400
+  max-cache-ttl 86400
+
 Running a build on GCE is easy enough. Here are some notes if of use.
 Create an instance. 4CPU/15G/10G disk seems to work well enough.
 Once up, run the below to make your machine fit for RC building:

dev-support/create-release/do-release-docker.sh

@@ -75,6 +75,7 @@ Options:
   -s [step]    runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
                If none specified, runs tag, then publish-dist, and then publish-release.
                'publish-snapshot' is also an allowed, less used, option.
+  -x           debug. do less clean up. (env file, gpg forwarding on mac)
 EOF
   exit 0
 }
@@ -84,7 +85,7 @@ IMGTAG=latest
 JAVA=
 RELEASE_STEP=
 GIT_REPO=
-while getopts "d:fhj:p:r:s:t:" opt; do
+while getopts "d:fhj:p:r:s:t:x" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
     f) DRY_RUN=0 ;;
@@ -93,6 +94,7 @@ while getopts "d:fhj:p:r:s:t:" opt; do
     p) PROJECT="$OPTARG" ;;
     r) GIT_REPO="$OPTARG" ;;
     s) RELEASE_STEP="$OPTARG" ;;
+    x) DEBUG=1 ;;
     h) usage ;;
     ?) error "Invalid option. Run with -h for help." ;;
   esac
@@ -113,12 +115,26 @@ if [ -d "$WORKDIR/output" ]; then
   fi
 fi
 
+if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ] || \
+   [ -f "${WORKDIR}/gpg-proxy.cid" ] || \
+   [ -f "${WORKDIR}/release.cid" ]; then
+  read -r -p "container/pid files from prior run exists. Overwrite and continue? [y/n] " ANSWER
+  if [ "$ANSWER" != "y" ]; then
+    error "Exiting."
+  fi
+fi
+
 cd "$WORKDIR"
 rm -rf "$WORKDIR/output"
+rm -rf "${WORKDIR}/gpg-proxy.ssh.pid" "${WORKDIR}/gpg-proxy.cid" "${WORKDIR}/release.cid"
 mkdir "$WORKDIR/output"
 
+banner "Gathering release details."
+HOST_OS="$(get_host_os)"
 get_release_info
 
+banner "Setup"
+
 # Place all RM scripts and necessary data in a local directory that must be defined in the command
 # line. This directory is mounted into the image. Its WORKDIR, the arg passed with -d.
 for f in "$SELF"/*; do
@@ -127,25 +143,65 @@ for f in "$SELF"/*; do
   fi
 done
 
-GPG_KEY_FILE="$WORKDIR/gpg.key"
+# We need to import that public key in the container in order to use the private key via the agent.
+GPG_KEY_FILE="$WORKDIR/gpg.key.public"
+echo "Exporting public key for ${GPG_KEY}"
 fcreate_secure "$GPG_KEY_FILE"
-$GPG --passphrase "$GPG_PASSPHRASE" --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
+$GPG "${GPG_ARGS[@]}" --export "${GPG_KEY}" > "${GPG_KEY_FILE}"
+
+function cleanup {
+  local id
+  banner "Release Cleanup"
+  if is_debug; then
+    echo "skipping due to debug run"
+    return 0
+  fi
+  echo "details in cleanup.log"
+  if [ -f "${ENVFILE}" ]; then
+    rm -f "$ENVFILE"
+  fi
+  rm -f "$GPG_KEY_FILE"
+  if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ]; then
+    id=$(cat "${WORKDIR}/gpg-proxy.ssh.pid")
+    echo "Stopping ssh tunnel for gpg-agent at PID ${id}" | tee -a cleanup.log
+    kill -9 "${id}" >>cleanup.log 2>&1 || true
+    rm -f "${WORKDIR}/gpg-proxy.ssh.pid" >>cleanup.log 2>&1
+  fi
+  if [ -f "${WORKDIR}/gpg-proxy.cid" ]; then
+    id=$(cat "${WORKDIR}/gpg-proxy.cid")
+    echo "Stopping gpg-proxy container with ID ${id}" | tee -a cleanup.log
+    docker kill "${id}" >>cleanup.log 2>&1 || true
+    rm -f "${WORKDIR}/gpg-proxy.cid" >>cleanup.log 2>&1
+    # TODO we should remove the gpgagent volume?
+  fi
+  if [ -f "${WORKDIR}/release.cid" ]; then
+    id=$(cat "${WORKDIR}/release.cid")
+    echo "Stopping release container with ID ${id}" | tee -a cleanup.log
+    docker kill "${id}" >>cleanup.log 2>&1 || true
+    rm -f "${WORKDIR}/release.cid" >>cleanup.log 2>&1
+  fi
+}
+
+trap cleanup EXIT
+
+echo "Host OS: ${HOST_OS}"
+if [ "${HOST_OS}" == "DARWIN" ]; then
+  run_silent "Building gpg-agent-proxy image with tag ${IMGTAG}..." "docker-proxy-build.log" \
+    docker build --build-arg "UID=${UID}" --build-arg "RM_USER=${USER}" \
+        --tag "org.apache.hbase/gpg-agent-proxy:${IMGTAG}" "${SELF}/mac-sshd-gpg-agent"
+fi
 
 run_silent "Building hbase-rm image with tag $IMGTAG..." "docker-build.log" \
-  docker build -t "hbase-rm:$IMGTAG" --build-arg UID=$UID "$SELF/hbase-rm"
+  docker build --tag "org.apache.hbase/hbase-rm:$IMGTAG" --build-arg "UID=$UID" \
+      --build-arg "RM_USER=${USER}" "$SELF/hbase-rm"
 
+banner "Final prep for container launch."
+echo "Writing out environment for container."
 # Write the release information to a file with environment variables to be used when running the
 # image.
 ENVFILE="$WORKDIR/env.list"
 fcreate_secure "$ENVFILE"
 
-function cleanup {
-  rm -f "$ENVFILE"
-  rm -f "$GPG_KEY_FILE"
-}
-
-trap cleanup EXIT
-
 cat > "$ENVFILE" <<EOF
 PROJECT=$PROJECT
 DRY_RUN=$DRY_RUN
@@ -161,15 +217,15 @@ GIT_NAME=$GIT_NAME
 GIT_EMAIL=$GIT_EMAIL
 GPG_KEY=$GPG_KEY
 ASF_PASSWORD=$ASF_PASSWORD
-GPG_PASSPHRASE=$GPG_PASSPHRASE
 RELEASE_STEP=$RELEASE_STEP
 API_DIFF_TAG=$API_DIFF_TAG
+HOST_OS=$HOST_OS
 EOF
 
-JAVA_VOL=()
+JAVA_MOUNT=()
 if [ -n "$JAVA" ]; then
   echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
-  JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
+  JAVA_MOUNT=(--mount "type=bind,src=${JAVA},dst=/opt/hbase-java,readonly")
 fi
 
 #TODO some debug output would be good here
@@ -223,12 +279,59 @@ if [ -n "${GIT_REPO}" ]; then
   echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
 fi
 
-echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
-cmd=(docker run -ti \
+GPG_PROXY_MOUNT=()
+if [ "${HOST_OS}" == "DARWIN" ]; then
+  GPG_PROXY_MOUNT=(--mount "type=volume,src=gpgagent,dst=/home/${USER}/.gnupg/")
+  echo "Setting up GPG agent proxy container needed on OS X."
+  echo "	we should clean this up for you. If that fails the container ID is below and in " \
+      "gpg-proxy.cid"
+  #TODO the key pair used should be configurable
+  docker run --rm -p 62222:22 \
+     --detach --cidfile "${WORKDIR}/gpg-proxy.cid" \
+     --mount \
+     "type=bind,src=${HOME}/.ssh/id_rsa.pub,dst=/home/${USER}/.ssh/authorized_keys,readonly" \
+     "${GPG_PROXY_MOUNT[@]}" \
+     "org.apache.hbase/gpg-agent-proxy:${IMGTAG}"
+  # gotta trust the container host
+  ssh-keyscan -p 62222 localhost 2>/dev/null | sort > "${WORKDIR}/gpg-agent-proxy.ssh-keyscan"
+  cat "${HOME}/.ssh/known_hosts" | sort | comm -1 -3 - "${WORKDIR}/gpg-agent-proxy.ssh-keyscan" > "${WORKDIR}/gpg-agent-proxy.known_hosts"
+  if [ -s "${WORKDIR}/gpg-agent-proxy.known_hosts" ]; then
+    declare host_key
+    echo "Your ssh known_hosts does not include the entries for the gpg-agent proxy container."
+    echo "The following entry(ies) arre missing:"
+    cat "${WORKDIR}/gpg-agent-proxy.known_hosts" | sed -e 's/^/	/'
+    read -r -p "Okay to add these entries to ${HOME}/.ssh/known_hosts? [y/n] " ANSWER
+    if [ "$ANSWER" != "y" ]; then
+      error "Exiting."
+    fi
+    cat "${WORKDIR}/gpg-agent-proxy.known_hosts" >> "${HOME}/.ssh/known_hosts"
+  fi
+  echo "Launching ssh reverse tunnel from the container to gpg agent."
+  echo "	we should clean this up for you. If that fails the PID is in gpg-proxy.ssh.pid"
+  ssh -p 62222 -R "/home/${USER}/.gnupg/S.gpg-agent:$(gpgconf --list-dir agent-extra-socket)" \
+      -i "${HOME}/.ssh/id_rsa" -N -n localhost >gpg-proxy.ssh.log 2>&1 &
+  echo $! > "${WORKDIR}/gpg-proxy.ssh.pid"
+else
+  # TODO this presumes we are still trying to make a local gpg-agent available to the container.
+  #      add an option so that we can run the buid on a remote machine and get the forwarded
+  #      gpg-agent in the container. Should look like the side-car container mount above.
+  #      it is important not to do that for a local linux agent because we only want the container
+  #      to get access to the restricted extra socket on our local gpg-agent.
+  GPG_PROXY_MOUNT=(--mount \
+      "type=bind,src=$(gpgconf --list-dir agent-extra-socket),dst=/home/${USER}/.gnupg/S.gpg-agent")
+fi
+
+banner "Building $RELEASE_TAG; output will be at $WORKDIR/output"
+echo "We should clean the container up when we are done. If that fails then the container ID " \
+    "is in release.cid"
+echo
+cmd=(docker run --rm -ti \
   --env-file "$ENVFILE" \
-  --volume "$WORKDIR:/opt/hbase-rm" \
-  "${JAVA_VOL[@]}" \
+  --cidfile "${WORKDIR}/release.cid" \
+  --mount "type=bind,src=${WORKDIR},dst=/home/${USER}/hbase-rm" \
+  "${JAVA_MOUNT[@]}" \
   "${GIT_REPO_MOUNT[@]}" \
-  "hbase-rm:$IMGTAG")
+  "${GPG_PROXY_MOUNT[@]}" \
+  "org.apache.hbase/hbase-rm:$IMGTAG")
 echo "${cmd[*]}"
 "${cmd[@]}"

dev-support/create-release/do-release.sh

@@ -42,27 +42,41 @@ fi
 
 # If running in docker, import and then cache keys.
 if [ "$RUNNING_IN_DOCKER" = "1" ]; then
-  # Run gpg agent.
-  eval "$(gpg-agent --disable-scdaemon --daemon --no-grab  --allow-preset-passphrase \
-          --default-cache-ttl=86400 --max-cache-ttl=86400)"
-  echo "GPG Version: $(gpg --version)"
-  # Inside docker, need to import the GPG keyfile stored in the current directory.
-  # (On workstation, assume GPG has access to keychain/cache with key_id already imported.)
-  echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
+  # when Docker Desktop for mac is running under load there is a delay before the mounted volume
+  # becomes available. if we do not pause then we may try to use the gpg-agent socket before docker
+  # has got it ready and we will not think there is a gpg-agent.
+  if [ "${HOST_OS}" == "DARWIN" ]; then
+    sleep 5
+  fi
+  # in docker our working dir is set to where all of our scripts are held
+  # and we want default output to go into the "output" directory that should be in there.
+  if [ -d "output" ]; then
+    cd output
+  fi
+  echo "GPG Version: $("${GPG}" "${GPG_ARGS[@]}" --version)"
+  # Inside docker, need to import the GPG key stored in the current directory.
+  $GPG "${GPG_ARGS[@]}" --import "$SELF/gpg.key.public"
 
   # We may need to adjust the path since JAVA_HOME may be overridden by the driver script.
   if [ -n "$JAVA_HOME" ]; then
+    echo "Using JAVA_HOME from host."
     export PATH="$JAVA_HOME/bin:$PATH"
   else
     # JAVA_HOME for the openjdk package.
-    export JAVA_HOME=/usr
+    export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
   fi
 else
   # Outside docker, need to ask for information about the release.
   get_release_info
 fi
+
 GPG_TTY="$(tty)"
 export GPG_TTY
+echo "Testing gpg signing."
+echo "foo" > gpg_test.txt
+"${GPG}" "${GPG_ARGS[@]}" --detach --armor --sign gpg_test.txt
+# In --batch mode we have to be explicit about what we are verifying
+"${GPG}" "${GPG_ARGS[@]}" --verify gpg_test.txt.asc gpg_test.txt
 
 if [[ -z "$RELEASE_STEP" ]]; then
   # If doing all stages, leave out 'publish-snapshot'

dev-support/create-release/hbase-rm/Dockerfile

@@ -50,10 +50,15 @@ RUN wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&fi
         tar xvz -C /opt
 ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
 
-WORKDIR /opt/hbase-rm/output
-
 ARG UID
-RUN useradd -m -s /bin/bash -p hbase-rm -u $UID hbase-rm
-USER hbase-rm:hbase-rm
+ARG RM_USER
+RUN groupadd hbase-rm && \
+    useradd --create-home --shell /bin/bash -p hbase-rm -u $UID $RM_USER && \
+    mkdir /home/$RM_USER/.gnupg && \
+    chown -R $RM_USER:hbase-rm /home/$RM_USER && \
+    chmod -R 700 /home/$RM_USER
+
+USER $RM_USER:hbase-rm
+WORKDIR /home/$RM_USER/hbase-rm/
 
-ENTRYPOINT [ "/opt/hbase-rm/do-release.sh" ]
+ENTRYPOINT [ "./do-release.sh" ]