HBASE-27847 Introduce HBase image

.gitignore

@@ -21,3 +21,6 @@ linklint-*.zip
 linklint/
 .checkstyle
 **/.checkstyle
+
+# detritus produced by kuttl
+kubeconfig*

dev-support/jenkins/Dockerfile

@@ -20,16 +20,29 @@
 FROM hadolint/hadolint:latest-debian as hadolint
 
 FROM maven:3.8-jdk-8
-
-# hadolint ignore=DL3008
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
-       binutils \
-       git \
-       rsync \
-       shellcheck \
-       wget && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+# hadolint ignore=SC1091
+RUN install -m 0755 -d /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && chmod a+r /etc/apt/keyrings/docker.gpg \
+    && echo \
+        "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+        "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+        tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get -q update \
+    && apt-get -q install --no-install-recommends -y \
+        binutils=* \
+        ca-certificates=* \
+        curl=* \
+        docker-buildx-plugin=* \
+        docker-ce-cli=* \
+        git=* \
+        gnupg=* \
+        rsync=* \
+        shellcheck=* \
+        wget=* \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=hadolint /bin/hadolint /bin/hadolint
 

dev-support/jenkins/jenkins_precommit_github_yetus.sh

@@ -111,6 +111,8 @@ YETUS_ARGS+=("--tests-filter=test4tests")
 # Dockerfile since we don't want to use the auto-pulled version.
 YETUS_ARGS+=("--docker")
 YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+# enabled docker-in-docker so that we can build container images
+YETUS_ARGS+=("--dockerind=true")
 YETUS_ARGS+=("--mvn-custom-repos")
 YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
 # effectively treat dev-support as a custom maven module
@@ -127,4 +129,4 @@ YETUS_ARGS+=("--proclimit=5000")
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
 
-/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"

hbase-kubernetes-deployment/dockerfiles/hbase/Dockerfile

@@ -0,0 +1,134 @@
+# syntax=docker/dockerfile:1.2
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG IMG_BASE='alpine'
+ARG IMG_BASE_TAG='latest'
+
+FROM ${IMG_BASE}:${IMG_BASE_TAG} as hbase_base
+# define ARGs for identites we may need to share with the environment.
+ARG HBASE_USER='hbase'
+ARG HBASE_ID='1000'
+ARG HADOOP_USER='hadoop'
+ARG HADOOP_ID='1001'
+ARG HDFS_USER='hdfs'
+ARG HDFS_ID='1002'
+
+ENV HBASE_USER="${HBASE_USER}"
+ENV HBASE_ID="${HBASE_ID}"
+ENV HADOOP_USER="${HADOOP_USER}"
+ENV HADOOP_ID="${HADOOP_ID}"
+ENV HDFS_USER="${HDFS_USER}"
+ENV HDFS_ID="${HDFS_ID}"
+
+# make a nonroot user kinda sorta like distroless. switch to using distroless in the future?
+# https://github.com/GoogleContainerTools/distroless
+ENV NON_ROOT_USER=nonroot
+ENV NON_ROOT_USER_ID=65532
+
+# Create `nonroot`, `hadoop`, and `hbase` groups ; `nonroot`, `hdfs`, and `hbase` users
+RUN addgroup -g "${NON_ROOT_USER_ID}" "${NON_ROOT_USER}" \
+    && addgroup -S -g "${HADOOP_ID}" "${HADOOP_USER}" \
+    && addgroup -S -g "${HBASE_ID}" "${HBASE_USER}" \
+    && adduser -S -D -H \
+        -h /dev/null \
+        -u "${NON_ROOT_USER_ID}" \
+        -G "${NON_ROOT_USER}" \
+        -g "${NON_ROOT_USER}" \
+    "${NON_ROOT_USER}" \
+    && adduser "${NON_ROOT_USER}" "${HBASE_USER}" \
+    && adduser "${NON_ROOT_USER}" "${HADOOP_USER}" \
+    && adduser -S -D -H \
+        -h /dev/null \
+        -u "${HDFS_ID}" \
+        -G "${HADOOP_USER}" \
+        -g "${HDFS_USER}" \
+        "${HDFS_USER}" \
+    && adduser -S -D -H \
+        -h /dev/null \
+        -u "${HBASE_ID}" \
+        -G "${HADOOP_USER}" \
+        -g "${HBASE_USER}" \
+        "${HBASE_USER}"
+
+FROM alpine AS verification_tools
+RUN apk add --no-cache gpg
+
+## -- Stage staged_hbase --
+# A stage that prepares the HBase installation directory.
+FROM verification_tools AS staged_hbase
+ARG HBASE_TGZ
+ARG HBASE_TGZ_SHA512
+COPY "${HBASE_TGZ}" /tmp/
+COPY "${HBASE_TGZ_SHA512}" /tmp/
+RUN ash -c 'cd /tmp ; diff <(gpg --print-md SHA512 "${HBASE_TGZ}") "${HBASE_TGZ_SHA512}"' \
+    && mkdir /tmp/hbase \
+    && cd /tmp/hbase \
+    && tar -xzf "/tmp/${HBASE_TGZ}" --strip-components=1 \
+    && rm -rf docs/ lib/shaded-clients/
+
+## -- Stage staged_hbase_operator_tools --
+# A stage that prepares the HBase Operator Tools installation directory.
+FROM verification_tools AS staged_hbase_operator_tools
+ARG HBASE_OPERATOR_TOOLS_TGZ
+ARG HBASE_OPERATOR_TOOLS_TGZ_SHA512
+COPY ${HBASE_OPERATOR_TOOLS_TGZ} /tmp/
+COPY ${HBASE_OPERATOR_TOOLS_TGZ_SHA512} /tmp/
+RUN ash -c 'cd /tmp ; diff <(gpg --print-md SHA512 "${HBASE_OPERATOR_TOOLS_TGZ}") "${HBASE_OPERATOR_TOOLS_TGZ_SHA512}"' \
+    && mkdir /tmp/hbase-operator-tools \
+    && cd /tmp/hbase-operator-tools \
+    && tar -xzf "/tmp/${HBASE_OPERATOR_TOOLS_TGZ}" --strip-components=1 \
+    && mkdir /tmp/bin
+COPY hbase-kubernetes-deployment/dockerfiles/hbase/hbck2 /tmp/bin/hbck2
+
+## -- Stage installed_jmx_exporter --
+# A stage that prepares the JMX Exporter installation directory. The jar must be in well-known location so can refer
+# to it in command yaml value.
+FROM alpine AS installed_jmx_exporter
+ARG JMX_PROMETHEUS_JAR
+COPY ${JMX_PROMETHEUS_JAR} /tmp/jmx_prometheus_javaagent.jar
+
+## stage final
+# copy all the bits out of prepared stages
+FROM hbase_base as final
+RUN apk add --no-cache \
+    bash \
+    curl \
+    openjdk17-jre-headless \
+    jq
+
+ENV HBASE_HOME="/opt/hbase"
+ENV HBASE_LOG_DIR="/var/log/hbase"
+ENV HBASE_OPERATOR_TOOLS_HOME="/opt/hbase-operator-tools"
+ENV JMX_PROMETHEUS_JAR="${HBASE_HOME}/jmx_prometheus_javaagent.jar"
+ENV HADOOP_HOME="${HBASE_HOME}"
+ENV PATH="${HBASE_HOME}/bin:${PATH}"
+ENV JAVA_HOME="/usr/lib/jvm/default-jvm"
+
+COPY --from=staged_hbase /tmp/hbase/ "${HBASE_HOME}/"
+COPY --from=staged_hbase_operator_tools \
+    /tmp/hbase-operator-tools/ "${HBASE_OPERATOR_TOOLS_HOME}/"
+COPY --from=staged_hbase_operator_tools /tmp/bin/hbck2 "${HBASE_HOME}/bin/"
+COPY --from=installed_jmx_exporter /tmp/jmx_prometheus_javaagent.jar "${JMX_PROMETHEUS_JAR}"
+
+RUN mkdir -p "${HBASE_LOG_DIR}" \
+    && chown "${HBASE_USER}":"${HBASE_USER}" "${HBASE_LOG_DIR}" \
+    && chmod g+w "${HBASE_LOG_DIR}"
+
+ENV USER="${NON_ROOT_USER}"
+USER "${USER}"
+CMD ["hbase", "version"]

hbase-kubernetes-deployment/dockerfiles/hbase/README.md

@@ -0,0 +1,57 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# dockerfiles/hbase
+
+There is a contract that must be maintained between the container image run in the pod and the
+infrastructure that launched the pod. Details like paths, users/groups, permissions, and
+environment variables must align so that the deployment layer can pass runtime concerns down to
+the container.
+
+Ideally this image would start from some baseline image provided by the HBase project and extend
+it as necissary according to the Kubernetes deployment needs. Such an image is not yet provided by
+HBase, so this image serves dual purposes.
+
+## Build
+
+Start by downloading all the relevant binaries for your platform and naming them appropriately,
+see comments in [docker-bake.override.hcl](./docker-bake.override.hcl) for details.
+
+Next, create a buildx context that supports (optionally) multi-platform images. If you've created
+this context previously, it's enough to ensure that it's active via `docker buildx ls`.
+
+```shell
+$ docker buildx create \
+  --driver docker-container \
+  --platform linux/amd64,linux/arm64 \
+  --use \
+  --bootstrap
+```
+
+Finally, build the image using,
+
+```shell
+$ docker buildx bake \
+  --file dockerfiles/kuttl/docker-bake.hcl \
+  --file dockerfiles/kuttl/docker-bake.override.hcl \
+  --pull \
+  --load
+```
+
+This exports an image to your local repository that is tagged as
+`${USER}/hbase/operator-tools/hbase:latest`.

hbase-kubernetes-deployment/dockerfiles/hbase/docker-bake.hcl

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# A convenience script for build the hbase image.
+# See hbase-kubernetes-deployment/dockerfiles/hbase/README.md
+#
+
+# input variables
+variable HBASE_TGZ {}
+variable HBASE_TGZ_SHA512 {}
+variable HBASE_OPERATOR_TOOLS_TGZ {}
+variable HBASE_OPERATOR_TOOLS_TGZ_SHA512 {}
+variable JMX_PROMETHEUS_JAR {}
+
+# output variables
+variable USER {}
+variable IMAGE_TAG {
+  default = "latest"
+}
+variable IMAGE_NAME {
+  default = "${USER}/hbase/operator-tools/hbase"
+}
+
+group default {
+  targets = [ "hbase" ]
+}
+
+target hbase {
+  dockerfile = "hbase-kubernetes-deployment/dockerfiles/hbase/Dockerfile"
+  args = {
+    HBASE_TGZ = HBASE_TGZ
+    HBASE_TGZ_SHA512 = HBASE_TGZ_SHA512
+    HBASE_OPERATOR_TOOLS_TGZ = HBASE_OPERATOR_TOOLS_TGZ
+    HBASE_OPERATOR_TOOLS_TGZ_SHA512 = HBASE_OPERATOR_TOOLS_TGZ_SHA512
+    JMX_PROMETHEUS_JAR = JMX_PROMETHEUS_JAR
+  }
+  platforms = [
+    "linux/amd64",
+    "linux/arm64"
+  ]
+  tags = [ "${IMAGE_NAME}:${IMAGE_TAG}" ]
+}

hbase-kubernetes-deployment/dockerfiles/hbase/docker-bake.override.hcl

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Externalize default values of build parameters and document how to retrieve them.
+#
+
+variable HBASE_TGZ {
+  # wget 'https://downloads.apache.org/hbase/2.5.4/hbase-2.5.4-hadoop3-bin.tar.gz'
+  default = "hbase-2.5.4-hadoop3-bin.tar.gz"
+}
+
+variable HBASE_TGZ_SHA512 {
+  # wget 'https://downloads.apache.org/hbase/2.5.4/hbase-2.5.4-hadoop3-bin.tar.gz.sha512'
+  default = "hbase-2.5.4-hadoop3-bin.tar.gz.sha512"
+}
+
+variable HBASE_OPERATOR_TOOLS_TGZ {
+  # wget 'https://downloads.apache.org/hbase/hbase-operator-tools-1.2.0/hbase-operator-tools-1.2.0-bin.tar.gz'
+  default = "hbase-operator-tools-1.2.0-bin.tar.gz"
+}
+
+variable HBASE_OPERATOR_TOOLS_TGZ_SHA512 {
+  # wget 'https://downloads.apache.org/hbase/hbase-operator-tools-1.2.0/hbase-operator-tools-1.2.0-bin.tar.gz.sha512'
+  default = "hbase-operator-tools-1.2.0-bin.tar.gz.sha512"
+}
+
+variable JMX_PROMETHEUS_JAR {
+  # wget 'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar'
+  default = "jmx_prometheus_javaagent-0.16.1.jar"
+}

hbase-kubernetes-deployment/dockerfiles/hbase/hbck2

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Convenience script for launching hbck2
+
+HBCK2_JAR="$(ls -1 "${HBASE_OPERATOR_TOOLS_HOME}"/hbase-hbck2/hbase-hbck2-*.jar)"
+
+# Redefine HBASE_CONF_DIR from /etc/hbase to /etc/hbck2
+# hbck2 depends on log4j1. hbase 2.5.0+ depends on log4j2. If a log4j2.properites in classpath,
+# the process won't launch.
+export HBASE_CONF_DIR=/etc/hbck2
+
+exec hbase hbck -j "${HBCK2_JAR}" "$@"