From 2a46d9cada6d7c48c1ca935e65d6ce94306b8589 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 1 Sep 2025 20:40:36 +0800 Subject: [PATCH] dev box riscv64 --- .../docker/Dockerfile_ubuntu_24_riscv64 | 104 ++++++++++++++++++ dev-support/docker/hadoop_env_checks.sh | 23 ++++ .../docker/pkg-resolver/install-protobuf.sh | 20 ++-- dev-support/docker/pkg-resolver/packages.json | 63 ++++++++++- .../docker/pkg-resolver/platforms.json | 1 + hadoop-common-project/hadoop-common/pom.xml | 6 +- start-build-env.sh | 14 ++- 7 files changed, 214 insertions(+), 17 deletions(-) create mode 100644 dev-support/docker/Dockerfile_ubuntu_24_riscv64 diff --git a/dev-support/docker/Dockerfile_ubuntu_24_riscv64 b/dev-support/docker/Dockerfile_ubuntu_24_riscv64 new file mode 100644 index 0000000000000..d5b77b1f1c864 --- /dev/null +++ b/dev-support/docker/Dockerfile_ubuntu_24_riscv64 @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM ubuntu:noble + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_TERSE=true + +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:noble::riscv64) \ + && apt-get clean \ + && update-java-alternatives -s java-1.17.0-openjdk-riscv64 \ + && rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +ENV PYTHONIOENCODING=utf-8 + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME=/opt/maven +ENV PATH="${PATH}:${MAVEN_HOME}/bin" +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-riscv64 + +####### +# Set env vars for SpotBugs 4.2.2 +####### +ENV SPOTBUGS_HOME=/opt/spotbugs + +####### +# Set env vars for Google Protobuf 3.25.5 +####### +ENV PROTOBUF_HOME=/opt/protobuf +ENV PATH="${PATH}:/opt/protobuf/bin" + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS="-Xms256m -Xmx3072m" + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION=true + +#### +# Install packages +#### +RUN pkg-resolver/install-maven.sh ubuntu:noble::riscv64 +RUN pkg-resolver/install-spotbugs.sh ubuntu:noble::riscv64 +RUN pkg-resolver/install-boost.sh ubuntu:noble::riscv64 +RUN pkg-resolver/install-protobuf.sh ubuntu:noble::riscv64 +RUN pkg-resolver/install-hadolint.sh ubuntu:noble::riscv64 + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/hadoop_env_checks.sh b/dev-support/docker/hadoop_env_checks.sh index c9420e79e9aad..eca9eeda6d00a 100755 --- a/dev-support/docker/hadoop_env_checks.sh +++ b/dev-support/docker/hadoop_env_checks.sh @@ -110,8 +110,31 @@ End-of-message # ------------------------------------------------------- +function tips { + CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1) + if [[ "$CPU_ARCH" == "riscv64" ]]; then + cat < aarch64 - false - aarch64 + !x86_64 @@ -1195,9 +1194,8 @@ x86_64 - false - !aarch64 + x86_64 diff --git a/start-build-env.sh b/start-build-env.sh index 4f5f85947e317..aaf0051834f4f 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -25,6 +25,7 @@ OS_PLATFORM="${1:-}" DEFAULT_OS_PLATFORM="ubuntu_20" OS_PLATFORM_SUFFIX="" +DOCKER_PLATFORM_ARGS="" if [[ -n ${OS_PLATFORM} ]] && [[ "${OS_PLATFORM}" != "${DEFAULT_OS_PLATFORM}" ]]; then # ubuntu_20 (default) platform does not have suffix in Dockerfile. @@ -34,9 +35,14 @@ fi DOCKER_DIR=dev-support/docker DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}" -CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1) +CPU_ARCH=${CPU_ARCH:-$(echo "$MACHTYPE" | cut -d- -f1)} if [[ "$CPU_ARCH" == "aarch64" || "$CPU_ARCH" == "arm64" ]]; then DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}_aarch64" +elif [[ "$CPU_ARCH" == "riscv64" ]]; then + DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}_riscv64" + # Enable cross-platform emulator on non-riscv64 platform by running: + # docker run --rm --privileged tonistiigi/binfmt --install riscv64 + DOCKER_PLATFORM_ARGS="--platform linux/riscv64" fi if [ ! -e "${DOCKER_FILE}" ] ; then @@ -44,7 +50,7 @@ if [ ! -e "${DOCKER_FILE}" ] ; then exit 1 fi -docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR +docker build ${DOCKER_PLATFORM_ARGS} -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR USER_NAME=${SUDO_USER:=$USER} USER_ID=$(id -u "${USER_NAME}") @@ -86,7 +92,7 @@ fi # Set the home directory in the Docker container. DOCKER_HOME_DIR=${DOCKER_HOME_DIR:-/home/${USER_NAME}} -docker build -t "hadoop-build${OS_PLATFORM_SUFFIX}-${USER_ID}" - </dev/null || : @@ -105,7 +111,7 @@ DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"} # within the container and use the result on your normal # system. And this also is a significant speedup in subsequent # builds because the dependencies are downloaded only once. -docker run --rm=true $DOCKER_INTERACTIVE_RUN \ +docker run ${DOCKER_PLATFORM_ARGS} --rm=true ${DOCKER_INTERACTIVE_RUN} \ -v "${PWD}:${DOCKER_HOME_DIR}/hadoop${V_OPTS:-}" \ -w "${DOCKER_HOME_DIR}/hadoop" \ -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \