From b9a44de4998b07228698dba022b15bdfdb3be590 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Tue, 12 Jan 2021 12:39:56 +0530 Subject: [PATCH 01/17] HADOOP-17465. Upgrade to Focal * We are installing a lot of recent versions of the libraries as those provided by the Bionic toolchain isn't recent enough. Thus, it's better to move to Focal for better maintainability. --- dev-support/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 7f47303002a34..eaf808434d729 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -18,7 +18,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM ubuntu:bionic +FROM ubuntu:focal WORKDIR /root From 23f9922ac35def8844586f796a6913c18e607a2e Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Tue, 12 Jan 2021 14:17:30 +0530 Subject: [PATCH 02/17] Install python-pip separately * python-pip is removed from the Focal toolchain as it's old. Thus, we need to install it separately. --- dev-support/docker/Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index eaf808434d729..a70f9c26b38ce 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -71,10 +71,8 @@ RUN apt-get -q update \ pkg-config \ python \ python2.7 \ - python-pip \ python-pkg-resources \ python-setuptools \ - python-wheel \ rsync \ shellcheck \ software-properties-common \ @@ -92,6 +90,14 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV FINDBUGS_HOME /usr +RUN mkdir -p /opt/pip \ + && curl -L https://bootstrap.pypa.io/get-pip.py > get-pip.py \ + && mv get-pip.py /opt/pip \ + && cd /opt/pip \ + && python2.7 get-pip.py \ + && cd /root \ + && rm -rf /opt/pip + ####### # Install Boost 1.72 (1.65 ships with Bionic) ####### From 8085cb5fc1b68648a2ee60e0f45e828b897af207 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 13 Jan 2021 10:24:21 +0530 Subject: [PATCH 03/17] Use Boost from Focal toolchain * Removing separate installation of Boost and using the one provided by Focal instead. --- dev-support/docker/Dockerfile | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index a70f9c26b38ce..009e7ba53cc03 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -52,6 +52,7 @@ RUN apt-get -q update \ git \ gnupg-agent \ libbcprov-java \ + libboost-all-dev \ libbz2-dev \ libcurl4-openssl-dev \ libfuse-dev \ @@ -90,6 +91,7 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV FINDBUGS_HOME /usr +# hadolint ignore=DL3003 RUN mkdir -p /opt/pip \ && curl -L https://bootstrap.pypa.io/get-pip.py > get-pip.py \ && mv get-pip.py /opt/pip \ @@ -98,21 +100,6 @@ RUN mkdir -p /opt/pip \ && cd /root \ && rm -rf /opt/pip -####### -# Install Boost 1.72 (1.65 ships with Bionic) -####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - ###### # Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic) ###### From fda981a4c72eed1bafc1458e963bc823cd55c6d4 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 13 Jan 2021 12:52:14 +0530 Subject: [PATCH 04/17] Use Java protobuf from Focal toolchain * Using libprotobuf-java provided by the Focal toolchain instead of installing it separately. --- dev-support/docker/Dockerfile | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 009e7ba53cc03..c1e0236435875 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -57,6 +57,7 @@ RUN apt-get -q update \ libcurl4-openssl-dev \ libfuse-dev \ libprotobuf-dev \ + libprotobuf-java \ libprotoc-dev \ libsasl2-dev \ libsnappy-dev \ @@ -90,6 +91,8 @@ ENV MAVEN_HOME /usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV FINDBUGS_HOME /usr +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" # hadolint ignore=DL3003 RUN mkdir -p /opt/pip \ @@ -100,23 +103,6 @@ RUN mkdir -p /opt/pip \ && cd /root \ && rm -rf /opt/pip -###### -# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" - #### # Install pylint at fixed version (2.0.0 removed python2 support) # https://github.com/PyCQA/pylint/issues/2294 From ad8f707628f12fd066289934e44280e2e0f7d54d Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 13 Jan 2021 13:12:41 +0530 Subject: [PATCH 05/17] Use nodejs from Focal toolchain * Removing the nodejs installed separately and using the one provided by Focal toolchain instead. --- dev-support/docker/Dockerfile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index c1e0236435875..66ca426823603 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -67,6 +67,7 @@ RUN apt-get -q update \ locales \ make \ maven \ + nodejs \ openjdk-11-jdk \ openjdk-8-jdk \ pinentry-curses \ @@ -119,14 +120,10 @@ RUN pip2 install \ RUN pip2 install python-dateutil==2.7.3 ### -# Install node.js 10.x for web UI framework (4.2.6 ships with Xenial) +# Install bower ### # hadolint ignore=DL3008 -RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \ - && apt-get install -y --no-install-recommends nodejs \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && npm install -g bower@1.8.8 +RUN npm install -g bower@1.8.8 ### ## Install Yarn 1.22.5 for web UI framework From b659d6f7e958094218fe93ddd5118ea6aade9eff Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 13 Jan 2021 13:20:01 +0530 Subject: [PATCH 06/17] Use node-yarn from Focal toolchain * Removing the separate installation of yarnpkg and using the one provided by the Focal toolchain instead. --- dev-support/docker/Dockerfile | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 66ca426823603..e8f68512ebff2 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -68,6 +68,7 @@ RUN apt-get -q update \ make \ maven \ nodejs \ + node-yarn \ openjdk-11-jdk \ openjdk-8-jdk \ pinentry-curses \ @@ -125,16 +126,6 @@ RUN pip2 install python-dateutil==2.7.3 # hadolint ignore=DL3008 RUN npm install -g bower@1.8.8 -### -## Install Yarn 1.22.5 for web UI framework -#### -RUN curl -s -S https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ - && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends yarn=1.22.5-1 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - ### # Install hadolint #### From a59bf66274eef7f545f3f463d4cd8d2446425fc9 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 13 Jan 2021 13:23:59 +0530 Subject: [PATCH 07/17] Use hugo from Focal toolchain * Removing the separate installation of hugo and using the one provided by the Focal toolchain instead. --- dev-support/docker/Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index e8f68512ebff2..2de950791b5b4 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -51,6 +51,7 @@ RUN apt-get -q update \ gcc \ git \ gnupg-agent \ + hugo \ libbcprov-java \ libboost-all-dev \ libbz2-dev \ @@ -170,12 +171,6 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true # YETUS CUT HERE ### -# Hugo static website generator for new hadoop site -RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-64bit.deb \ - && dpkg --install hugo.deb \ - && rm hugo.deb - - # Add a welcome message and environment checks. COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh RUN chmod 755 /root/hadoop_env_checks.sh From 013ead24a91260aafcb5993471d3b582ac26f739 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Wed, 13 Jan 2021 13:52:19 +0530 Subject: [PATCH 08/17] Use npm from Focal toolchain * Using npm from Focal toolchain. --- dev-support/docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 2de950791b5b4..53363e8d06135 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -70,6 +70,7 @@ RUN apt-get -q update \ maven \ nodejs \ node-yarn \ + npm \ openjdk-11-jdk \ openjdk-8-jdk \ pinentry-curses \ From 37ae7b19a04f81f2fa5cccf69e03a20f0f91ce3c Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Wed, 13 Jan 2021 16:04:54 +0530 Subject: [PATCH 09/17] Add comment for pip * Added a comment describing the reason for pip installation. --- dev-support/docker/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 53363e8d06135..a9acd16d28867 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -98,6 +98,9 @@ ENV FINDBUGS_HOME /usr ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" +#### +# Install pip (deprecated from Focal toolchain) +#### # hadolint ignore=DL3003 RUN mkdir -p /opt/pip \ && curl -L https://bootstrap.pypa.io/get-pip.py > get-pip.py \ @@ -122,9 +125,9 @@ RUN pip2 install \ #### RUN pip2 install python-dateutil==2.7.3 -### +#### # Install bower -### +#### # hadolint ignore=DL3008 RUN npm install -g bower@1.8.8 From e7b5b235939a08e59fe0219bf89591f9c7b11211 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 13 Jan 2021 23:56:25 +0530 Subject: [PATCH 10/17] Update docker aarch64 to use Focal * Updated docker file for aarch64 to use the Focal image. Also, removed the packages that were installed separately. --- dev-support/docker/Dockerfile | 1 - dev-support/docker/Dockerfile_aarch64 | 80 ++++++++------------------- 2 files changed, 23 insertions(+), 58 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index a9acd16d28867..ac3d2a8263296 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -1,4 +1,3 @@ - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 0c4db95537e3a..9a4c358006dce 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM ubuntu:bionic +FROM ubuntu:focal WORKDIR /root @@ -54,11 +54,14 @@ RUN apt-get -q update \ gcc \ git \ gnupg-agent \ + hugo \ libbcprov-java \ + libboost-all-dev \ libbz2-dev \ libcurl4-openssl-dev \ libfuse-dev \ libprotobuf-dev \ + libprotobuf-java \ libprotoc-dev \ libsasl2-dev \ libsnappy-dev \ @@ -68,16 +71,17 @@ RUN apt-get -q update \ locales \ make \ maven \ + nodejs \ + node-yarn \ + npm \ openjdk-11-jdk \ openjdk-8-jdk \ pinentry-curses \ pkg-config \ python \ python2.7 \ - python-pip \ python-pkg-resources \ python-setuptools \ - python-wheel \ rsync \ shellcheck \ software-properties-common \ @@ -92,40 +96,22 @@ RUN apt-get -q update \ ###### ENV MAVEN_HOME /usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV FINDBUGS_HOME /usr +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" -####### -# Install Boost 1.72 (1.65 ships with Bionic) -####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic) -###### +#### +# Install pip (deprecated from Focal toolchain) +#### # hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make install \ +RUN mkdir -p /opt/pip \ + && curl -L https://bootstrap.pypa.io/get-pip.py > get-pip.py \ + && mv get-pip.py /opt/pip \ + && cd /opt/pip \ + && python2.7 get-pip.py \ && cd /root \ - && rm -rf /opt/protobuf-src -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" + && rm -rf /opt/pip #### # Install pylint at fixed version (2.0.0 removed python2 support) @@ -142,25 +128,11 @@ RUN pip2 install \ #### RUN pip2 install python-dateutil==2.7.3 -### -# Install node.js 10.x for web UI framework (4.2.6 ships with Xenial) -### -# hadolint ignore=DL3008 -RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \ - && apt-get install -y --no-install-recommends nodejs \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && npm install -g bower@1.8.8 - -### -## Install Yarn 1.22.5 for web UI framework #### -RUN curl -s -S https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ - && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends yarn=1.22.5-1 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +# Install bower +#### +# hadolint ignore=DL3008 +RUN npm install -g bower@1.8.8 ### # Install phantomjs built for aarch64 @@ -187,12 +159,6 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true # YETUS CUT HERE ### -# Hugo static website generator (for new hadoop site docs) -RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-ARM64.deb \ - && dpkg --install hugo.deb \ - && rm hugo.deb - - # Add a welcome message and environment checks. COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh RUN chmod 755 /root/hadoop_env_checks.sh From d16aea4fd71cb2ac170945bb4be1863abf9cfb01 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 14 Jan 2021 00:01:25 +0530 Subject: [PATCH 11/17] Touch files to run tests * Modifying files in HDFS and YARN to force the tests to run on this PR. --- .../src/main/native/libhdfspp/lib/fs/filesystem.h | 1 - .../hadoop/yarn/server/nodemanager/NodeResourceMonitor.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h index 935e7c96c7b02..671b1c8f9a645 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h @@ -242,7 +242,6 @@ class FileSystemImpl : public FileSystem { void FindShim(const Status &stat, const std::vector & stat_infos, bool directory_has_more, std::shared_ptr current_state, std::shared_ptr shared_state); - }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java index 9d8b80d8d2712..4c457068ffdb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java @@ -25,6 +25,7 @@ * Interface for monitoring the resources of a node. */ public interface NodeResourceMonitor extends Service { + /** * Get the resource utilization of the node. * @return resource utilization of the node. From b91b923c419b71d89135578808ae34b5637aaf25 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 14 Jan 2021 08:08:17 +0530 Subject: [PATCH 12/17] Downgrade Boost version to 1.71.0 * Focal toolchain provides Boost version 1.71.0. Hence, using the same. --- .../src/main/native/libhdfspp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 6528fa8897279..0ff8f58006539 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -28,7 +28,7 @@ project (libhdfspp) cmake_minimum_required(VERSION 2.8) -find_package (Boost 1.72.0 REQUIRED) +find_package (Boost 1.71.0 REQUIRED) enable_testing() include (CTest) From 3076f213cb8432707729e7533797c1dc802cac33 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Thu, 14 Jan 2021 10:41:14 +0530 Subject: [PATCH 13/17] Touch file to build * Touching the file to retrigger the build. --- .../src/main/native/libhdfspp/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 0ff8f58006539..17d2d9bd312aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -27,7 +27,6 @@ project (libhdfspp) cmake_minimum_required(VERSION 2.8) - find_package (Boost 1.71.0 REQUIRED) enable_testing() From b1d2f16c3516f2e0ac56fa4882c478961a66ec24 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 14 Jan 2021 11:09:04 +0530 Subject: [PATCH 14/17] Install Boost separately * Build is failing with Boost 1.71. Trying to install Boost 1.72 separately. --- dev-support/docker/Dockerfile | 16 +++++++++++++++- dev-support/docker/Dockerfile_aarch64 | 16 +++++++++++++++- .../src/main/native/libhdfspp/CMakeLists.txt | 3 ++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index ac3d2a8263296..b02aaa4029b63 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -52,7 +52,6 @@ RUN apt-get -q update \ gnupg-agent \ hugo \ libbcprov-java \ - libboost-all-dev \ libbz2-dev \ libcurl4-openssl-dev \ libfuse-dev \ @@ -97,6 +96,21 @@ ENV FINDBUGS_HOME /usr ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" +####### +# Install Boost 1.72 (1.71 ships with Focal) +####### +# hadolint ignore=DL3003 +RUN mkdir -p /opt/boost-library \ + && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ + && mv boost_1_72_0.tar.bz2 /opt/boost-library \ + && cd /opt/boost-library \ + && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ + && cd /opt/boost-library/boost_1_72_0 \ + && ./bootstrap.sh --prefix=/usr/ \ + && ./b2 --without-python install \ + && cd /root \ + && rm -rf /opt/boost-library + #### # Install pip (deprecated from Focal toolchain) #### diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 9a4c358006dce..1dec45bf224ca 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -56,7 +56,6 @@ RUN apt-get -q update \ gnupg-agent \ hugo \ libbcprov-java \ - libboost-all-dev \ libbz2-dev \ libcurl4-openssl-dev \ libfuse-dev \ @@ -101,6 +100,21 @@ ENV FINDBUGS_HOME /usr ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" +####### +# Install Boost 1.72 (1.71 ships with Focal) +####### +# hadolint ignore=DL3003 +RUN mkdir -p /opt/boost-library \ + && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ + && mv boost_1_72_0.tar.bz2 /opt/boost-library \ + && cd /opt/boost-library \ + && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ + && cd /opt/boost-library/boost_1_72_0 \ + && ./bootstrap.sh --prefix=/usr/ \ + && ./b2 --without-python install \ + && cd /root \ + && rm -rf /opt/boost-library + #### # Install pip (deprecated from Focal toolchain) #### diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 17d2d9bd312aa..6528fa8897279 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -27,7 +27,8 @@ project (libhdfspp) cmake_minimum_required(VERSION 2.8) -find_package (Boost 1.71.0 REQUIRED) + +find_package (Boost 1.72.0 REQUIRED) enable_testing() include (CTest) From 72b4ac25e5419cc256febd65f334fab0e8f5f194 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 14 Jan 2021 14:25:15 +0530 Subject: [PATCH 15/17] Revert "Use Java protobuf from Focal toolchain" This reverts commit 4af90ad8437acf1eeb852f42f61bd52e7eac0aa6. --- dev-support/docker/Dockerfile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index b02aaa4029b63..b5a82d99342de 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -56,7 +56,6 @@ RUN apt-get -q update \ libcurl4-openssl-dev \ libfuse-dev \ libprotobuf-dev \ - libprotobuf-java \ libprotoc-dev \ libsasl2-dev \ libsnappy-dev \ @@ -93,8 +92,6 @@ ENV MAVEN_HOME /usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV FINDBUGS_HOME /usr -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" ####### # Install Boost 1.72 (1.71 ships with Focal) @@ -123,6 +120,23 @@ RUN mkdir -p /opt/pip \ && cd /root \ && rm -rf /opt/pip +###### +# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic) +###### +# hadolint ignore=DL3003 +RUN mkdir -p /opt/protobuf-src \ + && curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz \ + && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ + && cd /opt/protobuf-src \ + && ./configure --prefix=/opt/protobuf \ + && make install \ + && cd /root \ + && rm -rf /opt/protobuf-src +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + #### # Install pylint at fixed version (2.0.0 removed python2 support) # https://github.com/PyCQA/pylint/issues/2294 From 76f361699fe59bb398d03f32062444f372f4e634 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Fri, 22 Jan 2021 23:52:19 +0530 Subject: [PATCH 16/17] Revert "Touch files to run tests" This reverts commit d16aea4fd71cb2ac170945bb4be1863abf9cfb01. --- .../src/main/native/libhdfspp/lib/fs/filesystem.h | 1 + .../hadoop/yarn/server/nodemanager/NodeResourceMonitor.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h index 671b1c8f9a645..935e7c96c7b02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.h @@ -242,6 +242,7 @@ class FileSystemImpl : public FileSystem { void FindShim(const Status &stat, const std::vector & stat_infos, bool directory_has_more, std::shared_ptr current_state, std::shared_ptr shared_state); + }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java index 4c457068ffdb2..9d8b80d8d2712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java @@ -25,7 +25,6 @@ * Interface for monitoring the resources of a node. */ public interface NodeResourceMonitor extends Service { - /** * Get the resource utilization of the node. * @return resource utilization of the node. From 6027c1fd5cd88035f468e818c087da3131110c76 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Fri, 22 Jan 2021 23:59:52 +0530 Subject: [PATCH 17/17] Install Java Protobuf for aarch64 * Need to install Java Protobuf library separately for aarch64 docker builder image. --- dev-support/docker/Dockerfile | 2 +- dev-support/docker/Dockerfile_aarch64 | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index b5a82d99342de..60873f04652cd 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -121,7 +121,7 @@ RUN mkdir -p /opt/pip \ && rm -rf /opt/pip ###### -# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic) +# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) ###### # hadolint ignore=DL3003 RUN mkdir -p /opt/protobuf-src \ diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 1dec45bf224ca..25e9526dee051 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -60,7 +60,6 @@ RUN apt-get -q update \ libcurl4-openssl-dev \ libfuse-dev \ libprotobuf-dev \ - libprotobuf-java \ libprotoc-dev \ libsasl2-dev \ libsnappy-dev \ @@ -95,10 +94,8 @@ RUN apt-get -q update \ ###### ENV MAVEN_HOME /usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 ENV FINDBUGS_HOME /usr -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" ####### # Install Boost 1.72 (1.71 ships with Focal) @@ -127,6 +124,23 @@ RUN mkdir -p /opt/pip \ && cd /root \ && rm -rf /opt/pip +###### +# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) +###### +# hadolint ignore=DL3003 +RUN mkdir -p /opt/protobuf-src \ + && curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz \ + && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ + && cd /opt/protobuf-src \ + && ./configure --prefix=/opt/protobuf \ + && make install \ + && cd /root \ + && rm -rf /opt/protobuf-src +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + #### # Install pylint at fixed version (2.0.0 removed python2 support) # https://github.com/PyCQA/pylint/issues/2294