merge

BUILDING.txt

@@ -6,6 +6,7 @@ Requirements:
 * Unix System
 * JDK 1.8
 * Maven 3.3 or later
+* Boost 1.72 (if compiling native code)
 * Protocol Buffers 3.7.1 (if compiling native code)
 * CMake 3.1 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
@@ -72,6 +73,12 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
         && ./configure\
         && make install \
         && rm -rf /opt/protobuf-3.7-src
+* Boost
+  $ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
+    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
+    && cd boost_1_72_0 \
+    && ./bootstrap.sh --prefix=/usr/ \
+    && ./b2 --without-python install
 
 Optional packages:
 
@@ -468,6 +475,7 @@ Requirements:
 * Windows System
 * JDK 1.8
 * Maven 3.0 or later
+* Boost 1.72
 * Protocol Buffers 3.7.1
 * CMake 3.1 or newer
 * Visual Studio 2010 Professional or Higher

Jenkinsfile

@@ -23,7 +23,7 @@ pipeline {
 
     options {
         buildDiscarder(logRotator(numToKeepStr: '5'))
-        timeout (time: 5, unit: 'HOURS')
+        timeout (time: 20, unit: 'HOURS')
         timestamps()
         checkoutToSubdirectory('src')
     }
@@ -154,6 +154,11 @@ pipeline {
                         # use emoji vote so it is easier to find the broken line
                         YETUS_ARGS+=("--github-use-emoji-vote")
 
+                        # test with Java 8 and 11
+                        YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
+                        YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
+                        YETUS_ARGS+=("--multijdktests=compile")
+
                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
                         '''
                 }

dev-support/docker/Dockerfile

@@ -33,16 +33,10 @@ RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
-######
-# Install common dependencies from packages. Versions here are either
-# sufficient or irrelevant.
-#
-# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
-# Ubuntu Java.  See Java section below!
-######
 # hadolint ignore=DL3008
 RUN apt-get -q update \
     && apt-get -q install -y --no-install-recommends \
+        ant \
         apt-utils \
         bats \
         build-essential \
@@ -51,11 +45,13 @@ RUN apt-get -q update \
         cmake \
         curl \
         doxygen \
+        findbugs \
         fuse \
         g++ \
         gcc \
         git \
         gnupg-agent \
+        libbcprov-java \
         libbz2-dev \
         libcurl4-openssl-dev \
         libfuse-dev \
@@ -64,11 +60,13 @@ RUN apt-get -q update \
         libsasl2-dev \
         libsnappy-dev \
         libssl-dev \
-        libsnappy-dev \
         libtool \
         libzstd1-dev \
         locales \
         make \
+        maven \
+        openjdk-11-jdk \
+        openjdk-8-jdk \
         pinentry-curses \
         pkg-config \
         python \
@@ -86,15 +84,28 @@ RUN apt-get -q update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /usr
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV FINDBUGS_HOME /usr
 
 #######
-# OpenJDK 8
+# Install Boost 1.72 (1.65 ships with Bionic)
 #######
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/boost-library \
+    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
+    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
+    && cd /opt/boost-library \
+    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
+    && cd /opt/boost-library/boost_1_72_0 \
+    && ./bootstrap.sh --prefix=/usr/ \
+    && ./b2 --without-python install \
+    && cd /root \
+    && rm -rf /opt/boost-library
 
 ######
 # Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic)
@@ -113,29 +124,6 @@ RUN mkdir -p /opt/protobuf-src \
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-######
-# Install Apache Maven 3.6.0 (3.6.0 ships with Bionic)
-######
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends maven \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-ENV MAVEN_HOME /usr
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-
-######
-# Install findbugs 3.1.0 (3.1.0 ships with Bionic)
-# Ant is needed for findbugs
-######
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends findbugs ant \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-ENV FINDBUGS_HOME /usr
-
 ####
 # Install pylint at fixed version (2.0.0 removed python2 support)
 # https://github.com/PyCQA/pylint/issues/2294
@@ -150,10 +138,10 @@ RUN pip2 install \
 RUN pip2 install python-dateutil==2.7.3
 
 ###
-# Install node.js 8.17.0 for web UI framework (4.2.6 ships with Xenial)
+# Install node.js 10.21.0 for web UI framework (4.2.6 ships with Xenial)
 ###
-RUN curl -L -s -S https://deb.nodesource.com/setup_8.x | bash - \
-    && apt-get install -y --no-install-recommends nodejs=8.17.0-1nodesource1 \
+RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs=10.21.0-1nodesource1 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
     && npm install -g [email protected]

dev-support/docker/Dockerfile_aarch64

@@ -17,7 +17,7 @@
 # Dockerfile for installing the necessary dependencies for building Hadoop.
 # See BUILDING.txt.
 
-FROM ubuntu:xenial
+FROM ubuntu:bionic
 
 WORKDIR /root
 
@@ -35,24 +35,26 @@ ENV DEBCONF_TERSE true
 ######
 # Install common dependencies from packages. Versions here are either
 # sufficient or irrelevant.
-#
-# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
-# Ubuntu Java.  See Java section below!
 ######
 # hadolint ignore=DL3008
 RUN apt-get -q update \
     && apt-get -q install -y --no-install-recommends \
+        ant \
         apt-utils \
+        bats \
         build-essential \
         bzip2 \
         clang \
+        cmake \
         curl \
         doxygen \
+        findbugs \
         fuse \
         g++ \
         gcc \
         git \
         gnupg-agent \
+        libbcprov-java \
         libbz2-dev \
         libcurl4-openssl-dev \
         libfuse-dev \
@@ -65,6 +67,9 @@ RUN apt-get -q update \
         libzstd1-dev \
         locales \
         make \
+        maven \
+        openjdk-11-jdk \
+        openjdk-8-jdk \
         pinentry-curses \
         pkg-config \
         python \
@@ -74,47 +79,39 @@ RUN apt-get -q update \
         python-setuptools \
         python-wheel \
         rsync \
+        shellcheck \
         software-properties-common \
-        snappy \
         sudo \
         valgrind \
         zlib1g-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /usr
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
+ENV FINDBUGS_HOME /usr
 
 #######
-# OpenJDK 8
+# Install Boost 1.72 (1.65 ships with Bionic)
 #######
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-
-######
-# Install cmake 3.1.0 (3.5.1 ships with Xenial)
-# There is no cmake binary available for aarch64. Build from source.
-######
 # hadolint ignore=DL3003
-RUN mkdir -p /opt/cmake/src \
-    && curl -L -s -S \
-      https://cmake.org/files/v3.1/cmake-3.1.0-1-src.tar.bz2 \
-      -o /opt/cmake/cmake-src.tar.bz2 \
-    && tar xvjf /opt/cmake/cmake-src.tar.bz2 -C /opt/cmake/src \
-    && cd /opt/cmake/src \
-    && tar xvjf cmake-3.1.0.tar.bz2 \
-    && cd cmake-3.1.0 && patch -p0 -i ../cmake-3.1.0-1.patch && mkdir .build && cd .build \
-    && ../bootstrap --parallel=2 \
-    && make -j2 && ./bin/cpack \
-    && tar xzf cmake-3.1.0-Linux-aarch64.tar.gz --strip-components 1 -C /opt/cmake \
-    && cd /opt/cmake && rm -rf /opt/cmake/src
-ENV CMAKE_HOME /opt/cmake
-ENV PATH "${PATH}:/opt/cmake/bin"
+RUN mkdir -p /opt/boost-library \
+    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
+    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
+    && cd /opt/boost-library \
+    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
+    && cd /opt/boost-library/boost_1_72_0 \
+    && ./bootstrap.sh --prefix=/usr/ \
+    && ./b2 --without-python install \
+    && cd /root \
+    && rm -rf /opt/boost-library
 
 ######
-# Install Google Protobuf 3.7.1 (2.6.0 ships with Xenial)
+# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic)
 ######
 # hadolint ignore=DL3003
 RUN mkdir -p /opt/protobuf-src \
@@ -130,46 +127,6 @@ RUN mkdir -p /opt/protobuf-src \
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-######
-# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
-######
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends maven \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-ENV MAVEN_HOME /usr
-
-######
-# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
-# Ant is needed for findbugs
-######
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends findbugs ant \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-ENV FINDBUGS_HOME /usr
-
-####
-# Install shellcheck (0.4.6, the latest as of 2017-09-26)
-####
-# hadolint ignore=DL3008
-RUN add-apt-repository -y ppa:hvr/ghc \
-    && apt-get -q update \
-    && apt-get -q install -y --no-install-recommends shellcheck \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-####
-# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
-####
-# hadolint ignore=DL3008
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends bats \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
 ####
 # Install pylint at fixed version (2.0.0 removed python2 support)
 # https://github.com/PyCQA/pylint/issues/2294
@@ -184,10 +141,10 @@ RUN pip2 install \
 RUN pip2 install python-dateutil==2.7.3
 
 ###
-# Install node.js 8.17.0 for web UI framework (4.2.6 ships with Xenial)
+# Install node.js 10.21.0 for web UI framework (4.2.6 ships with Xenial)
 ###
-RUN curl -L -s -S https://deb.nodesource.com/setup_8.x | bash - \
-    && apt-get install -y --no-install-recommends nodejs=8.17.0-1nodesource1 \
+RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs=10.21.0-1nodesource1 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
     && npm install -g [email protected]

hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml

@@ -47,6 +47,14 @@
       <outputDirectory>/libexec/shellprofile.d</outputDirectory>
       <fileMode>0755</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>../hadoop-federation-balance/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>../hadoop-extras/src/main/shellprofile.d</directory>
       <includes>
@@ -111,6 +119,13 @@
         <include>*-sources.jar</include>
       </includes>
     </fileSet>
+    <fileSet>
+      <directory>../hadoop-federation-balance/target</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+      <includes>
+        <include>*-sources.jar</include>
+      </includes>
+    </fileSet>
     <fileSet>
       <directory>../hadoop-extras/target</directory>
       <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>

hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml

@@ -119,7 +119,12 @@
 
         <!-- Checks for imports                              -->
         <!-- See http://checkstyle.sf.net/config_import.html -->
-        <module name="IllegalImport"/> <!-- defaults to sun.* packages -->
+        <module name="IllegalImport">
+          <property name="regexp" value="true"/>
+          <property name="illegalPkgs" value="^sun\.[^.]+"/>
+          <property name="illegalClasses"
+            value="^com\.google\.common\.base\.(Optional|Function|Predicate|Supplier), ^com\.google\.common\.collect\.(ImmutableListMultimap)"/>
+        </module>
         <module name="RedundantImport"/>
         <module name="UnusedImports"/>