Fdp branch 3.2.1

.gitignore

@@ -51,13 +51,3 @@ patchprocess/
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
 
-# Ignore files generated by HDDS acceptance tests.
-hadoop-ozone/acceptance-test/docker-compose.log
-hadoop-ozone/acceptance-test/junit-results.xml
-
-#robotframework outputs
-log.html
-output.xml
-report.html
-
-hadoop-ozone/docs/public

BUILDING.txt

@@ -14,7 +14,6 @@ Requirements:
   Clang (community version), Clang (version for iOS 9 and later) (if compiling native code)
 * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)
 * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)
-* Jansson C XML parsing library ( if compiling libwebhdfs )
 * Doxygen ( if compiling libhdfspp and generating the documents )
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
 * python (for releasedocs)
@@ -83,8 +82,6 @@ Optional packages:
   (OR https://github.com/01org/isa-l)
 * Bzip2
   $ sudo apt-get install bzip2 libbz2-dev
-* Jansson (C Library for JSON)
-  $ sudo apt-get install libjansson-dev
 * Linux FUSE
   $ sudo apt-get install fuse libfuse-dev
 * ZStandard compression
@@ -355,6 +352,50 @@ export MAVEN_OPTS="-Xms256m -Xmx1536m"
 
 ----------------------------------------------------------------------------------
 
+Building on macOS (without Docker)
+
+----------------------------------------------------------------------------------
+Installing required dependencies for clean install of macOS 10.14:
+
+* Install Xcode Command Line Tools
+  $ xcode-select --install
+* Install Homebrew
+  $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+* Install OpenJDK 8
+  $ brew tap AdoptOpenJDK/openjdk
+  $ brew cask install adoptopenjdk8
+* Install maven and tools
+  $ brew install maven autoconf automake cmake wget
+* Install native libraries, only openssl is required to compile native code,
+you may optionally install zlib, lz4, etc.
+  $ brew install openssl
+* Protocol Buffers 2.5.0 (required), since 2.5.0 is no longer in Homebrew,
+we need to compile it from source
+  $ wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
+  $ tar zxvf protobuf-2.5.0.tar.gz
+  $ cd protobuf-2.5.0
+  $ ./configure
+  $ make
+  $ make check
+  $ make install
+  $ protoc --version
+
+Note that building Hadoop 3.1.1/3.1.2/3.2.0 native code from source is broken
+on macOS. For 3.1.1/3.1.2, you need to manually backport YARN-8622. For 3.2.0,
+you need to backport both YARN-8622 and YARN-9487 in order to build native code.
+
+----------------------------------------------------------------------------------
+Building command example:
+
+* Create binary distribution with native code but without documentation:
+  $ mvn package -Pdist,native -DskipTests -Dmaven.javadoc.skip \
+    -Dopenssl.prefix=/usr/local/opt/openssl
+
+Note that the command above manually specified the openssl library and include
+path. This is necessary at least for Homebrewed OpenSSL.
+
+----------------------------------------------------------------------------------
+
 Building on Windows
 
 ----------------------------------------------------------------------------------

LICENSE.txt

@@ -1835,7 +1835,7 @@ any resulting litigation.
 
 The binary distribution of this product bundles these dependencies under the
 following license:
-JSch 0.1.51
+JSch 0.1.54
 ParaNamer Core 2.3
 JLine 0.9.94
 leveldbjni-all 1.8

NOTICE.txt

@@ -8,10 +8,10 @@ following notices:
 * Copyright 2011 FuseSource Corp. http://fusesource.com
 
 The binary distribution of this product bundles binaries of
-AWS SDK for Java - Bundle 1.11.134,
-AWS Java SDK for AWS KMS 1.11.134,
-AWS Java SDK for Amazon S3 1.11.134,
-AWS Java SDK for AWS STS 1.11.134,
+AWS SDK for Java - Bundle 1.11.375,
+AWS Java SDK for AWS KMS 1.11.375,
+AWS Java SDK for Amazon S3 1.11.375,
+AWS Java SDK for AWS STS 1.11.375,
 JMES Path Query library 1.0,
 which has the following notices:
  * This software includes third party software subject to the following

buildScripts/buildCmake.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+tar -xvf cmake-3.13.3-Linux-x86_64.tar.gz 
+cd cmake-3.13.3-Linux-x86_64/
+echo "Creating cmake Symlink"
+sudo ln -s  `pwd`/bin/cmake /usr/bin/cmake

buildScripts/buildHadoop.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+find . -name pom.xml -exec sed -i -e 's/<require.snappy>false<\/require.snappy>/<require.snappy>true<\/require.snappy>/g' {} \;
+mvn package -Pdist,native -Dyarn.arg="install --offline" -DskipTests -Drequire.snappy -Dmaven.javadoc.skip=true -U -X 

buildScripts/buildProtobuff.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+cd protobuf-2.5.0
+sudo ./configure
+sudo make clean
+sudo make
+sudo make install
+sudo ldconfig
+protoc --version
+cd $cwd
+

buildScripts/installDependencies.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+sudo grep -r security.debian.org /etc/apt/
+sudo sed -i '/security.debian.org/s/^/#/'  /etc/apt/sources.list 
+sudo apt-get update
+sudo apt-get install protobuf-compiler liblzo2-2 libc6 --allow-unauthenticated -y
+sudo apt-get install pkg-config  --yes --allow-unauthenticated
+sudo apt-get install --yes --allow-unauthenticated build-essential
+sudo apt-get install --yes --allow-unauthenticated  make  libsnappy-dev bzip2 libbz2-dev libjansson-dev zlib1g-dev libncurses5-dev libssl-dev libfuse-dev fuse zstd

buildScripts/pushTarToRepo.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+mkdir pushHadoopTar/HadoopSourceBinary
+cp -rf hadoop-dist/target/hadoop-3.2.1.tar.gz pushHadoopTar/HadoopSourceBinary/
+ls
+
+cd pushHadoopTar
+ls
+
+mkdir -p HadoopSourceBinary
+mkdir -p deb/tmp/
+cp -rf HadoopSourceBinary deb/tmp/
+cp -rf DEBIAN deb/
+
+sh pushHadoopTar/make-flipkart-hadoop-source-tar-3.2.1-deb
+dpkg -b deb flipkart-hadoop-source-tar-repo.deb
+reposervice --host 10.24.0.41 --port "8080" pubrepo --repo  fk-hadoop-source-tar-repo  --appkey dummy --debs flipkart-hadoop-source-tar-repo.deb
+cd $cwd

dev-support/bin/create-release

@@ -169,11 +169,6 @@ function run()
   fi
 }
 
-function domd5()
-{
-  run "${MD5SUM}" "${1}" > "${1}.md5"
-}
-
 function header()
 {
   echo
@@ -235,11 +230,6 @@ function set_defaults
     fi
   fi
 
-  MD5SUM=$(command -v md5sum)
-  if [[ -z "${MD5SUM}" ]]; then
-    MD5SUM=$(command -v md5)
-  fi
-
   NATIVE=false
   OSNAME=$(uname -s)
 
@@ -253,7 +243,8 @@ function startgpgagent
   if [[ "${SIGN}" = true ]]; then
     if [[ -n "${GPGAGENT}" && -z "${GPG_AGENT_INFO}" ]]; then
       echo "starting gpg agent"
-      echo "default-cache-ttl 14400" > "${LOGDIR}/gpgagent.conf"
+      echo "default-cache-ttl 36000" > "${LOGDIR}/gpgagent.conf"
+      echo "max-cache-ttl 36000" >> "${LOGDIR}/gpgagent.conf"
       # shellcheck disable=2046
       eval $("${GPGAGENT}" --daemon \
         --options "${LOGDIR}/gpgagent.conf" \
@@ -533,7 +524,7 @@ function makearelease
   big_console_header "Cleaning the Source Tree"
 
   # git clean to clear any remnants from previous build
-  run "${GIT}" clean -xdf
+  run "${GIT}" clean -xdf -e /patchprocess
 
   mkdir -p "${LOGDIR}"
 
@@ -640,9 +631,6 @@ function signartifacts
   declare ret
 
   if [[ "${SIGN}" = false ]]; then
-    for i in ${ARTIFACTS_DIR}/*; do
-      domd5 "${i}"
-    done
     echo ""
     echo "Remember to sign the artifacts before staging them on the open"
     echo ""
@@ -653,8 +641,7 @@ function signartifacts
 
   for i in ${ARTIFACTS_DIR}/*; do
     ${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
-    ${GPG} --print-mds "${i}" > "${i}.mds"
-    domd5 "${i}"
+    sha512sum --tag "${i}" > "${i}.sha512"
   done
 
   if [[ "${ASFRELEASE}" = true ]]; then

dev-support/bin/dist-layout-stitching

@@ -21,9 +21,6 @@ VERSION=$1
 # project.build.directory
 BASEDIR=$2
 
-#hdds.version
-HDDS_VERSION=$3
-
 function run()
 {
   declare res

dev-support/bin/yetus-wrapper

@@ -68,20 +68,30 @@ function yetus_abs
   return 1
 }
 
+function version_ge()
+{
+  test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1";
+}
 
 WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.8.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.10.0}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 
+## HADOOP_YETUS_VERSION >= 0.9.0 the tarball named with apache-yetus prefix
+if version_ge "${HADOOP_YETUS_VERSION}" "0.9.0"; then
+  YETUS_PREFIX=apache-yetus
+else
+  YETUS_PREFIX=yetus
+fi
+
 ###
 ###  if YETUS_HOME is set, then try to use it
 ###
-if [[ -n "${YETUS_HOME}"
-   && -x "${YETUS_HOME}/bin/${WANTED}" ]]; then
+if [[ -n "${YETUS_HOME}" && -x "${YETUS_HOME}/bin/${WANTED}" ]]; then
   exec "${YETUS_HOME}/bin/${WANTED}" "${ARGV[@]}"
 fi
 
@@ -105,16 +115,16 @@ HADOOP_PATCHPROCESS=${mytmpdir}
 ##
 ## if we've already DL'd it, then short cut
 ##
-if [[ -x "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
-  exec "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
+if [[ -x "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
+  exec "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
 fi
 
 ##
 ## need to DL, etc
 ##
 
 BASEURL="https://archive.apache.org/dist/yetus/${HADOOP_YETUS_VERSION}/"
-TARBALL="yetus-${HADOOP_YETUS_VERSION}-bin.tar"
+TARBALL="${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}-bin.tar"
 
 GPGBIN=$(command -v gpg)
 CURLBIN=$(command -v curl)
@@ -166,9 +176,9 @@ if ! (gunzip -c "${TARBALL}.gz" | tar xpf -); then
   exit 1
 fi
 
-if [[ -x "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
+if [[ -x "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
   popd >/dev/null
-  exec "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
+  exec "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
 fi
 
 ##

dev-support/docker/Dockerfile

@@ -63,6 +63,7 @@ RUN apt-get -q update \
         libsnappy-dev \
         libssl-dev \
         libtool \
+        libzstd1-dev \
         locales \
         make \
         pinentry-curses \

hadoop-assemblies/pom.xml

@@ -18,16 +18,16 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-                      http://maven.apache.org/xsd/maven-4.0.0.xsd">
+                      https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.2.0-SNAPSHOT</version>
+    <version>3.2.1</version>
     <relativePath>../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-assemblies</artifactId>
-  <version>3.2.0-SNAPSHOT</version>
+  <version>3.2.1</version>
   <name>Apache Hadoop Assemblies</name>
   <description>Apache Hadoop Assemblies</description>
 

hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml

@@ -50,8 +50,6 @@
         <exclude>**/build/**</exclude>
         <exclude>**/file:/**</exclude>
         <exclude>**/SecurityAuth.audit*</exclude>
-        <exclude>hadoop-ozone/**</exclude>
-        <exclude>hadoop-hdds/**</exclude>
       </excludes>
     </fileSet>
   </fileSets>