HADOOP-13551. AWS metrics wire-up (#2778)

.gitignore

@@ -9,6 +9,7 @@
 *.suo
 *.vcxproj.user
 *.patch
+*.diff
 .idea
 .svn
 .classpath
@@ -50,16 +51,12 @@ patchprocess/
 .history/
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
-
-# Ignore files generated by HDDS acceptance tests.
-hadoop-ozone/acceptance-test/docker-compose.log
-hadoop-ozone/acceptance-test/junit-results.xml
+phantomjsdriver.log
 
 #robotframework outputs
 log.html
 output.xml
 report.html
 
-hadoop-hdds/docs/public
 
 .mvn

BUILDING.txt

@@ -63,19 +63,16 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
 * Native libraries
   $ sudo apt-get -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev libsasl2-dev
 * Protocol Buffers 3.7.1 (required to build native code)
-  $ mkdir -p /opt/protobuf-3.7-src \
-        && curl -L -s -S \
-          https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-          -o /opt/protobuf-3.7.1.tar.gz \
-        && tar xzf /opt/protobuf-3.7.1.tar.gz --strip-components 1 -C /opt/protobuf-3.7-src \
-        && cd /opt/protobuf-3.7-src \
-        && ./configure\
-        && make install \
-        && rm -rf /opt/protobuf-3.7-src
+  $ curl -L -s -S https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz -o protobuf-3.7.1.tar.gz
+  $ mkdir protobuf-3.7-src
+  $ tar xzf protobuf-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7-src && cd protobuf-3.7-src
+  $ ./configure
+  $ make -j$(nproc)
+  $ sudo make install
 
 Optional packages:
 
-* Snappy compression
+* Snappy compression (only used for hadoop-mapreduce-client-nativetask)
   $ sudo apt-get install snappy libsnappy-dev
 * Intel ISA-L library for erasure coding
   Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
@@ -96,7 +93,7 @@ Maven main modules:
          - hadoop-project               (Parent POM for all Hadoop Maven modules.             )
                                         (All plugins & dependencies versions are defined here.)
          - hadoop-project-dist          (Parent POM for modules that generate distributions.)
-         - hadoop-annotations           (Generates the Hadoop doclet used to generated the Javadocs)
+         - hadoop-annotations           (Generates the Hadoop doclet used to generate the Javadocs)
          - hadoop-assemblies            (Maven assemblies used by the different modules)
          - hadoop-maven-plugins         (Maven plugins used in project)
          - hadoop-build-tools           (Build tools like checkstyle, etc.)
@@ -113,7 +110,7 @@ Maven main modules:
 ----------------------------------------------------------------------------------
 Where to run Maven from?
 
-  It can be run from any module. The only catch is that if not run from utrunk
+  It can be run from any module. The only catch is that if not run from trunk
   all modules that are not part of the build run must be installed in the local
   Maven cache or available in a Maven repository.
 
@@ -124,7 +121,7 @@ Maven build goals:
  * Compile                   : mvn compile [-Pnative]
  * Run tests                 : mvn test [-Pnative] [-Pshelltest]
  * Create JAR                : mvn package
- * Run findbugs              : mvn compile findbugs:findbugs
+ * Run spotbugs              : mvn compile spotbugs:spotbugs
  * Run checkstyle            : mvn compile checkstyle:checkstyle
  * Install JAR in M2 cache   : mvn install
  * Deploy JAR to Maven repo  : mvn deploy
@@ -161,14 +158,14 @@ Maven build goals:
 
    Snappy is a compression library that can be utilized by the native code.
    It is currently an optional component, meaning that Hadoop can be built with
-   or without this dependency.
+   or without this dependency. Snappy library as optional dependency is only
+   used for hadoop-mapreduce-client-nativetask.
 
   * Use -Drequire.snappy to fail the build if libsnappy.so is not found.
     If this option is not specified and the snappy library is missing,
     we silently build a version of libhadoop.so that cannot make use of snappy.
     This option is recommended if you plan on making use of snappy and want
     to get more repeatable builds.
-
   * Use -Dsnappy.prefix to specify a nonstandard location for the libsnappy
     header files and library files. You do not need this option if you have
     installed snappy using a package manager.

LICENSE-binary

@@ -215,7 +215,7 @@ com.aliyun:aliyun-java-sdk-ecs:4.2.0
 com.aliyun:aliyun-java-sdk-ram:3.0.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.4.1
-com.amazonaws:aws-java-sdk-bundle:1.11.563
+com.amazonaws:aws-java-sdk-bundle:1.11.901
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
 com.fasterxml.jackson.core:jackson-annotations:2.9.9
@@ -225,7 +225,7 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.9.9
 com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.9
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.9.9
 com.fasterxml.uuid:java-uuid-generator:3.1.4
-com.fasterxml.woodstox:woodstox-core:5.0.3
+com.fasterxml.woodstox:woodstox-core:5.3.0
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google:guice:4.0
@@ -366,7 +366,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage
 
 com.github.luben:zstd-jni:1.4.3-1
 dnsjava:dnsjava:2.1.7
-org.codehaus.woodstox:stax2-api:3.1.4
+org.codehaus.woodstox:stax2-api:4.2.1
 
 
 BSD 3-Clause
@@ -385,7 +385,7 @@ com.google.protobuf:protobuf-java:3.6.1
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.54
 com.thoughtworks.paranamer:paranamer:2.3
-javax.activation:javax.activation-api:1.2.0
+jakarta.activation:jakarta.activation-api:1.2.1
 org.fusesource.leveldbjni:leveldbjni-all:1.8
 org.jline:jline:3.9.0
 org.hamcrest:hamcrest-core:1.3
@@ -468,8 +468,8 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk15on:1.60
-org.bouncycastle:bcprov-jdk15on:1.60
+org.bouncycastle:bcpkix-jdk15on:1.68
+org.bouncycastle:bcprov-jdk15on:1.68
 org.checkerframework:checker-qual:2.5.2
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.jruby.jcodings:jcodings:1.0.13

Jenkinsfile → dev-support/Jenkinsfile

@@ -23,7 +23,7 @@ pipeline {
 
     options {
         buildDiscarder(logRotator(numToKeepStr: '5'))
-        timeout (time: 5, unit: 'HOURS')
+        timeout (time: 20, unit: 'HOURS')
         timestamps()
         checkoutToSubdirectory('src')
     }
@@ -35,7 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='rel/0.11.1'
+        YETUS_VERSION='11eb9b09786e401fbdeaa3be83a19a4066fd7813'
     }
 
     parameters {
@@ -61,7 +61,7 @@ pipeline {
             steps {
                 withCredentials(
                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.202132.xyz',
-                                  passwordVariable: 'GITHUB_PASSWORD',
+                                  passwordVariable: 'GITHUB_TOKEN',
                                   usernameVariable: 'GITHUB_USER'),
                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
                                         passwordVariable: 'JIRA_PASSWORD',
@@ -96,17 +96,16 @@ pipeline {
                         YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
 
                         # our project defaults come from a personality file
-                        # which will get loaded automatically by setting the project name
                         YETUS_ARGS+=("--project=hadoop")
+                        YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
 
                         # lots of different output formats
                         YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
                         YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
                         YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
 
                         # enable writing back to Github
-                        YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
-                        YETUS_ARGS+=(--github-user=${GITHUB_USER})
+                        YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
 
                         # enable writing back to ASF JIRA
                         YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
@@ -119,13 +118,13 @@ pipeline {
                         # changing these to higher values may cause problems
                         # with other jobs on systemd-enabled machines
                         YETUS_ARGS+=("--proclimit=5500")
-                        YETUS_ARGS+=("--dockermemlimit=20g")
+                        YETUS_ARGS+=("--dockermemlimit=22g")
 
-                        # -1 findbugs issues that show up prior to the patch being applied
-                        YETUS_ARGS+=("--findbugs-strict-precheck")
+                        # -1 spotbugs issues that show up prior to the patch being applied
+                        YETUS_ARGS+=("--spotbugs-strict-precheck")
 
                         # rsync these files back into the archive dir
-                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
 
                         # URL for user-side presentation in reports and such to our artifacts
                         # (needs to match the archive bits below)
@@ -134,9 +133,6 @@ pipeline {
                         # plugins to enable
                         YETUS_ARGS+=("--plugins=all")
 
-                        # use Hadoop's bundled shelldocs
-                        YETUS_ARGS+=("--shelldocs=/testptch/hadoop/dev-support/bin/shelldocs")
-
                         # don't let these tests cause -1s because we aren't really paying that
                         # much attention to them
                         YETUS_ARGS+=("--tests-filter=checkstyle")
@@ -145,14 +141,19 @@ pipeline {
                         # Dockerfile since we don't want to use the auto-pulled version.
                         YETUS_ARGS+=("--docker")
                         YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+                        YETUS_ARGS+=("--mvn-custom-repos")
 
                         # effectively treat dev-suport as a custom maven module
                         YETUS_ARGS+=("--skip-dirs=dev-support")
 
                         # help keep the ASF boxes clean
                         YETUS_ARGS+=("--sentinel")
 
-                        # use emoji vote so it is easier to find the broken line
+                        # custom javadoc goals
+                        YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
+
+                        # write Yetus report as GitHub comment (YETUS-1102)
+                        YETUS_ARGS+=("--github-write-comment")
                         YETUS_ARGS+=("--github-use-emoji-vote")
 
                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
@@ -166,6 +167,19 @@ pipeline {
     post {
         always {
           script {
+            // Publish status if it was missed (YETUS-1059)
+            withCredentials(
+                [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER')]) {
+                sh '''#!/usr/bin/env bash
+                    YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
+                    YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+                    TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
+                    /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
+                    '''
+            }
+
             // Yetus output
             archiveArtifacts "${env.PATCHDIR}/**"
             // Publish the HTML report so that it can be looked at

dev-support/bin/checkcompatibility.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -30,33 +30,16 @@
 import shutil
 import subprocess
 import sys
-import urllib2
-try:
-  import argparse
-except ImportError:
-  sys.stderr.write("Please install argparse, e.g. via `pip install argparse`.")
-  sys.exit(2)
+import urllib.request
+import argparse
 
 # Various relative paths
 REPO_DIR = os.getcwd()
 
 def check_output(*popenargs, **kwargs):
-  r"""Run command with arguments and return its output as a byte string.
-  Backported from Python 2.7 as it's implemented as pure python on stdlib.
-  >>> check_output(['/usr/bin/python', '--version'])
-  Python 2.6.2
-  """
-  process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
-  output, _ = process.communicate()
-  retcode = process.poll()
-  if retcode:
-    cmd = kwargs.get("args")
-    if cmd is None:
-      cmd = popenargs[0]
-    error = subprocess.CalledProcessError(retcode, cmd)
-    error.output = output
-    raise error
-  return output
+  """ Run command with arguments and return its output as a string. """
+  return subprocess.check_output(*popenargs, **kwargs, encoding='utf-8')
+
 
 def get_repo_dir():
   """ Return the path to the top of the repo. """
@@ -139,7 +122,7 @@ def checkout_java_acc(force):
   url = "https://github.com/lvc/japi-compliance-checker/archive/1.8.tar.gz"
   scratch_dir = get_scratch_dir()
   path = os.path.join(scratch_dir, os.path.basename(url))
-  jacc = urllib2.urlopen(url)
+  jacc = urllib.request.urlopen(url)
   with open(path, 'wb') as w:
     w.write(jacc.read())
 
@@ -194,7 +177,7 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations):
     annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
     with file(annotations_path, "w") as f:
       for ann in annotations:
-        print >>f, ann
+        print(ann, file=f)
     args += ["-annotations-list", annotations_path]
 
   subprocess.check_call(args)
@@ -264,8 +247,8 @@ def main():
   parser.add_argument("--skip-build",
                       action="store_true",
                       help="Skip building the projects.")
-  parser.add_argument("src_rev", nargs=1, help="Source revision.")
-  parser.add_argument("dst_rev", nargs="?", default="HEAD",
+  parser.add_argument("src_rev", nargs=1, type=str, help="Source revision.")
+  parser.add_argument("dst_rev", nargs="?", type=str, default="HEAD",
                       help="Destination revision. " +
                       "If not specified, will use HEAD.")
 

dev-support/bin/create-release

@@ -651,10 +651,12 @@ function signartifacts
 
   big_console_header "Signing the release"
 
-  for i in ${ARTIFACTS_DIR}/*; do
+  run cd "${ARTIFACTS_DIR}"
+  for i in *; do
     ${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
     sha512sum --tag "${i}" > "${i}.sha512"
   done
+  run cd "${BASEDIR}"
 
   if [[ "${ASFRELEASE}" = true ]]; then
     echo "Fetching the Apache Hadoop KEYS file..."

dev-support/bin/dist-copynativelibs

@@ -111,9 +111,6 @@ for i in "$@"; do
     --openssllibbundle=*)
       OPENSSLLIBBUNDLE=${i#*=}
     ;;
-    --snappybinbundle=*)
-      SNAPPYBINBUNDLE=${i#*=}
-    ;;
     --snappylib=*)
       SNAPPYLIB=${i#*=}
     ;;
@@ -167,7 +164,7 @@ fi
 
 # Windows doesn't have a LIB_DIR, everything goes into bin
 
-if [[ -d "${BIN_DIR}" ]] ; then
+if [[ -d "${BIN_DIR}" && $(ls -A "${BIN_DIR}") ]] ; then
   mkdir -p "${TARGET_BIN_DIR}"
   cd "${BIN_DIR}"  || exit 1
   ${TAR} ./* | (cd "${TARGET_BIN_DIR}"/ || exit 1; ${UNTAR})
@@ -176,8 +173,6 @@ if [[ -d "${BIN_DIR}" ]] ; then
     exit 1
   fi
 
-  bundle_native_bin "${SNAPPYBINBUNDLE}" "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}"
-
   bundle_native_bin "${ZSTDBINBUNDLE}" "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
 
   bundle_native_bin "${OPENSSLBINBUNDLE}" "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"

dev-support/bin/dist-layout-stitching

@@ -21,9 +21,6 @@ VERSION=$1
 # project.build.directory
 BASEDIR=$2
 
-#hdds.version
-HDDS_VERSION=$3
-
 function run()
 {
   declare res