From 1cad9efdb758c52efb35a044410774b1650de18e Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 13 May 2020 01:53:21 +0900 Subject: [PATCH 1/9] Refactor Dockerfile --- dev-support/docker/Dockerfile | 52 ++++++++--------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 57abed731d145..eb8cf3547a623 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -33,16 +33,10 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true -###### -# Install common dependencies from packages. Versions here are either -# sufficient or irrelevant. -# -# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default -# Ubuntu Java. See Java section below! -###### # hadolint ignore=DL3008 RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends \ + ant \ apt-utils \ bats \ build-essential \ @@ -51,11 +45,13 @@ RUN apt-get -q update \ cmake \ curl \ doxygen \ + findbugs \ fuse \ g++ \ gcc \ git \ gnupg-agent \ + libbcprov-java \ libbz2-dev \ libcurl4-openssl-dev \ libfuse-dev \ @@ -64,11 +60,12 @@ RUN apt-get -q update \ libsasl2-dev \ libsnappy-dev \ libssl-dev \ - libsnappy-dev \ libtool \ libzstd1-dev \ locales \ make \ + maven \ + openjdk-8-jdk \ pinentry-curses \ pkg-config \ python \ @@ -86,15 +83,13 @@ RUN apt-get -q update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - -####### -# OpenJDK 8 -####### -# hadolint ignore=DL3008 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME /usr +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +ENV FINDBUGS_HOME /usr ###### # Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic) @@ -113,29 +108,6 @@ RUN mkdir -p /opt/protobuf-src \ ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -###### -# Install Apache Maven 3.6.0 (3.6.0 ships with Bionic) -###### -# hadolint ignore=DL3008 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends maven \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -ENV MAVEN_HOME /usr -# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 - -###### -# Install findbugs 3.1.0 (3.1.0 ships with Bionic) -# Ant is needed for findbugs -###### -# hadolint ignore=DL3008 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends findbugs ant \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -ENV FINDBUGS_HOME /usr - #### # Install pylint at fixed version (2.0.0 removed python2 support) # https://github.com/PyCQA/pylint/issues/2294 From fc7d9e60fac61d6d312c7dbdafd71a5d85fd6fce Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 13 May 2020 02:00:06 +0900 Subject: [PATCH 2/9] Add Java 11 test in GitHub PR --- Jenkinsfile | 3 +++ dev-support/docker/Dockerfile | 1 + 2 files changed, 4 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 69a23fbb829f7..b30c628b74def 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -154,6 +154,9 @@ pipeline { # use emoji vote so it is easier to find the broken line YETUS_ARGS+=("--github-use-emoji-vote") + # test with Java 8 and 11 + YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") + "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ''' } diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index eb8cf3547a623..c03e7d2796e92 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -66,6 +66,7 @@ RUN apt-get -q update \ make \ maven \ openjdk-8-jdk \ + openjdk-11-jdk \ pinentry-curses \ pkg-config \ python \ From 1862abdc1e4e3cff78c6e5e1b5081425f0c47091 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 13 May 2020 02:01:22 +0900 Subject: [PATCH 3/9] Correct the order --- dev-support/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index c03e7d2796e92..dc242e8e597f4 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -65,8 +65,8 @@ RUN apt-get -q update \ locales \ make \ maven \ - openjdk-8-jdk \ openjdk-11-jdk \ + openjdk-8-jdk \ pinentry-curses \ pkg-config \ python \ From 2752cdb4ef85bd554c64ad2ff4ac2e97f343043a Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 13 May 2020 04:24:07 +0900 Subject: [PATCH 4/9] Set Java home to JDK 8 explicitly --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index b30c628b74def..8cce00f1dc85c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,6 +155,7 @@ pipeline { YETUS_ARGS+=("--github-use-emoji-vote") # test with Java 8 and 11 + YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" From 11558af865c5c8075ff56a376f6d1d50e98f1d3c Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 13 May 2020 11:07:35 +0900 Subject: [PATCH 5/9] Increase the timeout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8cce00f1dc85c..10ff0fd05bd1c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '5')) - timeout (time: 5, unit: 'HOURS') + timeout (time: 10, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') } From c6de1453c5afe94217edc46b1e56c0fff748ff25 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 18 May 2020 17:22:15 +0900 Subject: [PATCH 6/9] Increase the timeout to 24 hours. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 10ff0fd05bd1c..e6128550462d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '5')) - timeout (time: 10, unit: 'HOURS') + timeout (time: 24, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') } From 9a3526c554e93642de9d41512dcda2673f2ce67b Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 19 May 2020 10:31:20 +0900 Subject: [PATCH 7/9] Stop running unit tests in Java 11. --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e6128550462d4..16f881101774a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '5')) - timeout (time: 24, unit: 'HOURS') + timeout (time: 10, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') } @@ -157,6 +157,7 @@ pipeline { # test with Java 8 and 11 YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") + YETUS_ARGS+=("--multijdktests=compile") "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ''' From cb289d025eed4813b39f36beb09e8903e26eb148 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 20 May 2020 16:11:45 +0900 Subject: [PATCH 8/9] Increase timeout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 16f881101774a..933991863b76b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '5')) - timeout (time: 10, unit: 'HOURS') + timeout (time: 24, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') } From eda4159973bb6ef1c22cfec97806896b1756e205 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 13 May 2020 02:05:12 +0900 Subject: [PATCH 9/9] [DO NOT MERGE] Testing HADOOP-16888. --- .../apache/hadoop/io/compress/TestCompressorDecompressor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java index 1f035974883cf..6e88d5f833409 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java @@ -46,7 +46,7 @@ */ public class TestCompressorDecompressor { - private static final Random rnd = new Random(12345L); + private static final Random rnd = new Random(123456789L); @Test public void testCompressorDecompressor() {