From ac10e3975e40f68b5c82026127a9c3cfd28d38f6 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 1 Jul 2021 16:10:58 +0530 Subject: [PATCH 1/7] HADOOP-17786. Parallelize stages in Jenkins * We now build Hadoop on multiple platforms for validation. We need to parallelize them for faster validation. --- dev-support/Jenkinsfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index b9e56fd205498..ad55d896d4ebf 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -15,6 +15,15 @@ // specific language governing permissions and limitations // under the License. +def getGithubAndJiraCreds() { + return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', + passwordVariable: 'JIRA_PASSWORD', + usernameVariable: 'JIRA_USER')] +} + pipeline { agent { From 7b52a2b8a135ecb5512308ceeb19b9c0150ab78c Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 1 Jul 2021 16:12:41 +0530 Subject: [PATCH 2/7] Fail fast when any parallel stage fails * Added an option to the pipeline that fails as soon as any of the parallel stages fails. --- dev-support/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index ad55d896d4ebf..052625b736dd5 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -35,6 +35,7 @@ pipeline { timeout (time: 24, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') + parallelsAlwaysFailFast() } environment { From f31bac99ee47a78c6d3032ea7d1acc2e1f198590 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 1 Jul 2021 17:01:00 +0530 Subject: [PATCH 3/7] Have multiple checkouts --- dev-support/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 052625b736dd5..443c410ee96d4 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -34,7 +34,9 @@ pipeline { buildDiscarder(logRotator(numToKeepStr: '5')) timeout (time: 24, unit: 'HOURS') timestamps() - checkoutToSubdirectory('src') + checkoutToSubdirectory('src-centos-8') + checkoutToSubdirectory('src-debian-10') + checkoutToSubdirectory('src-ubuntu-focal') parallelsAlwaysFailFast() } From 8ef6fde8272cefdee928f6a14c35957b83714cf1 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Thu, 1 Jul 2021 17:01:27 +0530 Subject: [PATCH 4/7] Trigger CI --- .../src/main/native/libhdfspp/lib/x-platform/utils.h | 1 + .../src/main/native/libhdfspp/tests/bad_datanode_test.cc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h index 06c608ba969c8..3fe126373c36a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h @@ -16,6 +16,7 @@ * limitations under the License. */ + #ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS #define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc index 973212647e764..d6dcf96f9f705 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc @@ -31,7 +31,6 @@ #include #include - using hadoop::common::TokenProto; using hadoop::hdfs::DatanodeInfoProto; using hadoop::hdfs::DatanodeIDProto; From 84cb1b39a6194ef48c59fae0d34da82e184dbc17 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Tue, 27 Jul 2021 13:29:14 +0530 Subject: [PATCH 5/7] Parallelize after code checkout * Parallellization of the stages must happen after the code has been checked out for each of the environments. --- dev-support/Jenkinsfile | 334 ++++++++++++++++++++-------------------- 1 file changed, 163 insertions(+), 171 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 443c410ee96d4..ded7ca2ce42a4 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -15,15 +15,6 @@ // specific language governing permissions and limitations // under the License. -def getGithubAndJiraCreds() { - return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')] -} - pipeline { agent { @@ -34,10 +25,7 @@ pipeline { buildDiscarder(logRotator(numToKeepStr: '5')) timeout (time: 24, unit: 'HOURS') timestamps() - checkoutToSubdirectory('src-centos-8') - checkoutToSubdirectory('src-debian-10') - checkoutToSubdirectory('src-ubuntu-focal') - parallelsAlwaysFailFast() + checkoutToSubdirectory('src') } environment { @@ -153,186 +141,190 @@ pipeline { } } - // This is an optional stage which runs only when there's a change in - // C++/C++ build/platform. - // This stage serves as a means of cross platform validation, which is - // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Centos 8. - stage ('precommit-run Centos 8') { - environment { - SOURCEDIR = "${WORKSPACE}/centos-8/src" - PATCHDIR = "${WORKSPACE}/centos-8/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8" - IS_OPTIONAL = 1 - } - - steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { - sh '''#!/usr/bin/env bash + stage ('platform validation') { + parallel { + // This is an optional stage which runs only when there's a change in + // C++/C++ build/platform. + // This stage serves as a means of cross platform validation, which is + // really needed to ensure that any C++ related/platform change doesn't + // break the Hadoop build on Centos 8. + stage ('precommit-run Centos 8') { + environment { + SOURCEDIR = "${WORKSPACE}/centos-8/src" + PATCHDIR = "${WORKSPACE}/centos-8/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8" + IS_OPTIONAL = 1 + } - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" run_ci - ''' - } - } + steps { + withCredentials( + [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', + passwordVariable: 'JIRA_PASSWORD', + usernameVariable: 'JIRA_USER')]) { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci + ''' + } + } - post { - // Since this is an optional platform, we want to copy the artifacts - // and archive it only if the build fails, to help with debugging. - failure { - sh '''#!/usr/bin/env bash + post { + // Since this is an optional platform, we want to copy the artifacts + // and archive it only if the build fails, to help with debugging. + failure { + sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}" - ''' - archiveArtifacts "out/**" - } + cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}" + ''' + archiveArtifacts "out/**" + } - cleanup() { - script { - sh '''#!/usr/bin/env bash + cleanup() { + script { + sh '''#!/usr/bin/env bash - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc - ''' + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } } } - } - } - - // This is an optional stage which runs only when there's a change in - // C++/C++ build/platform. - // This stage serves as a means of cross platform validation, which is - // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Debian 10. - stage ('precommit-run Debian 10') { - environment { - SOURCEDIR = "${WORKSPACE}/debian-10/src" - PATCHDIR = "${WORKSPACE}/debian-10/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" - IS_OPTIONAL = 1 - } - steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { - sh '''#!/usr/bin/env bash + // This is an optional stage which runs only when there's a change in + // C++/C++ build/platform. + // This stage serves as a means of cross platform validation, which is + // really needed to ensure that any C++ related/platform change doesn't + // break the Hadoop build on Debian 10. + stage ('precommit-run Debian 10') { + environment { + SOURCEDIR = "${WORKSPACE}/debian-10/src" + PATCHDIR = "${WORKSPACE}/debian-10/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" + IS_OPTIONAL = 1 + } - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" run_ci - ''' - } - } + steps { + withCredentials( + [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', + passwordVariable: 'JIRA_PASSWORD', + usernameVariable: 'JIRA_USER')]) { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci + ''' + } + } - post { - // Since this is an optional platform, we want to copy the artifacts - // and archive it only if the build fails, to help with debugging. - failure { - sh '''#!/usr/bin/env bash + post { + // Since this is an optional platform, we want to copy the artifacts + // and archive it only if the build fails, to help with debugging. + failure { + sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" - ''' - archiveArtifacts "out/**" - } + cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" + ''' + archiveArtifacts "out/**" + } - cleanup() { - script { - sh '''#!/usr/bin/env bash + cleanup() { + script { + sh '''#!/usr/bin/env bash - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc - ''' + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } } } - } - } - - // We want to use Ubuntu Focal as our main CI and thus, this stage - // isn't optional (runs for all the PRs). - stage ('precommit-run Ubuntu focal') { - environment { - SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src" - PATCHDIR = "${WORKSPACE}/ubuntu-focal/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" - IS_OPTIONAL = 0 - } - steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { - sh '''#!/usr/bin/env bash - - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" run_ci - ''' - } - } + // We want to use Ubuntu Focal as our main CI and thus, this stage + // isn't optional (runs for all the PRs). + stage ('precommit-run Ubuntu focal') { + environment { + SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src" + PATCHDIR = "${WORKSPACE}/ubuntu-focal/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" + IS_OPTIONAL = 0 + } - post { - always { - script { - // Publish status if it was missed (YETUS-1059) + steps { withCredentials( - [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER')]) { - sh '''#!/usr/bin/env bash - - # Copy the artifacts of Ubuntu focal build to workspace - cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}" - - # Send Github status - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery - ''' + [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', + passwordVariable: 'JIRA_PASSWORD', + usernameVariable: 'JIRA_USER')]) { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci + ''' } + } - // YETUS output - archiveArtifacts "out/**" - - // Publish the HTML report so that it can be looked at - // Has to be relative to WORKSPACE. - publishHTML (target: [ - allowMissing: true, - keepAll: true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE - reportDir: "out", - reportFiles: 'report.html', - reportName: 'Yetus Report' - ]) - - // Publish JUnit results - try { - junit "${SOURCEDIR}/**/target/surefire-reports/*.xml" - } catch(e) { - echo 'junit processing: ' + e.toString() + post { + always { + script { + // Publish status if it was missed (YETUS-1059) + withCredentials( + [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER')]) { + sh '''#!/usr/bin/env bash + + # Copy the artifacts of Ubuntu focal build to workspace + cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}" + + # Send Github status + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery + ''' + } + + // YETUS output + archiveArtifacts "out/**" + + // Publish the HTML report so that it can be looked at + // Has to be relative to WORKSPACE. + publishHTML (target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE + reportDir: "out", + reportFiles: 'report.html', + reportName: 'Yetus Report' + ]) + + // Publish JUnit results + try { + junit "${SOURCEDIR}/**/target/surefire-reports/*.xml" + } catch(e) { + echo 'junit processing: ' + e.toString() + } + } } - } - } - cleanup() { - script { - sh '''#!/usr/bin/env bash + cleanup() { + script { + sh '''#!/usr/bin/env bash - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc - ''' + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } } } } @@ -352,4 +344,4 @@ pipeline { } } } -} +} \ No newline at end of file From 437eccf6ae04e4f34a1dd6f2ad4182790e336cdc Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Tue, 27 Jul 2021 17:25:39 +0530 Subject: [PATCH 6/7] Fail fast when any parallel stage fails * Added an option parallelsAlwaysFailFast to fail the full CI immediately when any stage fails. --- dev-support/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index ded7ca2ce42a4..ee666b84a0209 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -26,6 +26,7 @@ pipeline { timeout (time: 24, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') + parallelsAlwaysFailFast() } environment { From 26529d07f8139a392e246f7068dea98ad13a93fc Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Tue, 3 Aug 2021 11:23:26 +0530 Subject: [PATCH 7/7] Rebase with trunk --- dev-support/Jenkinsfile | 92 ++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index ee666b84a0209..00d17ff077feb 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -90,60 +90,60 @@ pipeline { } } - // This is an optional stage which runs only when there's a change in - // C++/C++ build/platform. - // This stage serves as a means of cross platform validation, which is - // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Centos 7. - stage ('precommit-run Centos 7') { - environment { - SOURCEDIR = "${WORKSPACE}/centos-7/src" - PATCHDIR = "${WORKSPACE}/centos-7/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7" - IS_OPTIONAL = 1 - } + stage ('platform validation') { + parallel { + // This is an optional stage which runs only when there's a change in + // C++/C++ build/platform. + // This stage serves as a means of cross platform validation, which is + // really needed to ensure that any C++ related/platform change doesn't + // break the Hadoop build on Centos 7. + stage ('precommit-run Centos 7') { + environment { + SOURCEDIR = "${WORKSPACE}/centos-7/src" + PATCHDIR = "${WORKSPACE}/centos-7/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7" + IS_OPTIONAL = 1 + } - steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { - sh '''#!/usr/bin/env bash - - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" run_ci - ''' - } - } + steps { + withCredentials( + [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', + passwordVariable: 'JIRA_PASSWORD', + usernameVariable: 'JIRA_USER')]) { + sh '''#!/usr/bin/env bash - post { - // Since this is an optional platform, we want to copy the artifacts - // and archive it only if the build fails, to help with debugging. - failure { - sh '''#!/usr/bin/env bash + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci + ''' + } + } - cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}" - ''' - archiveArtifacts "out/**" - } + post { + // Since this is an optional platform, we want to copy the artifacts + // and archive it only if the build fails, to help with debugging. + failure { + sh '''#!/usr/bin/env bash - cleanup() { - script { - sh '''#!/usr/bin/env bash + cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}" + ''' + archiveArtifacts "out/**" + } - chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc - ''' + cleanup() { + script { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } } } - } - } - stage ('platform validation') { - parallel { // This is an optional stage which runs only when there's a change in // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is