Merge pull request #522 from cmu-delphi/feature-jenkins-deployments

Add new Jenkins deployment workflow

Author: krivard

Jenkinsfile

@@ -1,90 +1,72 @@
 #!groovy
 
-// import shared library: https://github.com/cmu-delphi/jenkins-shared-library
+// Import shared lib.
 @Library('jenkins-shared-library') _
 
-pipeline {
+/*
+   Declare variables.
+   - indicator_list should contain all the indicators to handle in the pipeline.
+   - Keep in sync with '.github/workflows/python-ci.yml'.
+   - TODO: #527 Get this list automatically from python-ci.yml at runtime.
+ */
+def indicator_list = ["cdc_covidnet", "claims_hosp", "combo_cases_and_deaths", "google_symptoms", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph", "safegraph_patterns", "usafacts"]
+def build_package = [:]
+def deploy_staging = [:]
+def deploy_production = [:]
 
+pipeline {
     agent any
-
     stages {
-
-        stage ("Environment") {            
+        stage('Build and Package') {
             when {
-                anyOf {
-                    branch "deploy-*";
-                    changeRequest target: "deploy-*", comparator: "GLOB"
-                }
+                branch "main";
             }
             steps {
                 script {
-                    // Get the indicator name from the pipeline env.
-                    if ( env.CHANGE_TARGET ) {
-                        INDICATOR = env.CHANGE_TARGET.replaceAll("deploy-", "")
+                    indicator_list.each { indicator ->
+                        build_package[indicator] = {
+                            sh "jenkins/build-and-package.sh ${indicator}"
+                        }
                     }
-                    else if ( env.BRANCH_NAME ) {
-                        INDICATOR = env.BRANCH_NAME.replaceAll("deploy-", "")
-                    }
-                    else {
-                        INDICATOR = ""
-                    }
-                } 
-            }
-        }
-
-        stage('Build') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-build.sh"
-            }
-        }
-
-        stage('Test') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-test.sh"
-            }
-        }
-        
-        stage('Package') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-package.sh"
+                    parallel build_package
+                }
             }
         }
-
-        stage('Deploy to staging env') {
+        stage('Deploy staging') {
             when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
+                branch "main";
             }
             steps {
-                sh "jenkins/jenkins-deploy-staging.sh ${INDICATOR}"
+                script {
+                    indicator_list.each { indicator ->
+                        deploy_staging[indicator] = {
+                            sh "jenkins/deploy-staging.sh ${indicator}"
+                        }
+                    }
+                    parallel deploy_staging
+                }
             }
         }
-
-        stage('Deploy') {
+        stage('Deploy production') {
             when {
-                branch "deploy-*"
+                branch "prod";
             }
             steps {
-                sh "jenkins/${INDICATOR}-jenkins-deploy.sh"
+                script {
+                    indicator_list.each { indicator ->
+                        deploy_production[indicator] = {
+                            sh "jenkins/deploy-production.sh ${indicator}"
+                        }
+                    }
+                    parallel deploy_production
+                }
             }
         }
     }
-
     post {
         always {
             script {
-                /*
-                Use slackNotifier.groovy from shared library and provide current
-                build result as parameter.
-                */   
+                // Use slackNotifier.groovy from shared lib.
                 slackNotifier(currentBuild.currentResult)
             }
         }

jenkins/build-and-package.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Jenkins build-and-package
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+# Vars
+local_indicator=$1
+
+#
+# Build
+#
+
+cd "${WORKSPACE}/${local_indicator}" || exit
+
+# Set up venv
+python -m venv env
+source env/bin/activate
+pip install ../_delphi_utils_python/.
+pip install .
+
+#
+# Package
+#
+
+cd "${WORKSPACE}" || exit
+
+# Create .tar.gz for deployment
+tar -czvf "${JENKINS_HOME}/artifacts/${local_indicator}.tar.gz" "${local_indicator}"

jenkins/deploy-production.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Jenkins deploy
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Deploy
+#
+
+local_indicator=$1
+
+cd "${WORKSPACE}/ansible" || exit
+
+# Ansible!
+ansible-playbook ansible-deploy.yaml --extra-vars "indicator=${local_indicator}" -i inventory

jenkins/deploy-staging.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Jenkins deploy
+#
+
+set -eo pipefail
+source ~/.bash_profile
+
+#
+# Deploy
+#
+
+local_indicator=$1
+
+cd "${WORKSPACE}/ansible" || exit
+
+# Ansible!
+ansible-playbook ansible-deploy-staging.yaml --extra-vars "indicator=${local_indicator}" -i inventory