refactor!(NODE-3295): Support node 12+ and reconfigure CI

Author: nbbeeken

.evergreen/config.yml

@@ -245,25 +245,35 @@ functions:
           . ./prepare_atlas_connectivity.sh
           rm -f ./prepare_atlas_connectivity.sh
 
-          NODE_LTS_NAME='${NODE_LTS_NAME}' bash ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh
+          export PROJECT_DIRECTORY="$(pwd)"
+          export NODE_LTS_NAME='${NODE_LTS_NAME}'
+
+          bash ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh
 
   "run kerberos tests":
     - command: shell.exec
       type: test
       params:
         working_dir: src
         script: |
-          KRB5_KEYTAB='${gssapi_auth_keytab_base64}' KRB5_PRINCIPAL='${gssapi_auth_principal}' \
-          MONGODB_URI='${gssapi_auth_mongodb_uri}' \
-          NODE_LTS_NAME='${NODE_LTS_NAME}' bash ${PROJECT_DIRECTORY}/.evergreen/run-kerberos-tests.sh
+          export PROJECT_DIRECTORY="$(pwd)"
+          export KRB5_KEYTAB='${gssapi_auth_keytab_base64}'
+          export KRB5_PRINCIPAL='${gssapi_auth_principal}'
+          export MONGODB_URI='${gssapi_auth_mongodb_uri}'
+          export NODE_LTS_NAME='${NODE_LTS_NAME}'
+
+          bash ${PROJECT_DIRECTORY}/.evergreen/run-kerberos-tests.sh
 
   "run ldap tests":
     - command: shell.exec
       type: test
       params:
         working_dir: "src"
         script: |
-          MONGODB_URI='${plain_auth_mongodb_uri}' NODE_LTS_NAME='${NODE_LTS_NAME}' \
+          export PROJECT_DIRECTORY="$(pwd)"
+          export MONGODB_URI='${plain_auth_mongodb_uri}'
+          export NODE_LTS_NAME='${NODE_LTS_NAME}'
+
           bash ${PROJECT_DIRECTORY}/.evergreen/run-ldap-tests.sh
 
   "run data lake tests":
@@ -272,7 +282,10 @@ functions:
         params:
           working_dir: src
           script: |
-            MONGODB_URI='mongodb://mhuser:pencil@localhost' NODE_LTS_NAME='${NODE_LTS_NAME}' \
+            export PROJECT_DIRECTORY="$(pwd)"
+            export MONGODB_URI='mongodb://mhuser:pencil@localhost'
+            export NODE_LTS_NAME='${NODE_LTS_NAME}'
+
             bash ${PROJECT_DIRECTORY}/.evergreen/run-data-lake-tests.sh
 
   "run tls tests":
@@ -281,9 +294,14 @@ functions:
       params:
         working_dir: "src"
         script: |
-          NODE_LTS_NAME=${NODE_LTS_NAME} DRIVERS_TOOLS="${DRIVERS_TOOLS}" \
-          SSL_CA_FILE="${SSL_CA_FILE}" SSL_KEY_FILE="${SSL_KEY_FILE}" \
-          MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-tls-tests.sh
+          export PROJECT_DIRECTORY="$(pwd)"
+          export NODE_LTS_NAME=${NODE_LTS_NAME}
+          export DRIVERS_TOOLS="${DRIVERS_TOOLS}"
+          export SSL_CA_FILE="${SSL_CA_FILE}"
+          export SSL_KEY_FILE="${SSL_KEY_FILE}"
+          export MONGODB_URI="${MONGODB_URI}"
+
+          bash ${PROJECT_DIRECTORY}/.evergreen/run-tls-tests.sh
 
   "add aws auth variables to file":
     - command: shell.exec
@@ -497,14 +515,18 @@ functions:
       params:
         script: |
           cd ${DRIVERS_TOOLS}/.evergreen/ocsp
-          /opt/mongodbtoolchain/v3/bin/python3 -m venv ./venv
-          ./venv/bin/pip3 install -r ${DRIVERS_TOOLS}/.evergreen/ocsp/mock-ocsp-responder-requirements.txt
+          export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo "/opt/mongodbtoolchain/v3/bin/python3")
+          export PYTHON_VENV_PATH=$([ "Windows_NT" = "$OS" ] && echo "./venv/Scripts" || echo "./venv/bin")
+
+          $PYTHON_BINARY -m venv ./venv
+          $PYTHON_VENV_PATH/pip3 install -r ${DRIVERS_TOOLS}/.evergreen/ocsp/mock-ocsp-responder-requirements.txt
+
     - command: shell.exec
       params:
         background: true
         script: |
           cd ${DRIVERS_TOOLS}/.evergreen/ocsp
-          nohup ./venv/bin/python3 ocsp_mock.py \
+          nohup $PYTHON_VENV_PATH/python3 ocsp_mock.py \
             --ca_file rsa/ca.pem \
             --ocsp_responder_cert rsa/ca.crt \
             --ocsp_responder_key rsa/ca.key \
@@ -515,14 +537,19 @@ functions:
       params:
         script: |
           cd ${DRIVERS_TOOLS}/.evergreen/ocsp
-          /opt/mongodbtoolchain/v3/bin/python3 -m venv ./venv
-          ./venv/bin/pip3 install -r ${DRIVERS_TOOLS}/.evergreen/ocsp/mock-ocsp-responder-requirements.txt
+
+          export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo "/opt/mongodbtoolchain/v3/bin/python3")
+          export PYTHON_VENV_PATH=$([ "Windows_NT" = "$OS" ] && echo "./venv/Scripts" || echo "./venv/bin")
+
+          $PYTHON_BINARY -m venv ./venv
+          $PYTHON_VENV_PATH/pip3 install -r ${DRIVERS_TOOLS}/.evergreen/ocsp/mock-ocsp-responder-requirements.txt
+
     - command: shell.exec
       params:
         background: true
         script: |
           cd ${DRIVERS_TOOLS}/.evergreen/ocsp
-          nohup ./venv/bin/python3 ocsp_mock.py \
+          nohup $PYTHON_VENV_PATH/python3 ocsp_mock.py \
             --ca_file rsa/ca.pem \
             --ocsp_responder_cert rsa/ca.crt \
             --ocsp_responder_key rsa/ca.key \

.evergreen/config.yml.in

@@ -3,62 +3,40 @@ const fs = require('fs');
 const yaml = require('js-yaml');
 
 const LATEST_EFFECTIVE_VERSION = '5.0';
-const MONGODB_VERSIONS = ['latest', '4.4', '4.2', '4.0', '3.6', '3.4', '3.2', '3.0', '2.6'];
-const NODE_VERSIONS = ['dubnium', 'erbium', 'fermium'];
+const MONGODB_VERSIONS = ['latest', '5.0', '4.4', '4.2', '4.0', '3.6', '3.4', '3.2', '3.0', '2.6'];
+const LOWEST_LTS = 'erbium';
+const NODE_VERSIONS = [LOWEST_LTS, 'fermium'];
 const TOPOLOGIES = ['server', 'replica_set', 'sharded_cluster'];
-const AWS_AUTH_VERSIONS = ['latest', '4.4'];
-const OCSP_VERSIONS = ['latest', '4.4'];
-const TLS_VERSIONS = ['latest', '4.2']; // also test on 4.2 because 4.4+ currently skipped on windows
+const AWS_AUTH_VERSIONS = ['latest', '5.0', '4.4'];
+const OCSP_VERSIONS = ['latest', '5.0', '4.4'];
+const TLS_VERSIONS = ['latest', '5.0', '4.4', '4.2'];
+
+const DEFAULT_OS = 'ubuntu1804-large'
 
 const OPERATING_SYSTEMS = [
   {
     name: 'macos-1014',
     display_name: 'macOS 10.14',
-    run_on: 'macos-1014',
-    auth: false
-  },
-  {
-    name: 'rhel70',
-    display_name: 'RHEL 7.0',
-    run_on: 'rhel70-small'
-  },
-  {
-    name: 'ubuntu-14.04',
-    display_name: 'Ubuntu 14.04',
-    run_on: 'ubuntu1404-large',
-    mongoVersion: '<4.2'
+    run_on: 'macos-1014'
   },
   {
     name: 'ubuntu-18.04',
     display_name: 'Ubuntu 18.04',
-    run_on: 'ubuntu1804-large',
-    mongoVersion: '>=3.2',
-    clientEncryption: true
-  },
-  {
-    name: 'windows-64-vs2015',
-    display_name: 'Windows (VS2015)',
-    run_on: 'windows-64-vs2015-large',
-    msvsVersion: 2015,
-    mongoVersion: '<4.4'
+    run_on: 'ubuntu1804-large'
   },
   {
-    name: 'windows-64-vs2017',
-    display_name: 'Windows (VS2017)',
-    run_on: 'windows-64-vs2017-large',
-    msvsVersion: 2017,
-    mongoVersion: '<4.4'
+    name: 'windows-64-vs2019',
+    display_name: 'Windows (VS2019)',
+    run_on: 'windows-64-vs2019-large',
+    msvsVersion: 2019
   }
-].map(osConfig =>
-  Object.assign(
-    {
-      mongoVersion: '>=2.6',
-      nodeVersion: 'dubnium',
-      auth: false
-    },
-    osConfig
-  )
-);
+].map(osConfig => ({
+  mongoVersion: '>=2.6',
+  nodeVersion: LOWEST_LTS,
+  auth: false, // TODO test auth?
+  clientEncryption: false, // TODO(NODE-3372): Reenable with the next FLE release
+  ...osConfig
+}));
 
 // TODO: NODE-3060: enable skipped tests on windows
 const WINDOWS_SKIP_TAGS = new Set(['atlas-connect', 'auth']);
@@ -112,7 +90,7 @@ BASE_TASKS.push({
 });
 
 // manually added tasks
-Array.prototype.push.apply(TASKS, [
+TASKS.push(...[
   {
     name: 'test-atlas-connectivity',
     tags: ['atlas-connect'],
@@ -276,7 +254,7 @@ TLS_VERSIONS.forEach(VERSION => {
 
 OCSP_VERSIONS.forEach(VERSION => {
   // manually added tasks
-  Array.prototype.push.apply(TASKS, [
+  TASKS.push(...[
     {
       name: `test-${VERSION}-ocsp-valid-cert-server-staples`,
       tags: ['ocsp'],
@@ -400,7 +378,7 @@ OCSP_VERSIONS.forEach(VERSION => {
 const AWS_AUTH_TASKS = [];
 
 AWS_AUTH_VERSIONS.forEach(VERSION => {
-  const name = (ex) => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
+  const name = ex => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
   const aws_funcs = [
     { func: 'run aws auth test with regular aws credentials' },
     { func: 'run aws auth test with assume role credentials' },
@@ -427,13 +405,12 @@ AWS_AUTH_VERSIONS.forEach(VERSION => {
       { func: 'setup aws env' },
       fn
     ]
-  }))
+  }));
 
   TASKS.push(...aws_tasks);
-  AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name))
+  AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name));
 });
 
-
 const BUILD_VARIANTS = [];
 
 const getTaskList = (() => {
@@ -516,29 +493,33 @@ SINGLETON_TASKS.push({
     {
       func: 'install dependencies',
       vars: {
-        NODE_LTS_NAME: 'erbium'
+        NODE_LTS_NAME: LOWEST_LTS
       }
     },
     { func: 'run checks' }
   ]
 });
 
-BUILD_VARIANTS.push({
-  name: 'lint',
-  display_name: 'lint',
-  run_on: 'rhel70',
-  tasks: ['run-checks']
-}, {
-  name: 'ubuntu1804-custom-csfle-tests',
-  display_name: 'Custom FLE Version Test',
-  run_on: 'ubuntu1804-test',
-  tasks: ['run-custom-csfle-tests']
-},{
-  name: 'ubuntu1804-run-bson-ext-test',
-  display_name: 'BSON EXT Test',
-  run_on: 'ubuntu1804-test',
-  tasks: ['run-bson-ext-test']
-});
+BUILD_VARIANTS.push(
+  {
+    name: 'lint',
+    display_name: 'lint',
+    run_on: DEFAULT_OS,
+    tasks: ['run-checks']
+  },
+  {
+    name: 'ubuntu1804-custom-csfle-tests',
+    display_name: 'Custom FLE Version Test',
+    run_on: DEFAULT_OS,
+    tasks: ['run-custom-csfle-tests']
+  },
+  {
+    name: 'ubuntu1804-run-bson-ext-test',
+    display_name: 'BSON EXT Test',
+    run_on: DEFAULT_OS,
+    tasks: ['run-bson-ext-test']
+  }
+);
 
 // singleton build variant for mongosh integration tests
 SINGLETON_TASKS.push({
@@ -569,7 +550,7 @@ BUILD_VARIANTS.push({
   display_name: 'MONGODB-AWS Auth test',
   run_on: 'ubuntu1804-test',
   expansions: {
-    NODE_LTS_NAME: 'dubnium'
+    NODE_LTS_NAME: LOWEST_LTS
   },
   tasks: AWS_AUTH_TASKS
 });
@@ -582,8 +563,8 @@ SINGLETON_TASKS.push({
     {
       func: 'install dependencies',
       vars: {
-        NODE_LTS_NAME: 'erbium',
-      },
+        NODE_LTS_NAME: LOWEST_LTS
+      }
     },
     {
       func: 'bootstrap mongo-orchestration',
@@ -604,8 +585,8 @@ SINGLETON_TASKS.push({
     {
       func: 'install dependencies',
       vars: {
-        NODE_LTS_NAME: 'fermium',
-      },
+        NODE_LTS_NAME: LOWEST_LTS
+      }
     },
     {
       func: 'bootstrap mongo-orchestration',
@@ -614,9 +595,10 @@ SINGLETON_TASKS.push({
         TOPOLOGY: 'server'
       }
     },
-    { func: 'run bson-ext test',
+    {
+      func: 'run bson-ext test',
       vars: {
-        NODE_LTS_NAME: 'fermium',
+        NODE_LTS_NAME: LOWEST_LTS
       }
     }
   ]

.evergreen/generate_evergreen_tasks.js

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+export PATH="/opt/mongodbtoolchain/v2/bin:$PATH"
+NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
+export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
+
+if [[ "$OS" == "Windows_NT" ]]; then
+    NVM_HOME=$(cygpath -w "$NVM_DIR")
+    export NVM_HOME
+    NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
+    export NVM_SYMLINK
+    NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
+    export NVM_ARTIFACTS_PATH
+    PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
+    export PATH
+    echo "updated path on windows PATH=$PATH"
+else
+    [ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh"
+fi