diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b831934e8..4fdfe46b03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,12 +32,41 @@ jobs: run: npm ci - name: CI Self-Check run: npm run ci:check + - name: Create PR to upgrade CI environments if needed + if: ${{ always() }} + uses: peter-evans/create-pull-request@v3 + with: + title: Upgrade CI environments + body: This is an automated PR to upgrade package versions in CI environments. + commit-message: Upgrading versions of MongoDB / Node.js + branch: upgrade-ci-environments + base: master + check-lint: + name: Lint + timeout-minutes: 30 + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.NODE_VERSION }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Cache Node.js modules + uses: actions/cache@v2 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ matrix.NODE_VERSION }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ matrix.NODE_VERSION }}- + - name: Install dependencies + run: npm ci + - run: npm run lint check-mongo: strategy: matrix: include: - name: Mongo 4.4, ReplicaSet, WiredTiger - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: replicaset MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 14.15.5 @@ -58,27 +87,28 @@ jobs: NODE_VERSION: 14.15.5 - name: Redis Cache PARSE_SERVER_TEST_CACHE: redis - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 14.15.5 - name: Node 10 - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 10.23.3 - name: Node 12 - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 12.20.2 - name: Node 15 - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 15.8.0 name: ${{ matrix.name }} timeout-minutes: 30 + needs: check-ci runs-on: ubuntu-18.04 services: redis: @@ -106,8 +136,6 @@ jobs: ${{ runner.os }}-node-${{ matrix.NODE_VERSION }}- - name: Install dependencies run: npm ci - - if: ${{ matrix.name == 'Mongo 3.6.21' }} - run: npm run lint - run: npm run pretest - run: npm run coverage env: @@ -129,6 +157,7 @@ jobs: POSTGRES_IMAGE: postgis/postgis:13-3.1 name: ${{ matrix.name }} timeout-minutes: 30 + needs: check-ci runs-on: ubuntu-18.04 services: redis: diff --git a/package-lock.json b/package-lock.json index 01a2bd7794..be1db21b9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3997,6 +3997,14 @@ "parse-json": "^5.0.0", "path-type": "^4.0.0", "yaml": "^1.7.2" + }, + "dependencies": { + "yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "dev": true + } } }, "cross-env": { @@ -11915,9 +11923,9 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" }, "yaml": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", - "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "version": "2.0.0-3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.0.0-3.tgz", + "integrity": "sha512-gvtVaY+/mQ0OsXgaWy2Tf830JuXN7qEUYdXWsuiJVSkMRsBBQ90YVpQQofaURbhoA1xSbLBf7965oH6ddzNbBQ==", "dev": true }, "yargs": { diff --git a/package.json b/package.json index 753795d9ee..21480593a4 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "node-fetch": "2.6.1", "nyc": "15.1.0", "prettier": "2.0.5", - "yaml": "1.10.0" + "yaml": "2.0.0-3" }, "scripts": { "ci:check": "node ./resources/ci/ciCheck.js", diff --git a/resources/ci/CiVersionCheck.js b/resources/ci/CiVersionCheck.js index 763d907355..0ddfa813ed 100644 --- a/resources/ci/CiVersionCheck.js +++ b/resources/ci/CiVersionCheck.js @@ -1,6 +1,7 @@ const core = require('@actions/core'); const semver = require('semver'); const yaml = require('yaml'); +const lodash = require('lodash'); const fs = require('fs').promises; /** @@ -43,6 +44,8 @@ class CiVersionCheck { * test against 2.0.0. * If the latest version component is `major` then the check would * fail and recommend an upgrade to version 2.0.0. + * @param {Boolean} [config.updateYaml=false] Is true if the YAML file + * should be updated and package versions should be bumped. */ constructor(config) { const { @@ -54,6 +57,7 @@ class CiVersionCheck { releasedVersions, ignoreReleasedVersions = [], latestComponent = CiVersionCheck.versionComponents.patch, + updateYaml = false, } = config; // Ensure required params are set @@ -64,6 +68,9 @@ class CiVersionCheck { ciEnvironmentsKeyPath, ciVersionKey, releasedVersions, + ignoreReleasedVersions, + latestComponent, + updateYaml, ].includes(undefined)) { throw 'invalid configuration'; } @@ -80,6 +87,7 @@ class CiVersionCheck { this.releasedVersions = releasedVersions; this.ignoreReleasedVersions = ignoreReleasedVersions; this.latestComponent = latestComponent; + this.updateYaml = updateYaml; } /** @@ -101,9 +109,10 @@ class CiVersionCheck { // Get CI workflow const ciYaml = await fs.readFile(this.yamlFilePath, 'utf-8'); const ci = yaml.parse(ciYaml); + this.ci = ci; // Extract package versions - let versions = this.ciEnvironmentsKeyPath.split('.').reduce((o,k) => o !== undefined ? o[k] : undefined, ci); + let versions = this._getKeyAtPath(ci, this.ciEnvironmentsKeyPath); versions = Object.entries(versions) .map(entry => entry[1]) .filter(entry => entry[this.ciVersionKey]); @@ -114,6 +123,16 @@ class CiVersionCheck { } } + /** + * Returns the value at a given key path. + * @param {Object} obj The object to traverse. + * @param {String} keyPath The path to the key to return. + * @returns {Any} The value at the given key path. + */ + _getKeyAtPath(obj, keyPath) { + return keyPath.split('.').reduce((o,k) => o !== undefined ? o[k] : undefined, obj); + } + /** * Returns the package versions which are missing in the CI environment. * @param {Array} releasedVersions The released versions; need to @@ -144,11 +163,7 @@ class CiVersionCheck { // ]; // Determine operator for range comparison - const operator = versionComponent == CiVersionCheck.versionComponents.major - ? '>=' - : versionComponent == CiVersionCheck.versionComponents.minor - ? '^' - : '~' + const operator = this._getOperatorForVersionComponent(versionComponent); // Get all untested versions const untestedVersions = releasedVersions.reduce((m, v) => { @@ -187,11 +202,7 @@ class CiVersionCheck { */ getNewerVersion(versions, version, versionComponent) { // Determine operator for range comparison - const operator = versionComponent == CiVersionCheck.versionComponents.major - ? '>=' - : versionComponent == CiVersionCheck.versionComponents.minor - ? '^' - : '~' + const operator = this._getOperatorForVersionComponent(versionComponent); const latest = semver.maxSatisfying(versions, `${operator}${version}`); return semver.gt(latest, version) ? latest : undefined; } @@ -209,6 +220,57 @@ class CiVersionCheck { } } + /** + * Returns the semver range operator that relates to the a version component. + * For example, the operator for the `patch` version component defines a + * range up to the highest patch version. + * @param {String} component The version component (`patch`, `minor`, + * `major`). + * @returns {String} The semver operator. + */ + _getOperatorForVersionComponent(component) { + // Determine operator for range comparison + return component == CiVersionCheck.versionComponents.major + ? '>=' + : component == CiVersionCheck.versionComponents.minor + ? '^' + : '~' + } + + /** + * Updates a key in the CI YAML file with a given value. + * @param {Array} update The key to update. + * @param {String} update.old The old value to update. + * @param {String} update.new The new value to set. + */ + async _updateYaml(update) { + if (this.ci === undefined) { + throw 'YAML file has not been read.'; + } + + // Get environments + const newCi = lodash.cloneDeep(this.ci); + const envs = this._getKeyAtPath(newCi, this.ciEnvironmentsKeyPath); + + // Update keys + for (const key of Object.keys(update)) { + for (const oldValue of Object.keys(update[key])) { + const newValue = update[key][oldValue]; + + // Update value + for (const env of envs) { + if (env[key] == oldValue) { + env[key] = newValue; + } + } + } + } + + // Write to file + const newYaml = yaml.stringify(newCi); + await fs.writeFile(this.yamlFilePath, newYaml); + } + /** * Runs the check. */ @@ -233,6 +295,9 @@ class CiVersionCheck { // Is true if any of the checks failed let failed = false; + // The keys that should be updated + const keyUpdatesNeeded = {}; + // Check whether each tested version is the latest patch for (const test of tests) { const version = test[this.ciVersionKey]; @@ -249,6 +314,7 @@ class CiVersionCheck { if (newer) { console.log(`āŒ CI environment '${test.name}' uses an old ${this.packageName} ${this.latestComponent} version ${version} instead of ${newer}.`); failed = true; + keyUpdatesNeeded[this.ciVersionKey] = Object.assign(keyUpdatesNeeded[this.ciVersionKey] || {}, { [version]: newer }); } else { console.log(`āœ… CI environment '${test.name}' uses the latest ${this.packageName} ${this.latestComponent} version ${version}.`); } @@ -268,9 +334,19 @@ class CiVersionCheck { core.setFailed( `CI environments are not up-to-date with the latest ${this.packageName} versions.` + `\n\nCheck the error messages above and update the ${this.packageName} versions in the CI YAML ` + - `file.\n\nā„¹ļø Additionally, there may be versions of ${this.packageName} that have reached their official end-of-life ` + - `support date and should be removed from the CI, see ${this.packageSupportUrl}.` + `file. Additionally, check for versions of ${this.packageName} that have reached their official end-of-life ` + + `support date and may be removed from the CI, see ${this.packageSupportUrl}.` ); + + // If packages in YAML file should be updated + if (this.updateYaml && Object.keys(keyUpdatesNeeded).length > 0) { + try { + this._updateYaml(keyUpdatesNeeded); + } catch (e) { + console.log(`Failed to update ${this.packageName} versions in YAML file with error: ${e}`); + } + console.log(`\nšŸš€ Updated YAML file to use newer versions of ${this.packageName}. Check the Pull Request list for a PR.`); + } } } catch (e) { diff --git a/resources/ci/ciCheck.js b/resources/ci/ciCheck.js index 66ee6d9aa0..e855e69742 100644 --- a/resources/ci/ciCheck.js +++ b/resources/ci/ciCheck.js @@ -32,6 +32,7 @@ async function checkMongoDbVersions() { ciVersionKey: 'MONGODB_VERSION', releasedVersions, latestComponent: CiVersionCheck.versionComponents.path, + updateYaml: true, ignoreReleasedVersions: [ '<3.6.0', // These versions have reached their MongoDB end-of-life support date '~3.7.0', // This is a development release according to MongoDB support @@ -58,6 +59,7 @@ async function checkNodeVersions() { ciVersionKey: 'NODE_VERSION', releasedVersions, latestComponent: CiVersionCheck.versionComponents.minor, + updateYaml: true, ignoreReleasedVersions: [ '<10.0.0', // These versions have reached their end-of-life support date '>=11.0.0 <12.0.0', // These versions have reached their end-of-life support date