From 977ecc2f32b2ea5678b68632e2c31735cf2ee1a5 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Mon, 21 Nov 2022 11:46:31 -0500 Subject: [PATCH 1/9] ci: skip ts-next tests --- .evergreen/config.yml | 8 -------- .evergreen/generate_evergreen_tasks.js | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0bbcafae0a0..af704324bc4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2090,14 +2090,6 @@ tasks: vars: NODE_LTS_NAME: erbium - func: run lint checks - - name: run-typescript-next - tags: - - run-typescript-next - commands: - - func: install dependencies - vars: - NODE_LTS_NAME: erbium - - func: run typescript next - name: run-typescript-current tags: - run-typescript-current diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 19da365d62f..9e3fc090e98 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -446,19 +446,19 @@ SINGLETON_TASKS.push( { func: 'run lint checks' } ] }, - { - name: 'run-typescript-next', - tags: ['run-typescript-next'], - commands: [ - { - func: 'install dependencies', - vars: { - NODE_LTS_NAME: LOWEST_LTS - } - }, - { func: 'run typescript next' } - ] - }, + // { + // name: 'run-typescript-next', + // tags: ['run-typescript-next'], + // commands: [ + // { + // func: 'install dependencies', + // vars: { + // NODE_LTS_NAME: LOWEST_LTS + // } + // }, + // { func: 'run typescript next' } + // ] + // }, { name: 'run-typescript-current', tags: ['run-typescript-current'], From cb68bf304ec8544bf98fe5e2e181849b9b05c263 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Mon, 21 Nov 2022 11:53:14 -0500 Subject: [PATCH 2/9] asdf --- .evergreen/config.yml | 9 ++++++++- .evergreen/generate_evergreen_tasks.js | 28 +++++++++++++------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index af704324bc4..719aeb0ebe3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2090,6 +2090,14 @@ tasks: vars: NODE_LTS_NAME: erbium - func: run lint checks + - name: run-typescript-next + tags: + - run-typescript-next + commands: + - func: install dependencies + vars: + NODE_LTS_NAME: erbium + - func: run typescript next - name: run-typescript-current tags: - run-typescript-current @@ -3036,7 +3044,6 @@ buildvariants: - run-lint-checks - run-typescript-current - run-typescript-oldest - - run-typescript-next - name: generate-combined-coverage display_name: Generate Combined Coverage run_on: rhel80-large diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 9e3fc090e98..4f911761ece 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -446,19 +446,19 @@ SINGLETON_TASKS.push( { func: 'run lint checks' } ] }, - // { - // name: 'run-typescript-next', - // tags: ['run-typescript-next'], - // commands: [ - // { - // func: 'install dependencies', - // vars: { - // NODE_LTS_NAME: LOWEST_LTS - // } - // }, - // { func: 'run typescript next' } - // ] - // }, + { + name: 'run-typescript-next', + tags: ['run-typescript-next'], + commands: [ + { + func: 'install dependencies', + vars: { + NODE_LTS_NAME: LOWEST_LTS + } + }, + { func: 'run typescript next' } + ] + }, { name: 'run-typescript-current', tags: ['run-typescript-current'], @@ -497,7 +497,7 @@ BUILD_VARIANTS.push({ 'run-lint-checks', 'run-typescript-current', 'run-typescript-oldest', - 'run-typescript-next' + // 'run-typescript-next' ] }); From b33edd07a293497a79073f926912ceec9a85b91e Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 22 Nov 2022 15:36:22 -0500 Subject: [PATCH 3/9] fix(NODE-4830): lazily import aws module (#3476) --- src/cmap/auth/mongodb_aws.ts | 4 +++- src/deps.ts | 28 ++++++++++++++++------------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/cmap/auth/mongodb_aws.ts b/src/cmap/auth/mongodb_aws.ts index 57f0cc1baf2..fc7f8bef8f6 100644 --- a/src/cmap/auth/mongodb_aws.ts +++ b/src/cmap/auth/mongodb_aws.ts @@ -4,7 +4,7 @@ import * as url from 'url'; import type { Binary, BSONSerializeOptions } from '../../bson'; import * as BSON from '../../bson'; -import { aws4, credentialProvider } from '../../deps'; +import { aws4, getAwsCredentialProvider } from '../../deps'; import { MongoAWSError, MongoCompatibilityError, @@ -198,6 +198,8 @@ function makeTempCredentials(credentials: MongoCredentials, callback: Callback Promise; }; -export let credentialProvider: CredentialProvider | { kModuleError: MongoMissingDependencyError } = - makeErrorModule( - new MongoMissingDependencyError( - 'Optional module `@aws-sdk/credential-providers` not found.' + - ' Please install it to enable getting aws credentials via the official sdk.' - ) - ); - -try { - // Ensure you always wrap an optional require in the try block NODE-3199 - credentialProvider = require('@aws-sdk/credential-providers'); -} catch {} // eslint-disable-line +export function getAwsCredentialProvider(): + | CredentialProvider + | { kModuleError: MongoMissingDependencyError } { + try { + // Ensure you always wrap an optional require in the try block NODE-3199 + const credentialProvider = require('@aws-sdk/credential-providers'); + return credentialProvider; + } catch { + return makeErrorModule( + new MongoMissingDependencyError( + 'Optional module `@aws-sdk/credential-providers` not found.' + + ' Please install it to enable getting aws credentials via the official sdk.' + ) + ); + } +} type SnappyLib = { [PKG_VERSION]: { major: number; minor: number; patch: number }; From 24998c6ee9a797630849ce4b57fe721c27ccc53a Mon Sep 17 00:00:00 2001 From: Durran Jordan Date: Tue, 22 Nov 2022 23:04:29 +0100 Subject: [PATCH 4/9] fix(NODE-4831): check map value is not undefined (#3477) --- src/cmap/connection.ts | 2 +- test/unit/cmap/connection.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/cmap/connection.ts b/src/cmap/connection.ts index 7aa913fbe40..85f60008174 100644 --- a/src/cmap/connection.ts +++ b/src/cmap/connection.ts @@ -384,7 +384,7 @@ export class Connection extends TypedEventEmitter { } else { // Get the first orphaned operation description. const entry = this[kQueue].entries().next(); - if (entry) { + if (entry.value != null) { const [requestId, orphaned]: [number, OperationDescription] = entry.value; // If the orphaned operation description exists then set it. operationDescription = orphaned; diff --git a/test/unit/cmap/connection.test.ts b/test/unit/cmap/connection.test.ts index a9b95bcea2c..5c8d872bb84 100644 --- a/test/unit/cmap/connection.test.ts +++ b/test/unit/cmap/connection.test.ts @@ -287,6 +287,34 @@ describe('new Connection()', function () { }); }); + context('when no operation description is in the queue', function () { + const document = { ok: 1 }; + + beforeEach(function () { + // @ts-expect-error: driverSocket does not fully satisfy the stream type, but that's okay + connection = sinon.spy(new Connection(driverSocket, connectionOptionsDefaults)); + connection.isMonitoringConnection = true; + const queueSymbol = getSymbolFrom(connection, 'queue'); + queue = connection[queueSymbol]; + }); + + it('does not error', function () { + const msg = generateOpMsgBuffer(document); + const msgHeader: MessageHeader = { + length: msg.readInt32LE(0), + requestId: 2, + responseTo: 1, + opCode: msg.readInt32LE(12) + }; + const msgBody = msg.subarray(16); + + const message = new BinMsg(msg, msgHeader, msgBody); + expect(() => { + connection.onMessage(message); + }).to.not.throw(); + }); + }); + context('when more than one operation description is in the queue', function () { let spyOne; let spyTwo; From 9b0bd3c838293e361094a80fe5ef8aee31915cf3 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 23 Nov 2022 13:43:52 -0500 Subject: [PATCH 5/9] chore(release): 4.12.1 --- HISTORY.md | 8 ++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index a266f6ba691..62c63384b73 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [4.12.1](https://github.com/mongodb/node-mongodb-native/compare/v4.12.0...v4.12.1) (2022-11-23) + + +### Bug Fixes + +* **NODE-4830:** lazily import aws module ([#3476](https://github.com/mongodb/node-mongodb-native/issues/3476)) ([ff375e9](https://github.com/mongodb/node-mongodb-native/commit/ff375e9cfb84c85b7036a7ef0f87385d3ec126bb)) +* **NODE-4831:** check map value is not undefined ([#3477](https://github.com/mongodb/node-mongodb-native/issues/3477)) ([9795cdb](https://github.com/mongodb/node-mongodb-native/commit/9795cdb19f8e652b4dc4badd4fe8e6d1a7b837a6)) + ## [4.12.0](https://github.com/mongodb/node-mongodb-native/compare/v4.11.0...v4.12.0) (2022-11-16) diff --git a/package-lock.json b/package-lock.json index d91d133885a..e663264da2a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mongodb", - "version": "4.12.0", + "version": "4.12.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mongodb", - "version": "4.12.0", + "version": "4.12.1", "license": "Apache-2.0", "dependencies": { "bson": "^4.7.0", diff --git a/package.json b/package.json index aeca0e6342e..d1ad236aee9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mongodb", - "version": "4.12.0", + "version": "4.12.1", "description": "The official MongoDB driver for Node.js", "main": "lib/index.js", "files": [ From 5fd8de36cb5496e6ce1741f8db3d570c5be7563e Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Mon, 28 Nov 2022 15:13:41 -0500 Subject: [PATCH 6/9] feat(NODE-4691): interrupt in-flight operations on heartbeat failure (#3457) --- src/cmap/connection_pool.ts | 58 ++++++++++++++++--- src/cmap/errors.ts | 27 +++++++-- src/error.ts | 1 + src/sdam/monitor.ts | 5 +- src/sdam/server.ts | 6 +- src/sdam/topology.ts | 6 +- src/sessions.ts | 2 +- ...ection_monitoring_and_pooling.spec.test.ts | 45 ++++++++------ ...rver_discovery_and_monitoring.spec.test.ts | 13 +---- test/tools/cmap_spec_runner.ts | 6 +- test/unit/sdam/server.test.ts | 4 +- 11 files changed, 118 insertions(+), 55 deletions(-) diff --git a/src/cmap/connection_pool.ts b/src/cmap/connection_pool.ts index f621b4f9b4c..5c8cbc97654 100644 --- a/src/cmap/connection_pool.ts +++ b/src/cmap/connection_pool.ts @@ -41,7 +41,12 @@ import { ConnectionPoolReadyEvent, ConnectionReadyEvent } from './connection_pool_events'; -import { PoolClearedError, PoolClosedError, WaitQueueTimeoutError } from './errors'; +import { + PoolClearedError, + PoolClearedOnNetworkError, + PoolClosedError, + WaitQueueTimeoutError +} from './errors'; import { ConnectionPoolMetrics } from './metrics'; /** @internal */ @@ -382,6 +387,9 @@ export class ConnectionPool extends TypedEventEmitter { * @param connection - The connection to check in */ checkIn(connection: Connection): void { + if (!this[kCheckedOut].has(connection)) { + return; + } const poolClosed = this.closed; const stale = this.connectionIsStale(connection); const willDestroy = !!(poolClosed || stale || connection.closed); @@ -408,13 +416,19 @@ export class ConnectionPool extends TypedEventEmitter { * Pool reset is handled by incrementing the pool's generation count. Any existing connection of a * previous generation will eventually be pruned during subsequent checkouts. */ - clear(serviceId?: ObjectId): void { + clear(options: { serviceId?: ObjectId; interruptInUseConnections?: boolean } = {}): void { if (this.closed) { return; } // handle load balanced case - if (this.loadBalanced && serviceId) { + if (this.loadBalanced) { + const { serviceId } = options; + if (!serviceId) { + throw new MongoRuntimeError( + 'ConnectionPool.clear() called in load balanced mode with no serviceId.' + ); + } const sid = serviceId.toHexString(); const generation = this.serviceGenerations.get(sid); // Only need to worry if the generation exists, since it should @@ -431,19 +445,42 @@ export class ConnectionPool extends TypedEventEmitter { ); return; } - // handle non load-balanced case + const interruptInUseConnections = options.interruptInUseConnections ?? false; + const oldGeneration = this[kGeneration]; this[kGeneration] += 1; const alreadyPaused = this[kPoolState] === PoolState.paused; this[kPoolState] = PoolState.paused; this.clearMinPoolSizeTimer(); if (!alreadyPaused) { - this.emit(ConnectionPool.CONNECTION_POOL_CLEARED, new ConnectionPoolClearedEvent(this)); + this.emit( + ConnectionPool.CONNECTION_POOL_CLEARED, + new ConnectionPoolClearedEvent(this, { interruptInUseConnections }) + ); + } + + if (interruptInUseConnections) { + process.nextTick(() => this.interruptInUseConnections(oldGeneration)); } + this.processWaitQueue(); } + /** + * Closes all stale in-use connections in the pool with a resumable PoolClearedOnNetworkError. + * + * Only connections where `connection.generation <= minGeneration` are killed. + */ + private interruptInUseConnections(minGeneration: number) { + for (const connection of this[kCheckedOut]) { + if (connection.generation <= minGeneration) { + this.checkIn(connection); + connection.onError(new PoolClearedOnNetworkError(this)); + } + } + } + /** Close the pool */ close(callback: Callback): void; close(options: CloseOptions, callback: Callback): void; @@ -572,7 +609,12 @@ export class ConnectionPool extends TypedEventEmitter { return !!(this.options.maxIdleTimeMS && connection.idleTime > this.options.maxIdleTimeMS); } - private connectionIsPerished(connection: Connection) { + /** + * Destroys a connection if the connection is perished. + * + * @returns `true` if the connection was destroyed, `false` otherwise. + */ + private destroyConnectionIfPerished(connection: Connection): boolean { const isStale = this.connectionIsStale(connection); const isIdle = this.connectionIsIdle(connection); if (!isStale && !isIdle && !connection.closed) { @@ -658,7 +700,7 @@ export class ConnectionPool extends TypedEventEmitter { return; } - this[kConnections].prune(connection => this.connectionIsPerished(connection)); + this[kConnections].prune(connection => this.destroyConnectionIfPerished(connection)); if ( this.totalConnectionCount < minPoolSize && @@ -734,7 +776,7 @@ export class ConnectionPool extends TypedEventEmitter { break; } - if (!this.connectionIsPerished(connection)) { + if (!this.destroyConnectionIfPerished(connection)) { this[kCheckedOut].add(connection); this.emit( ConnectionPool.CONNECTION_CHECKED_OUT, diff --git a/src/cmap/errors.ts b/src/cmap/errors.ts index 4e0d25a27c7..f6d2ed58880 100644 --- a/src/cmap/errors.ts +++ b/src/cmap/errors.ts @@ -1,4 +1,4 @@ -import { MongoDriverError, MongoNetworkError } from '../error'; +import { MongoDriverError, MongoErrorLabel, MongoNetworkError } from '../error'; import type { ConnectionPool } from './connection_pool'; /** @@ -27,11 +27,14 @@ export class PoolClearedError extends MongoNetworkError { /** The address of the connection pool */ address: string; - constructor(pool: ConnectionPool) { - super( - `Connection pool for ${pool.address} was cleared because another operation failed with: "${pool.serverError?.message}"` - ); + constructor(pool: ConnectionPool, message?: string) { + const errorMessage = message + ? message + : `Connection pool for ${pool.address} was cleared because another operation failed with: "${pool.serverError?.message}"`; + super(errorMessage); this.address = pool.address; + + this.addErrorLabel(MongoErrorLabel.RetryableWriteError); } override get name(): string { @@ -39,6 +42,20 @@ export class PoolClearedError extends MongoNetworkError { } } +/** + * An error indicating that a connection pool has been cleared after the monitor for that server timed out. + * @category Error + */ +export class PoolClearedOnNetworkError extends PoolClearedError { + constructor(pool: ConnectionPool) { + super(pool, `Connection to ${pool.address} interrupted due to server monitor timeout`); + } + + override get name(): string { + return 'PoolClearedOnNetworkError'; + } +} + /** * An error thrown when a request to check out a connection times out * @category Error diff --git a/src/error.ts b/src/error.ts index 800be95c8b8..1dd426cb4fa 100644 --- a/src/error.ts +++ b/src/error.ts @@ -91,6 +91,7 @@ export const MongoErrorLabel = Object.freeze({ ResumableChangeStreamError: 'ResumableChangeStreamError', HandshakeError: 'HandshakeError', ResetPool: 'ResetPool', + InterruptInUseConnections: 'InterruptInUseConnections', NoWritesPerformed: 'NoWritesPerformed' } as const); diff --git a/src/sdam/monitor.ts b/src/sdam/monitor.ts index 3711dc59ed4..b35093b435a 100644 --- a/src/sdam/monitor.ts +++ b/src/sdam/monitor.ts @@ -4,7 +4,7 @@ import { Document, Long } from '../bson'; import { connect } from '../cmap/connect'; import { Connection, ConnectionOptions } from '../cmap/connection'; import { LEGACY_HELLO_COMMAND } from '../constants'; -import { MongoError, MongoErrorLabel } from '../error'; +import { MongoError, MongoErrorLabel, MongoNetworkTimeoutError } from '../error'; import { CancellationToken, TypedEventEmitter } from '../mongo_types'; import type { Callback } from '../utils'; import { calculateDurationInMs, EventEmitterWithState, makeStateMachine, now, ns } from '../utils'; @@ -221,6 +221,9 @@ function checkServer(monitor: Monitor, callback: Callback) { const error = !(err instanceof MongoError) ? new MongoError(err) : err; error.addErrorLabel(MongoErrorLabel.ResetPool); + if (error instanceof MongoNetworkTimeoutError) { + error.addErrorLabel(MongoErrorLabel.InterruptInUseConnections); + } monitor.emit('resetServer', error); callback(err); diff --git a/src/sdam/server.ts b/src/sdam/server.ts index a693bf3bc8f..ae7a1fd5f6a 100644 --- a/src/sdam/server.ts +++ b/src/sdam/server.ts @@ -354,8 +354,6 @@ export class Server extends TypedEventEmitter { } if (!(err instanceof PoolClearedError)) { this.handleError(err); - } else { - err.addErrorLabel(MongoErrorLabel.RetryableWriteError); } return cb(err); } @@ -400,14 +398,14 @@ export class Server extends TypedEventEmitter { error.addErrorLabel(MongoErrorLabel.ResetPool); markServerUnknown(this, error); } else if (connection) { - this.s.pool.clear(connection.serviceId); + this.s.pool.clear({ serviceId: connection.serviceId }); } } else { if (isSDAMUnrecoverableError(error)) { if (shouldHandleStateChangeError(this, error)) { const shouldClearPool = maxWireVersion(this) <= 7 || isNodeShuttingDownError(error); if (this.loadBalanced && connection && shouldClearPool) { - this.s.pool.clear(connection.serviceId); + this.s.pool.clear({ serviceId: connection.serviceId }); } if (!this.loadBalanced) { diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index d157ef63f69..601ae2c3824 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -839,7 +839,11 @@ function updateServers(topology: Topology, incomingServerDescription?: ServerDes incomingServerDescription.error instanceof MongoError && incomingServerDescription.error.hasErrorLabel(MongoErrorLabel.ResetPool) ) { - server.s.pool.clear(); + const interruptInUseConnections = incomingServerDescription.error.hasErrorLabel( + MongoErrorLabel.InterruptInUseConnections + ); + + server.s.pool.clear({ interruptInUseConnections }); } else if (incomingServerDescription.error == null) { const newTopologyType = topology.s.description.type; const shouldMarkPoolReady = diff --git a/src/sessions.ts b/src/sessions.ts index 21468aff11d..f2b7d5281e0 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -537,7 +537,7 @@ export function maybeClearPinnedConnection( ); if (options?.forceClear) { - loadBalancer.s.pool.clear(conn.serviceId); + loadBalancer.s.pool.clear({ serviceId: conn.serviceId }); } } diff --git a/test/integration/connection-monitoring-and-pooling/connection_monitoring_and_pooling.spec.test.ts b/test/integration/connection-monitoring-and-pooling/connection_monitoring_and_pooling.spec.test.ts index 66b0b40d28a..ac1066a168a 100644 --- a/test/integration/connection-monitoring-and-pooling/connection_monitoring_and_pooling.spec.test.ts +++ b/test/integration/connection-monitoring-and-pooling/connection_monitoring_and_pooling.spec.test.ts @@ -12,34 +12,43 @@ const LB_SKIP_TESTS: SkipDescription[] = [ 'clearing a paused pool emits no events', 'after clear, cannot check out connections until pool ready', 'readying a ready pool emits no events', - 'error during minPoolSize population clears pool' + 'error during minPoolSize population clears pool', + 'Connections MUST be interrupted as soon as possible (interruptInUseConnections=true)' ].map(description => ({ description, skipIfCondition: 'loadBalanced', skipReason: 'cannot run against a load balanced environment' })); -const INTERRUPT_IN_USE_CONNECTIONS_TESTS: SkipDescription[] = [ - 'Connections MUST be interrupted as soon as possible (interruptInUseConnections=true)', - 'Pool clear SHOULD schedule the next background thread run immediately (interruptInUseConnections: false)', - 'clear with interruptInUseConnections = true closes pending connections' -].map(description => ({ - description, - skipIfCondition: 'always', - skipReason: 'TODO(NODE-4691): cancel inflight operations when heartbeat fails' -})); +const INTERRUPT_IN_USE_SKIPPED_TESTS: SkipDescription[] = [ + { + description: 'clear with interruptInUseConnections = true closes pending connections', + skipIfCondition: 'always', + skipReason: 'TODO(NODE-4784): track and kill pending connections' + }, + { + description: + 'Pool clear SHOULD schedule the next background thread run immediately (interruptInUseConnections: false)', + skipIfCondition: 'always', + skipReason: + 'NodeJS does not have a background thread responsible for managing connections, and so already checked in connections are not pruned when in-use connections are interrupted.' + } +]; describe('Connection Monitoring and Pooling Spec Tests (Integration)', function () { const tests: CmapTest[] = loadSpecTests('connection-monitoring-and-pooling'); runCmapTestSuite(tests, { - testsToSkip: LB_SKIP_TESTS.concat([ - { - description: 'waiting on maxConnecting is limited by WaitQueueTimeoutMS', - skipIfCondition: 'always', - skipReason: - 'not applicable: waitQueueTimeoutMS limits connection establishment time in our driver' - } - ]).concat(INTERRUPT_IN_USE_CONNECTIONS_TESTS) + testsToSkip: LB_SKIP_TESTS.concat( + [ + { + description: 'waiting on maxConnecting is limited by WaitQueueTimeoutMS', + skipIfCondition: 'always', + skipReason: + 'not applicable: waitQueueTimeoutMS limits connection establishment time in our driver' + } + ], + INTERRUPT_IN_USE_SKIPPED_TESTS + ) }); }); diff --git a/test/integration/server-discovery-and-monitoring/server_discovery_and_monitoring.spec.test.ts b/test/integration/server-discovery-and-monitoring/server_discovery_and_monitoring.spec.test.ts index a990353acaa..74fc67efc61 100644 --- a/test/integration/server-discovery-and-monitoring/server_discovery_and_monitoring.spec.test.ts +++ b/test/integration/server-discovery-and-monitoring/server_discovery_and_monitoring.spec.test.ts @@ -4,16 +4,5 @@ import { loadSpecTests } from '../../spec'; import { runUnifiedSuite } from '../../tools/unified-spec-runner/runner'; describe('SDAM Unified Tests', function () { - const sdamPoolClearedTests = [ - 'Connection pool clear uses interruptInUseConnections=true after monitor timeout', - 'Error returned from connection pool clear with interruptInUseConnections=true is retryable', - 'Error returned from connection pool clear with interruptInUseConnections=true is retryable for write' - ]; - runUnifiedSuite( - loadSpecTests(path.join('server-discovery-and-monitoring', 'unified')), - ({ description }) => - sdamPoolClearedTests.includes(description) - ? 'TODO(NODE-4691): interrupt in-use operations on heartbeat failure' - : false - ); + runUnifiedSuite(loadSpecTests(path.join('server-discovery-and-monitoring', 'unified'))); }); diff --git a/test/tools/cmap_spec_runner.ts b/test/tools/cmap_spec_runner.ts index d5384fc287e..634e732f3e9 100644 --- a/test/tools/cmap_spec_runner.ts +++ b/test/tools/cmap_spec_runner.ts @@ -197,10 +197,8 @@ const getTestOpDefinitions = (threadContext: ThreadContext) => ({ return threadContext.pool.checkIn(connection); }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - clear: function (interruptInUseConnections: boolean) { - // TODO(NODE-4619): pass interruptInUseConnections into clear pool method - return threadContext.pool.clear(); + clear: function ({ interruptInUseConnections }: { interruptInUseConnections: boolean }) { + return threadContext.pool.clear({ interruptInUseConnections }); }, close: async function () { return await promisify(ConnectionPool.prototype.close).call(threadContext.pool); diff --git a/test/unit/sdam/server.test.ts b/test/unit/sdam/server.test.ts index cf810d87541..188c290ba41 100644 --- a/test/unit/sdam/server.test.ts +++ b/test/unit/sdam/server.test.ts @@ -105,7 +105,9 @@ describe('Server', () => { expect(newDescription).to.have.nested.property('[0].type', ServerType.Unknown); } else { expect(newDescription).to.be.undefined; - expect(server.s.pool.clear).to.have.been.calledOnceWith(connection!.serviceId); + expect(server.s.pool.clear).to.have.been.calledOnceWith({ + serviceId: connection!.serviceId + }); } }); From 77eb35c889d5578b3f97f9d4322e12cc28af1a88 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 29 Nov 2022 16:57:18 -0500 Subject: [PATCH 7/9] break apart ts into compile and check steps --- .evergreen/config.in.yml | 26 +++----- .evergreen/config.yml | 63 ++++++++++--------- .evergreen/generate_evergreen_tasks.js | 85 ++++++++++++++++---------- .evergreen/run-typescript.sh | 34 ++++++++--- 4 files changed, 119 insertions(+), 89 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 1d33e55a7f9..5fa63fea913 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -289,7 +289,7 @@ functions: args: - "${PROJECT_DIRECTORY}/.evergreen/run-unit-tests.sh" - "run typescript next": + "check types": - command: subprocess.exec type: test params: @@ -297,12 +297,13 @@ functions: timeout_secs: 60 env: PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} - TS_VERSION: "next" + TS_VERSION: ${TS_VERSION} + TS_CHECK: CHECK_TYPES binary: bash args: - "${PROJECT_DIRECTORY}/.evergreen/run-typescript.sh" - "run typescript oldest": + "compile driver": - command: subprocess.exec type: test params: @@ -310,21 +311,8 @@ functions: timeout_secs: 60 env: PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} - TS_VERSION: "4.1.6" - TRY_COMPILING_DRIVER: "false" # 4.1.6 can consume the public API but not compile the driver - binary: bash - args: - - "${PROJECT_DIRECTORY}/.evergreen/run-typescript.sh" - - "run typescript current": - - command: subprocess.exec - type: test - params: - working_dir: "src" - timeout_secs: 60 - env: - PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} - TS_VERSION: "" # leaving this empty will use the version in package-lock.json + TS_VERSION: ${TS_VERSION} + TS_CHECK: COMPILE_DRIVER binary: bash args: - "${PROJECT_DIRECTORY}/.evergreen/run-typescript.sh" @@ -886,7 +874,7 @@ functions: local_file: src/coverage/coverage-final.json optional: true # Upload the coverage report for all tasks in a single build to the same directory. - # TODO NODE-4707 - change upload directory to ${UPLOAD_BUCKET} + # TODO NODE-4707 - change upload directory to ${UPLOAD_BUCKET} # This change will require changing the `download and merge coverage` func as well remote_file: mongo-node-driver/${revision}/${version_id}/coverage.${build_variant}.${task_name}.json bucket: mciuploads diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 719aeb0ebe3..322406200e3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -255,7 +255,7 @@ functions: binary: bash args: - ${PROJECT_DIRECTORY}/.evergreen/run-unit-tests.sh - run typescript next: + check types: - command: subprocess.exec type: test params: @@ -263,24 +263,12 @@ functions: timeout_secs: 60 env: PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} - TS_VERSION: next - binary: bash - args: - - ${PROJECT_DIRECTORY}/.evergreen/run-typescript.sh - run typescript oldest: - - command: subprocess.exec - type: test - params: - working_dir: src - timeout_secs: 60 - env: - PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} - TS_VERSION: 4.1.6 - TRY_COMPILING_DRIVER: 'false' + TS_VERSION: ${TS_VERSION} + TS_CHECK: CHECK_TYPES binary: bash args: - ${PROJECT_DIRECTORY}/.evergreen/run-typescript.sh - run typescript current: + compile driver: - command: subprocess.exec type: test params: @@ -288,7 +276,8 @@ functions: timeout_secs: 60 env: PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} - TS_VERSION: '' + TS_VERSION: ${TS_VERSION} + TS_CHECK: COMPILE_DRIVER binary: bash args: - ${PROJECT_DIRECTORY}/.evergreen/run-typescript.sh @@ -2090,30 +2079,46 @@ tasks: vars: NODE_LTS_NAME: erbium - func: run lint checks - - name: run-typescript-next + - name: check-types-typescript-next tags: - - run-typescript-next + - check-types-typescript-next commands: - func: install dependencies vars: NODE_LTS_NAME: erbium - - func: run typescript next - - name: run-typescript-current + - func: check types + vars: + TS_VERSION: next + - name: compile-driver-typescript-current tags: - - run-typescript-current + - compile-driver-typescript-current commands: - func: install dependencies vars: NODE_LTS_NAME: erbium - - func: run typescript current - - name: run-typescript-oldest + - func: compile driver + vars: + TS_VERSION: current + - name: check-types-typescript-current tags: - - run-typescript-oldest + - check-types-typescript-current commands: - func: install dependencies vars: NODE_LTS_NAME: erbium - - func: run typescript oldest + - func: check types + vars: + TS_VERSION: current + - name: check-types-typescript-4.1.6 + tags: + - check-types-typescript-4.1.6 + commands: + - func: install dependencies + vars: + NODE_LTS_NAME: erbium + - func: check types + vars: + TS_VERSION: 4.1.6 - name: run-mongosh-integration-tests tags: - run-mongosh-integration-tests @@ -3042,8 +3047,10 @@ buildvariants: tasks: - run-unit-tests - run-lint-checks - - run-typescript-current - - run-typescript-oldest + - check-types-typescript-next + - compile-driver-typescript-current + - check-types-typescript-current + - check-types-typescript-4.1.6 - name: generate-combined-coverage display_name: Generate Combined Coverage run_on: rhel80-large diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 4f911761ece..5d497f4a5dc 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -446,22 +446,38 @@ SINGLETON_TASKS.push( { func: 'run lint checks' } ] }, - { - name: 'run-typescript-next', - tags: ['run-typescript-next'], - commands: [ - { - func: 'install dependencies', - vars: { - NODE_LTS_NAME: LOWEST_LTS - } - }, - { func: 'run typescript next' } - ] - }, - { - name: 'run-typescript-current', - tags: ['run-typescript-current'], + ...Array.from(makeTypescriptTasks()) + ] +); + +function* makeTypescriptTasks() { + for (const TS_VERSION of ["next", "current", "4.1.6"]) { + // 4.1.6 can consume the public API but not compile the driver + if (TS_VERSION !== '4.1.6' + && TS_VERSION !== 'next') { + yield { + name: `compile-driver-typescript-${TS_VERSION}`, + tags: [`compile-driver-typescript-${TS_VERSION}`], + commands: [ + { + func: 'install dependencies', + vars: { + NODE_LTS_NAME: LOWEST_LTS + } + }, + { + func: 'compile driver', + vars: { + TS_VERSION + } + } + ] + } + } + + yield { + name: `check-types-typescript-${TS_VERSION}`, + tags: [`check-types-typescript-${TS_VERSION}`], commands: [ { func: 'install dependencies', @@ -469,24 +485,29 @@ SINGLETON_TASKS.push( NODE_LTS_NAME: LOWEST_LTS } }, - { func: 'run typescript current' } - ] - }, - { - name: 'run-typescript-oldest', - tags: ['run-typescript-oldest'], - commands: [ { - func: 'install dependencies', + func: 'check types', vars: { - NODE_LTS_NAME: LOWEST_LTS + TS_VERSION } - }, - { func: 'run typescript oldest' } + } ] - } - ] -); + } + } + return { + name: 'run-typescript-next', + tags: ['run-typescript-next'], + commands: [ + { + func: 'install dependencies', + vars: { + NODE_LTS_NAME: LOWEST_LTS + } + }, + { func: 'run typescript next' } + ] + } +} BUILD_VARIANTS.push({ name: 'lint', @@ -495,9 +516,7 @@ BUILD_VARIANTS.push({ tasks: [ 'run-unit-tests', 'run-lint-checks', - 'run-typescript-current', - 'run-typescript-oldest', - // 'run-typescript-next' + ...Array.from(makeTypescriptTasks()).map(({ name }) => name) ] }); diff --git a/.evergreen/run-typescript.sh b/.evergreen/run-typescript.sh index 314caa01679..a8fdd71edb7 100644 --- a/.evergreen/run-typescript.sh +++ b/.evergreen/run-typescript.sh @@ -5,14 +5,25 @@ source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" set -o xtrace -function get_current_ts_version { - node -e "console.log(require('./package-lock.json').dependencies.typescript.version)" -} +if [ -z "$TS_CHECK" ]; then echo "TS_CHECK must be set to either COMPILE_DRIVER or CHECK_TYPES"; exit 1; fi +if [ -z "$TS_VERSION" ]; then echo "TS_VERSION must be set"; exit 1; fi -CURRENT_TS_VERSION=$(get_current_ts_version) +if [ ! -f "mongodb.d.ts" ]; then + # mongodb.d.ts should always exist because of the installation in prior steps + # but in case it doesn't, build it + npm i +fi + +function get_ts_version { + if [ "$TS_VERSION" == "current" ]; then + echo $(node -e "console.log(require('./package-lock.json').dependencies.typescript.version)") + else + echo $TS_VERSION + fi +} export TSC="./node_modules/typescript/bin/tsc" -export TS_VERSION=${TS_VERSION:=$CURRENT_TS_VERSION} +export TS_VERSION=$(get_ts_version) npm install --no-save --force typescript@"$TS_VERSION" @@ -21,9 +32,14 @@ echo "Typescript $($TSC -v)" # check resolution uses the default latest types echo "import * as mdb from '.'" > file.ts && node $TSC --noEmit --traceResolution file.ts | grep 'mongodb.d.ts' && rm file.ts -# check compilation -node $TSC mongodb.d.ts - -if [[ $TRY_COMPILING_DRIVER != "false" ]]; then +if [ "$TS_CHECK" == "COMPILE_DRIVER" ]; then + echo "compiling driver" npm run build:ts +elif [ "$TS_CHECK" == "CHECK_TYPES" ]; then + echo "checking driver types" + # check compilation + node $TSC mongodb.d.ts +else + "Invalid value $TS_CHECK for TS_CHECK environment variable." + exit 1 fi From 40403eb0cbc0fa1534ccf253728edd974a819d86 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 30 Nov 2022 13:53:14 -0500 Subject: [PATCH 8/9] Update .evergreen/run-typescript.sh Co-authored-by: Neal Beeken --- .evergreen/run-typescript.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-typescript.sh b/.evergreen/run-typescript.sh index a8fdd71edb7..26911439511 100644 --- a/.evergreen/run-typescript.sh +++ b/.evergreen/run-typescript.sh @@ -14,7 +14,7 @@ if [ ! -f "mongodb.d.ts" ]; then npm i fi -function get_ts_version { +function get_ts_version() { if [ "$TS_VERSION" == "current" ]; then echo $(node -e "console.log(require('./package-lock.json').dependencies.typescript.version)") else From db0058ab3f4e74188b236402a5129d11c685f5d2 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 30 Nov 2022 14:09:40 -0500 Subject: [PATCH 9/9] add check to valid values of TS_CHECK --- .evergreen/run-typescript.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.evergreen/run-typescript.sh b/.evergreen/run-typescript.sh index 26911439511..0a33654cf6b 100644 --- a/.evergreen/run-typescript.sh +++ b/.evergreen/run-typescript.sh @@ -5,12 +5,18 @@ source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" set -o xtrace -if [ -z "$TS_CHECK" ]; then echo "TS_CHECK must be set to either COMPILE_DRIVER or CHECK_TYPES"; exit 1; fi +case $TS_CHECK in + COMPILE_DRIVER|CHECK_TYPES) # Ok + ;; + *) + echo "TS_CHECK must be set to either COMPILE_DRIVER or CHECK_TYPES - received '$TS_CHECK'" + exit 1 +esac + if [ -z "$TS_VERSION" ]; then echo "TS_VERSION must be set"; exit 1; fi if [ ! -f "mongodb.d.ts" ]; then - # mongodb.d.ts should always exist because of the installation in prior steps - # but in case it doesn't, build it + echo "mongodb.d.ts should always exist because of the installation in prior steps but in case it doesn't, build it" npm i fi