Skip to content

Commit d59f8cf

Browse files
zhiyuanliang-mszhiyuanliang-ms
authored
Key vault secret refresh (#175)
* support startup retry and timeout * update * update * update * add testcase * clarify error type * update * update * update * fix lint * handle keyvault error * update * update * update * update * add secretRefreshIntervalInMs to KeyVaultOptions * update * update * handle keyvault reference error * update * wip * fix lint * add secret provider * add testcases * update * remove extra blank line * update * update * update * add boot loop protection * update * update * update testcase * update * update testcase * revert unintended change * update * update testcase * always cached secret with version * update * update error mesage * prevent refresh secrets operation when key alue refreshed * refresh secrets in parallel * update * enforce cache expiration for versioned secret * always clear cache * update
1 parent 867223e commit d59f8cf

17 files changed

+307
-122
lines changed

‎src/AzureAppConfigurationImpl.ts‎

Lines changed: 99 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { IKeyValueAdapter } from "./IKeyValueAdapter.js";
2222
import { JsonKeyValueAdapter } from "./JsonKeyValueAdapter.js";
2323
import { DEFAULT_STARTUP_TIMEOUT_IN_MS } from "./StartupOptions.js";
2424
import { DEFAULT_REFRESH_INTERVAL_IN_MS, MIN_REFRESH_INTERVAL_IN_MS } from "./refresh/refreshOptions.js";
25+
import { MIN_SECRET_REFRESH_INTERVAL_IN_MS } from "./keyvault/KeyVaultOptions.js";
2526
import { Disposable } from "./common/disposable.js";
2627
import {
2728
FEATURE_FLAGS_KEY_NAME,
@@ -91,16 +92,22 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
9192
/**
9293
* Aka watched settings.
9394
*/
95+
#refreshEnabled: boolean = false;
9496
#sentinels: ConfigurationSettingId[] = [];
9597
#watchAll: boolean = false;
9698
#kvRefreshInterval: number = DEFAULT_REFRESH_INTERVAL_IN_MS;
9799
#kvRefreshTimer: RefreshTimer;
98100

99101
// Feature flags
102+
#featureFlagEnabled: boolean = false;
103+
#featureFlagRefreshEnabled: boolean = false;
100104
#ffRefreshInterval: number = DEFAULT_REFRESH_INTERVAL_IN_MS;
101105
#ffRefreshTimer: RefreshTimer;
102106

103107
// Key Vault references
108+
#secretRefreshEnabled: boolean = false;
109+
#secretReferences: ConfigurationSetting[] = []; // cached key vault references
110+
#secretRefreshTimer: RefreshTimer;
104111
#resolveSecretsInParallel: boolean = false;
105112

106113
/**
@@ -129,14 +136,15 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
129136
this.#featureFlagTracing = new FeatureFlagTracingOptions();
130137
}
131138

132-
if (options?.trimKeyPrefixes) {
139+
if (options?.trimKeyPrefixes !== undefined) {
133140
this.#sortedTrimKeyPrefixes = [...options.trimKeyPrefixes].sort((a, b) => b.localeCompare(a));
134141
}
135142

136143
// if no selector is specified, always load key values using the default selector: key="*" and label="\0"
137144
this.#kvSelectors = getValidKeyValueSelectors(options?.selectors);
138145

139-
if (options?.refreshOptions?.enabled) {
146+
if (options?.refreshOptions?.enabled === true) {
147+
this.#refreshEnabled = true;
140148
const { refreshIntervalInMs, watchedSettings } = options.refreshOptions;
141149
if (watchedSettings === undefined || watchedSettings.length === 0) {
142150
this.#watchAll = true; // if no watched settings is specified, then watch all
@@ -156,53 +164,48 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
156164
if (refreshIntervalInMs !== undefined) {
157165
if (refreshIntervalInMs < MIN_REFRESH_INTERVAL_IN_MS) {
158166
throw new RangeError(`The refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
159-
} else {
160-
this.#kvRefreshInterval = refreshIntervalInMs;
161167
}
168+
this.#kvRefreshInterval = refreshIntervalInMs;
162169
}
163170
this.#kvRefreshTimer = new RefreshTimer(this.#kvRefreshInterval);
164171
}
165172

166173
// feature flag options
167-
if (options?.featureFlagOptions?.enabled) {
174+
if (options?.featureFlagOptions?.enabled === true) {
175+
this.#featureFlagEnabled = true;
168176
// validate feature flag selectors, only load feature flags when enabled
169177
this.#ffSelectors = getValidFeatureFlagSelectors(options.featureFlagOptions.selectors);
170178

171-
if (options.featureFlagOptions.refresh?.enabled) {
179+
if (options.featureFlagOptions.refresh?.enabled === true) {
180+
this.#featureFlagRefreshEnabled = true;
172181
const { refreshIntervalInMs } = options.featureFlagOptions.refresh;
173182
// custom refresh interval
174183
if (refreshIntervalInMs !== undefined) {
175184
if (refreshIntervalInMs < MIN_REFRESH_INTERVAL_IN_MS) {
176185
throw new RangeError(`The feature flag refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
177-
} else {
178-
this.#ffRefreshInterval = refreshIntervalInMs;
179186
}
187+
this.#ffRefreshInterval = refreshIntervalInMs;
180188
}
181189

182190
this.#ffRefreshTimer = new RefreshTimer(this.#ffRefreshInterval);
183191
}
184192
}
185193

186-
if (options?.keyVaultOptions?.parallelSecretResolutionEnabled) {
187-
this.#resolveSecretsInParallel = options.keyVaultOptions.parallelSecretResolutionEnabled;
194+
if (options?.keyVaultOptions !== undefined) {
195+
const { secretRefreshIntervalInMs } = options.keyVaultOptions;
196+
if (secretRefreshIntervalInMs !== undefined) {
197+
if (secretRefreshIntervalInMs < MIN_SECRET_REFRESH_INTERVAL_IN_MS) {
198+
throw new RangeError(`The Key Vault secret refresh interval cannot be less than ${MIN_SECRET_REFRESH_INTERVAL_IN_MS} milliseconds.`);
199+
}
200+
this.#secretRefreshEnabled = true;
201+
this.#secretRefreshTimer = new RefreshTimer(secretRefreshIntervalInMs);
202+
}
203+
this.#resolveSecretsInParallel = options.keyVaultOptions.parallelSecretResolutionEnabled ?? false;
188204
}
189-
190-
this.#adapters.push(new AzureKeyVaultKeyValueAdapter(options?.keyVaultOptions));
205+
this.#adapters.push(new AzureKeyVaultKeyValueAdapter(options?.keyVaultOptions, this.#secretRefreshTimer));
191206
this.#adapters.push(new JsonKeyValueAdapter());
192207
}
193208

194-
get #refreshEnabled(): boolean {
195-
return !!this.#options?.refreshOptions?.enabled;
196-
}
197-
198-
get #featureFlagEnabled(): boolean {
199-
return !!this.#options?.featureFlagOptions?.enabled;
200-
}
201-
202-
get #featureFlagRefreshEnabled(): boolean {
203-
return this.#featureFlagEnabled && !!this.#options?.featureFlagOptions?.refresh?.enabled;
204-
}
205-
206209
get #requestTraceOptions(): RequestTracingOptions {
207210
return {
208211
enabled: this.#requestTracingEnabled,
@@ -337,8 +340,8 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
337340
* Refreshes the configuration.
338341
*/
339342
async refresh(): Promise<void> {
340-
if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled) {
341-
throw new InvalidOperationError("Refresh is not enabled for key-values or feature flags.");
343+
if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled && !this.#secretRefreshEnabled) {
344+
throw new InvalidOperationError("Refresh is not enabled for key-values, feature flags or Key Vault secrets.");
342345
}
343346

344347
if (this.#refreshInProgress) {
@@ -356,8 +359,8 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
356359
* Registers a callback function to be called when the configuration is refreshed.
357360
*/
358361
onRefresh(listener: () => any, thisArg?: any): Disposable {
359-
if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled) {
360-
throw new InvalidOperationError("Refresh is not enabled for key-values or feature flags.");
362+
if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled && !this.#secretRefreshEnabled) {
363+
throw new InvalidOperationError("Refresh is not enabled for key-values, feature flags or Key Vault secrets.");
361364
}
362365

363366
const boundedListener = listener.bind(thisArg);
@@ -425,8 +428,20 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
425428

426429
async #refreshTasks(): Promise<void> {
427430
const refreshTasks: Promise<boolean>[] = [];
428-
if (this.#refreshEnabled) {
429-
refreshTasks.push(this.#refreshKeyValues());
431+
if (this.#refreshEnabled || this.#secretRefreshEnabled) {
432+
refreshTasks.push(
433+
this.#refreshKeyValues()
434+
.then(keyValueRefreshed => {
435+
// Only refresh secrets if key values didn't change and secret refresh is enabled
436+
// If key values are refreshed, all secret references will be refreshed as well.
437+
if (!keyValueRefreshed && this.#secretRefreshEnabled) {
438+
// Returns the refreshSecrets promise directly.
439+
// in a Promise chain, this automatically flattens nested Promises without requiring await.
440+
return this.#refreshSecrets();
441+
}
442+
return keyValueRefreshed;
443+
})
444+
);
430445
}
431446
if (this.#featureFlagRefreshEnabled) {
432447
refreshTasks.push(this.#refreshFeatureFlags());
@@ -530,35 +545,32 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
530545
* Loads selected key-values and watched settings (sentinels) for refresh from App Configuration to the local configuration.
531546
*/
532547
async #loadSelectedAndWatchedKeyValues() {
548+
this.#secretReferences = []; // clear all cached key vault reference configuration settings
533549
const keyValues: [key: string, value: unknown][] = [];
534550
const loadedSettings: ConfigurationSetting[] = await this.#loadConfigurationSettings();
535551
if (this.#refreshEnabled && !this.#watchAll) {
536552
await this.#updateWatchedKeyValuesEtag(loadedSettings);
537553
}
538554

539555
if (this.#requestTracingEnabled && this.#aiConfigurationTracing !== undefined) {
540-
// Reset old AI configuration tracing in order to track the information present in the current response from server.
556+
// reset old AI configuration tracing in order to track the information present in the current response from server
541557
this.#aiConfigurationTracing.reset();
542558
}
543559

544-
const secretResolutionPromises: Promise<void>[] = [];
545560
for (const setting of loadedSettings) {
546-
if (this.#resolveSecretsInParallel && isSecretReference(setting)) {
547-
// secret references are resolved asynchronously to improve performance
548-
const secretResolutionPromise = this.#processKeyValue(setting)
549-
.then(([key, value]) => {
550-
keyValues.push([key, value]);
551-
});
552-
secretResolutionPromises.push(secretResolutionPromise);
561+
if (isSecretReference(setting)) {
562+
this.#secretReferences.push(setting); // cache secret references for resolve/refresh secret separately
553563
continue;
554564
}
555565
// adapt configuration settings to key-values
556566
const [key, value] = await this.#processKeyValue(setting);
557567
keyValues.push([key, value]);
558568
}
559-
if (secretResolutionPromises.length > 0) {
560-
// wait for all secret resolution promises to be resolved
561-
await Promise.all(secretResolutionPromises);
569+
570+
if (this.#secretReferences.length > 0) {
571+
await this.#resolveSecretReferences(this.#secretReferences, (key, value) => {
572+
keyValues.push([key, value]);
573+
});
562574
}
563575

564576
this.#clearLoadedKeyValues(); // clear existing key-values in case of configuration setting deletion
@@ -626,7 +638,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
626638
*/
627639
async #refreshKeyValues(): Promise<boolean> {
628640
// if still within refresh interval/backoff, return
629-
if (!this.#kvRefreshTimer.canRefresh()) {
641+
if (this.#kvRefreshTimer === undefined || !this.#kvRefreshTimer.canRefresh()) {
630642
return Promise.resolve(false);
631643
}
632644

@@ -650,6 +662,9 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
650662
}
651663

652664
if (needRefresh) {
665+
for (const adapter of this.#adapters) {
666+
await adapter.onChangeDetected();
667+
}
653668
await this.#loadSelectedAndWatchedKeyValues();
654669
}
655670

@@ -663,7 +678,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
663678
*/
664679
async #refreshFeatureFlags(): Promise<boolean> {
665680
// if still within refresh interval/backoff, return
666-
if (!this.#ffRefreshTimer.canRefresh()) {
681+
if (this.#ffRefreshInterval === undefined || !this.#ffRefreshTimer.canRefresh()) {
667682
return Promise.resolve(false);
668683
}
669684

@@ -676,6 +691,25 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
676691
return Promise.resolve(needRefresh);
677692
}
678693

694+
async #refreshSecrets(): Promise<boolean> {
695+
// if still within refresh interval/backoff, return
696+
if (this.#secretRefreshTimer === undefined || !this.#secretRefreshTimer.canRefresh()) {
697+
return Promise.resolve(false);
698+
}
699+
700+
// if no cached key vault references, return
701+
if (this.#secretReferences.length === 0) {
702+
return Promise.resolve(false);
703+
}
704+
705+
await this.#resolveSecretReferences(this.#secretReferences, (key, value) => {
706+
this.#configMap.set(key, value);
707+
});
708+
709+
this.#secretRefreshTimer.reset();
710+
return Promise.resolve(true);
711+
}
712+
679713
/**
680714
* Checks whether the key-value collection has changed.
681715
* @param selectors - The @see PagedSettingSelector of the kev-value collection.
@@ -804,6 +838,27 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
804838
throw new Error("All fallback clients failed to get configuration settings.");
805839
}
806840

841+
async #resolveSecretReferences(secretReferences: ConfigurationSetting[], resultHandler: (key: string, value: unknown) => void): Promise<void> {
842+
if (this.#resolveSecretsInParallel) {
843+
const secretResolutionPromises: Promise<void>[] = [];
844+
for (const setting of secretReferences) {
845+
const secretResolutionPromise = this.#processKeyValue(setting)
846+
.then(([key, value]) => {
847+
resultHandler(key, value);
848+
});
849+
secretResolutionPromises.push(secretResolutionPromise);
850+
}
851+
852+
// Wait for all secret resolution promises to be resolved
853+
await Promise.all(secretResolutionPromises);
854+
} else {
855+
for (const setting of secretReferences) {
856+
const [key, value] = await this.#processKeyValue(setting);
857+
resultHandler(key, value);
858+
}
859+
}
860+
}
861+
807862
async #processKeyValue(setting: ConfigurationSetting<string>): Promise<[string, unknown]> {
808863
this.#setAIConfigurationTracing(setting);
809864

‎src/ConfigurationClientManager.ts‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import { ArgumentError } from "./common/error.js";
1212

1313
// Configuration client retry options
1414
const CLIENT_MAX_RETRIES = 2;
15-
const CLIENT_MAX_RETRY_DELAY = 60_000; // 1 minute in milliseconds
15+
const CLIENT_MAX_RETRY_DELAY_IN_MS = 60_000;
1616

1717
const TCP_ORIGIN_KEY_NAME = "_origin._tcp";
1818
const ALT_KEY_NAME = "_alt";
@@ -21,9 +21,9 @@ const ENDPOINT_KEY_NAME = "Endpoint";
2121
const ID_KEY_NAME = "Id";
2222
const SECRET_KEY_NAME = "Secret";
2323
const TRUSTED_DOMAIN_LABELS = [".azconfig.", ".appconfig."];
24-
const FALLBACK_CLIENT_EXPIRE_INTERVAL = 60 * 60 * 1000; // 1 hour in milliseconds
25-
const MINIMAL_CLIENT_REFRESH_INTERVAL = 30_000; // 30 seconds in milliseconds
26-
const DNS_RESOLVER_TIMEOUT = 3_000; // 3 seconds in milliseconds, in most cases, dns resolution should be within 200 milliseconds
24+
const FALLBACK_CLIENT_EXPIRE_INTERVAL_IN_MS = 60 * 60 * 1000;
25+
const MINIMAL_CLIENT_REFRESH_INTERVAL_IN_MS = 30_000;
26+
const DNS_RESOLVER_TIMEOUT_IN_MS = 3_000;
2727
const DNS_RESOLVER_TRIES = 2;
2828
const MAX_ALTNATIVE_SRV_COUNT = 10;
2929

@@ -120,11 +120,11 @@ export class ConfigurationClientManager {
120120
const currentTime = Date.now();
121121
// Filter static clients whose backoff time has ended
122122
let availableClients = this.#staticClients.filter(client => client.backoffEndTime <= currentTime);
123-
if (currentTime >= this.#lastFallbackClientRefreshAttempt + MINIMAL_CLIENT_REFRESH_INTERVAL &&
123+
if (currentTime >= this.#lastFallbackClientRefreshAttempt + MINIMAL_CLIENT_REFRESH_INTERVAL_IN_MS &&
124124
(!this.#dynamicClients ||
125125
// All dynamic clients are in backoff means no client is available
126126
this.#dynamicClients.every(client => currentTime < client.backoffEndTime) ||
127-
currentTime >= this.#lastFallbackClientUpdateTime + FALLBACK_CLIENT_EXPIRE_INTERVAL)) {
127+
currentTime >= this.#lastFallbackClientUpdateTime + FALLBACK_CLIENT_EXPIRE_INTERVAL_IN_MS)) {
128128
await this.#discoverFallbackClients(this.endpoint.hostname);
129129
return availableClients.concat(this.#dynamicClients);
130130
}
@@ -142,7 +142,7 @@ export class ConfigurationClientManager {
142142
async refreshClients() {
143143
const currentTime = Date.now();
144144
if (this.#isFailoverable &&
145-
currentTime >= this.#lastFallbackClientRefreshAttempt + MINIMAL_CLIENT_REFRESH_INTERVAL) {
145+
currentTime >= this.#lastFallbackClientRefreshAttempt + MINIMAL_CLIENT_REFRESH_INTERVAL_IN_MS) {
146146
await this.#discoverFallbackClients(this.endpoint.hostname);
147147
}
148148
}
@@ -185,7 +185,7 @@ export class ConfigurationClientManager {
185185

186186
try {
187187
// https://nodejs.org/api/dns.html#dnspromisesresolvesrvhostname
188-
const resolver = new this.#dns.Resolver({timeout: DNS_RESOLVER_TIMEOUT, tries: DNS_RESOLVER_TRIES});
188+
const resolver = new this.#dns.Resolver({timeout: DNS_RESOLVER_TIMEOUT_IN_MS, tries: DNS_RESOLVER_TRIES});
189189
// On success, resolveSrv() returns an array of SrvRecord
190190
// On failure, resolveSrv() throws an error with code 'ENOTFOUND'.
191191
const originRecords = await resolver.resolveSrv(`${TCP_ORIGIN_KEY_NAME}.${host}`); // look up SRV records for the origin host
@@ -266,7 +266,7 @@ function getClientOptions(options?: AzureAppConfigurationOptions): AppConfigurat
266266
// retry options
267267
const defaultRetryOptions = {
268268
maxRetries: CLIENT_MAX_RETRIES,
269-
maxRetryDelayInMs: CLIENT_MAX_RETRY_DELAY,
269+
maxRetryDelayInMs: CLIENT_MAX_RETRY_DELAY_IN_MS,
270270
};
271271
const retryOptions = Object.assign({}, defaultRetryOptions, options?.clientOptions?.retryOptions);
272272

‎src/IKeyValueAdapter.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,9 @@ export interface IKeyValueAdapter {
1313
* This method process the original configuration setting, and returns processed key and value in an array.
1414
*/
1515
processKeyValue(setting: ConfigurationSetting): Promise<[string, unknown]>;
16+
17+
/**
18+
* This method is called when a change is detected in the configuration setting.
19+
*/
20+
onChangeDetected(): Promise<void>;
1621
}

‎src/JsonKeyValueAdapter.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,8 @@ export class JsonKeyValueAdapter implements IKeyValueAdapter {
3535
}
3636
return [setting.key, parsedValue];
3737
}
38+
39+
async onChangeDetected(): Promise<void> {
40+
return;
41+
}
3842
}

‎src/StartupOptions.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT license.
33

4-
export const DEFAULT_STARTUP_TIMEOUT_IN_MS = 100 * 1000; // 100 seconds in milliseconds
4+
export const DEFAULT_STARTUP_TIMEOUT_IN_MS = 100_000;
55

66
export interface StartupOptions {
77
/**

0 commit comments

Comments
 (0)