Failover support (#98)

* resolve conflicts

* resolve conflicts

* resolve conflicts

* add tests

* resolve conflicts and update

* fix lint

* resolve conflicts

* resolve comments

* update package-lock

* update

* update

* update failover error

* update

* update

* update failoverable error with 'ENOTFOUND'

* fix lint

* update

* added ENOENT error

* update

* update error message in test

* update test

* update test

* update

* resolve conflicts

Author: linglingye001

package-lock.json

@@ -35,6 +35,7 @@ import { AzureKeyVaultKeyValueAdapter } from "./keyvault/AzureKeyVaultKeyValueAd
 import { RefreshTimer } from "./refresh/RefreshTimer.js";
 import { getConfigurationSettingWithTrace, listConfigurationSettingsWithTrace, requestTracingEnabled } from "./requestTracing/utils.js";
 import { KeyFilter, LabelFilter, SettingSelector } from "./types.js";
+import { ConfigurationClientManager } from "./ConfigurationClientManager.js";
 
 type PagedSettingSelector = SettingSelector & {
     /**
@@ -56,10 +57,10 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      */
     #sortedTrimKeyPrefixes: string[] | undefined;
     readonly #requestTracingEnabled: boolean;
-    #client: AppConfigurationClient;
-    #clientEndpoint: string | undefined;
+    #clientManager: ConfigurationClientManager;
     #options: AzureAppConfigurationOptions | undefined;
     #isInitialLoadCompleted: boolean = false;
+    #isFailoverRequest: boolean = false;
 
     // Refresh
     #refreshInterval: number = DEFAULT_REFRESH_INTERVAL_IN_MS;
@@ -78,13 +79,11 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
     #featureFlagSelectors: PagedSettingSelector[] = [];
 
     constructor(
-        client: AppConfigurationClient,
-        clientEndpoint: string | undefined,
-        options: AzureAppConfigurationOptions | undefined
+        clientManager: ConfigurationClientManager,
+        options: AzureAppConfigurationOptions | undefined,
     ) {
-        this.#client = client;
-        this.#clientEndpoint = clientEndpoint;
         this.#options = options;
+        this.#clientManager = clientManager;
 
         // Enable request tracing if not opt-out
         this.#requestTracingEnabled = requestTracingEnabled();
@@ -197,35 +196,66 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         return {
             requestTracingEnabled: this.#requestTracingEnabled,
             initialLoadCompleted: this.#isInitialLoadCompleted,
-            appConfigOptions: this.#options
+            appConfigOptions: this.#options,
+            isFailoverRequest: this.#isFailoverRequest
         };
     }
 
-    async #loadSelectedKeyValues(): Promise<ConfigurationSetting[]> {
-        const loadedSettings: ConfigurationSetting[] = [];
+    async #executeWithFailoverPolicy(funcToExecute: (client: AppConfigurationClient) => Promise<any>): Promise<any> {
+        const clientWrappers = await this.#clientManager.getClients();
 
-        // validate selectors
-        const selectors = getValidKeyValueSelectors(this.#options?.selectors);
+        let successful: boolean;
+        for (const clientWrapper of clientWrappers) {
+            successful = false;
+            try {
+                const result = await funcToExecute(clientWrapper.client);
+                this.#isFailoverRequest = false;
+                successful = true;
+                clientWrapper.updateBackoffStatus(successful);
+                return result;
+            } catch (error) {
+                if (isFailoverableError(error)) {
+                    clientWrapper.updateBackoffStatus(successful);
+                    this.#isFailoverRequest = true;
+                    continue;
+                }
 
-        for (const selector of selectors) {
-            const listOptions: ListConfigurationSettingsOptions = {
-                keyFilter: selector.keyFilter,
-                labelFilter: selector.labelFilter
-            };
+                throw error;
+            }
+        }
 
-            const settings = listConfigurationSettingsWithTrace(
-                this.#requestTraceOptions,
-                this.#client,
-                listOptions
-            );
+        this.#clientManager.refreshClients();
+        throw new Error("All clients failed to get configuration settings.");
+    }
 
-            for await (const setting of settings) {
-                if (!isFeatureFlag(setting)) { // exclude feature flags
-                    loadedSettings.push(setting);
+    async #loadSelectedKeyValues(): Promise<ConfigurationSetting[]> {
+        // validate selectors
+        const selectors = getValidKeyValueSelectors(this.#options?.selectors);
+
+        const funcToExecute = async (client) => {
+            const loadedSettings: ConfigurationSetting[] = [];
+            for (const selector of selectors) {
+                const listOptions: ListConfigurationSettingsOptions = {
+                    keyFilter: selector.keyFilter,
+                    labelFilter: selector.labelFilter
+                };
+
+                const settings = listConfigurationSettingsWithTrace(
+                    this.#requestTraceOptions,
+                    client,
+                    listOptions
+                );
+
+                for await (const setting of settings) {
+                    if (!isFeatureFlag(setting)) { // exclude feature flags
+                        loadedSettings.push(setting);
+                    }
                 }
             }
-        }
-        return loadedSettings;
+            return loadedSettings;
+        };
+
+        return await this.#executeWithFailoverPolicy(funcToExecute) as ConfigurationSetting[];
     }
 
     /**
@@ -279,29 +309,42 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
     }
 
     async #loadFeatureFlags() {
-        const featureFlagSettings: ConfigurationSetting[] = [];
-        for (const selector of this.#featureFlagSelectors) {
-            const listOptions: ListConfigurationSettingsOptions = {
-                keyFilter: `${featureFlagPrefix}${selector.keyFilter}`,
-                labelFilter: selector.labelFilter
-            };
+        // Temporary map to store feature flags, key is the key of the setting, value is the raw value of the setting
+        const funcToExecute = async (client) => {
+            const featureFlagSettings: ConfigurationSetting[] = [];
+            // deep copy selectors to avoid modification if current client fails
+            const selectors = JSON.parse(
+                JSON.stringify(this.#featureFlagSelectors)
+            );
 
-            const pageEtags: string[] = [];
-            const pageIterator = listConfigurationSettingsWithTrace(
-                this.#requestTraceOptions,
-                this.#client,
-                listOptions
-            ).byPage();
-            for await (const page of pageIterator) {
-                pageEtags.push(page.etag ?? "");
-                for (const setting of page.items) {
-                    if (isFeatureFlag(setting)) {
-                        featureFlagSettings.push(setting);
+            for (const selector of selectors) {
+                const listOptions: ListConfigurationSettingsOptions = {
+                    keyFilter: `${featureFlagPrefix}${selector.keyFilter}`,
+                    labelFilter: selector.labelFilter
+                };
+
+                const pageEtags: string[] = [];
+                const pageIterator = listConfigurationSettingsWithTrace(
+                    this.#requestTraceOptions,
+                    client,
+                    listOptions
+                ).byPage();
+                for await (const page of pageIterator) {
+                    pageEtags.push(page.etag ?? "");
+                    for (const setting of page.items) {
+                        if (isFeatureFlag(setting)) {
+                            featureFlagSettings.push(setting);
+                        }
                     }
                 }
+                selector.pageEtags = pageEtags;
             }
-            selector.pageEtags = pageEtags;
-        }
+
+            this.#featureFlagSelectors = selectors;
+            return featureFlagSettings;
+        };
+
+        const featureFlagSettings = await this.#executeWithFailoverPolicy(funcToExecute) as ConfigurationSetting[];
 
         // parse feature flags
         const featureFlags = await Promise.all(
@@ -389,7 +432,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         // check if any refresh task failed
         for (const result of results) {
             if (result.status === "rejected") {
-                throw result.reason;
+                console.warn("Refresh failed:", result.reason);
             }
         }
 
@@ -430,13 +473,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         }
 
         if (needRefresh) {
-            try {
-                await this.#loadSelectedAndWatchedKeyValues();
-            } catch (error) {
-                // if refresh failed, backoff
-                this.#refreshTimer.backoff();
-                throw error;
-            }
+            await this.#loadSelectedAndWatchedKeyValues();
         }
 
         this.#refreshTimer.reset();
@@ -454,39 +491,32 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         }
 
         // check if any feature flag is changed
-        let needRefresh = false;
-        for (const selector of this.#featureFlagSelectors) {
-            const listOptions: ListConfigurationSettingsOptions = {
-                keyFilter: `${featureFlagPrefix}${selector.keyFilter}`,
-                labelFilter: selector.labelFilter,
-                pageEtags: selector.pageEtags
-            };
-            const pageIterator = listConfigurationSettingsWithTrace(
-                this.#requestTraceOptions,
-                this.#client,
-                listOptions
-            ).byPage();
-
-            for await (const page of pageIterator) {
-                if (page._response.status === 200) { // created or changed
-                    needRefresh = true;
-                    break;
+        const funcToExecute = async (client) => {
+            for (const selector of this.#featureFlagSelectors) {
+                const listOptions: ListConfigurationSettingsOptions = {
+                    keyFilter: `${featureFlagPrefix}${selector.keyFilter}`,
+                    labelFilter: selector.labelFilter,
+                    pageEtags: selector.pageEtags
+                };
+
+                const pageIterator = listConfigurationSettingsWithTrace(
+                    this.#requestTraceOptions,
+                    client,
+                    listOptions
+                ).byPage();
+
+                for await (const page of pageIterator) {
+                    if (page._response.status === 200) { // created or changed
+                        return true;
+                    }
                 }
             }
+            return false;
+        };
 
-            if (needRefresh) {
-                break; // short-circuit if result from any of the selectors is changed
-            }
-        }
-
+        const needRefresh: boolean = await this.#executeWithFailoverPolicy(funcToExecute);
         if (needRefresh) {
-            try {
-                await this.#loadFeatureFlags();
-            } catch (error) {
-                // if refresh failed, backoff
-                this.#featureFlagRefreshTimer.backoff();
-                throw error;
-            }
+            await this.#loadFeatureFlags();
         }
 
         this.#featureFlagRefreshTimer.reset();
@@ -540,14 +570,18 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      * Get a configuration setting by key and label. If the setting is not found, return undefine instead of throwing an error.
      */
     async #getConfigurationSetting(configurationSettingId: ConfigurationSettingId, customOptions?: GetConfigurationSettingOptions): Promise<GetConfigurationSettingResponse | undefined> {
-        let response: GetConfigurationSettingResponse | undefined;
-        try {
-            response = await getConfigurationSettingWithTrace(
+        const funcToExecute = async (client) => {
+            return getConfigurationSettingWithTrace(
                 this.#requestTraceOptions,
-                this.#client,
+                client,
                 configurationSettingId,
                 customOptions
             );
+        };
+
+        let response: GetConfigurationSettingResponse | undefined;
+        try {
+            response = await this.#executeWithFailoverPolicy(funcToExecute);
         } catch (error) {
             if (isRestError(error) && error.statusCode === 404) {
                 response = undefined;
@@ -634,7 +668,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
     }
 
     #createFeatureFlagReference(setting: ConfigurationSetting<string>): string {
-        let featureFlagReference = `${this.#clientEndpoint}kv/${setting.key}`;
+        let featureFlagReference = `${this.#clientManager.endpoint.origin}/kv/${setting.key}`;
         if (setting.label && setting.label.trim().length !== 0) {
             featureFlagReference += `?label=${setting.label}`;
         }
@@ -794,3 +828,9 @@ function getValidFeatureFlagSelectors(selectors?: SettingSelector[]): SettingSel
         return getValidSelectors(selectors);
     }
 }
+
+function isFailoverableError(error: any): boolean {
+    // ENOTFOUND: DNS lookup failed, ENOENT: no such file or directory
+    return isRestError(error) && (error.code === "ENOTFOUND" || error.code === "ENOENT" ||
+        (error.statusCode !== undefined && (error.statusCode === 401 || error.statusCode === 403 || error.statusCode === 408 || error.statusCode === 429 || error.statusCode >= 500)));
+}

src/AzureAppConfigurationImpl.ts

@@ -12,7 +12,7 @@ export const MaxRetryDelayInMs = 60000;
 
 export interface AzureAppConfigurationOptions {
     /**
-     * Specify what key-values to include in the configuration provider.
+     * Specifies what key-values to include in the configuration provider.
      *
      * @remarks
      * If no selectors are specified then all key-values with no label will be included.
@@ -47,4 +47,12 @@ export interface AzureAppConfigurationOptions {
      * Specifies options used to configure feature flags.
      */
     featureFlagOptions?: FeatureFlagOptions;
+
+    /**
+     * Specifies whether to enable replica discovery or not.
+     *
+     * @remarks
+     * If not specified, the default value is true.
+     */
+    replicaDiscoveryEnabled?: boolean;
 }