Update retry in Kusto emulator actions to handle any non-permanent error (#11752)

* Extract resilence pipeline and add tests

* Generalize reslience pipeline to handle any non-permanent exception

* Clean up naming

* Fix up names

* Add comment

* Fix namespace

* Apply suggestions from code review

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: MattKotsenas

src/Aspire.Hosting.Azure.Kusto/AzureKustoBuilderExtensions.cs

@@ -14,7 +14,6 @@
 using Kusto.Data.Net.Client;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
-using Polly;
 
 namespace Aspire.Hosting;
 
@@ -23,15 +22,6 @@ namespace Aspire.Hosting;
 /// </summary>
 public static class AzureKustoBuilderExtensions
 {
-    private static readonly ResiliencePipeline s_pipeline = new ResiliencePipelineBuilder()
-        .AddRetry(new()
-        {
-            MaxRetryAttempts = 3,
-            Delay = TimeSpan.FromSeconds(2),
-            ShouldHandle = new PredicateBuilder().Handle<KustoRequestThrottledException>(),
-        })
-        .Build();
-
     /// <summary>
     /// Adds an Azure Data Explorer (Kusto) cluster resource to the application model.
     /// </summary>
@@ -293,7 +283,7 @@ private static async Task CreateDatabaseAsync(ICslAdminProvider adminProvider, A
 
         try
         {
-            await s_pipeline.ExecuteAsync(async cancellationToken => await adminProvider.ExecuteControlCommandAsync(databaseResource.DatabaseName, script, crp).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
+            await AzureKustoEmulatorResiliencePipelines.Default.ExecuteAsync(async ct => await adminProvider.ExecuteControlCommandAsync(databaseResource.DatabaseName, script, crp).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
             logger.LogDebug("Database '{DatabaseName}' created successfully", databaseResource.DatabaseName);
         }
         catch (KustoBadRequestException e) when (e.Message.Contains("EntityNameAlreadyExistsException"))

src/Aspire.Hosting.Azure.Kusto/AzureKustoEmulatorResiliencePipelines.cs

@@ -0,0 +1,35 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Kusto.Cloud.Platform.Utils;
+using Polly;
+
+namespace Aspire.Hosting.Azure;
+
+/// <summary>
+/// Provides pre-configured resilience pipelines for Azure Kusto emulator operations.
+/// </summary>
+internal static class AzureKustoEmulatorResiliencePipelines
+{
+    /// <summary>
+    /// Gets a resilience pipeline configured to handle non-permanent exceptions.
+    /// </summary>
+    public static ResiliencePipeline Default { get; } = new ResiliencePipelineBuilder()
+        .AddRetry(new()
+        {
+            MaxRetryAttempts = 10,
+            Delay = TimeSpan.FromMilliseconds(100),
+            BackoffType = DelayBackoffType.Exponential,
+            ShouldHandle = new PredicateBuilder().Handle<Exception>(IsTransient),
+        })
+        .Build();
+
+    /// <summary>
+    /// Determines whether the specified exception represents a transient error that may succeed if retried.
+    /// </summary>
+    /// <remarks>
+    /// There's no common base exception type between exceptions in the Kusto.Data and Kusto.Ingest libraries, however
+    /// they do share a common interface, <see cref="ICloudPlatformException"/>, which has the <c>IsPermanent</c> property.
+    /// </remarks>
+    private static bool IsTransient(Exception ex) => ex is ICloudPlatformException cpe && !cpe.IsPermanent;
+}

tests/Aspire.Hosting.Azure.Kusto.Tests/KustoResiliencePipelinesTests.cs

@@ -0,0 +1,66 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Kusto.Data.Exceptions;
+
+namespace Aspire.Hosting.Azure.Kusto.Tests;
+
+public class KustoResiliencePipelinesTests
+{
+    [Fact]
+    public async Task ShouldRetryOnTemporaryExceptions()
+    {
+        // Arrange
+        var attemptCount = 0;
+        ValueTask work(CancellationToken ct)
+        {
+            attemptCount++;
+            throw new KustoRequestThrottledException();
+        }
+
+        // Act + Assert
+        await Assert.ThrowsAsync<KustoRequestThrottledException>(async () =>
+        {
+            await AzureKustoEmulatorResiliencePipelines.Default.ExecuteAsync(work, TestContext.Current.CancellationToken);
+        });
+        Assert.True(attemptCount > 1, "Operation should have been retried");
+    }
+
+    [Fact]
+    public async Task ShouldNotRetryOnOtherExceptions()
+    {
+        // Arrange
+        var attemptCount = 0;
+        ValueTask work(CancellationToken ct)
+        {
+            attemptCount++;
+            throw new InvalidOperationException();
+        }
+
+        // Act + Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+        {
+            await AzureKustoEmulatorResiliencePipelines.Default.ExecuteAsync(work, TestContext.Current.CancellationToken);
+        });
+        Assert.Equal(1, attemptCount);
+    }
+
+    [Fact]
+    public async Task ShouldNotRetryOnPermanentExceptions()
+    {
+        // Arrange
+        var attemptCount = 0;
+        ValueTask work(CancellationToken ct)
+        {
+            attemptCount++;
+            throw new KustoBadRequestException();
+        }
+
+        // Act + Assert
+        await Assert.ThrowsAsync<KustoBadRequestException>(async () =>
+        {
+            await AzureKustoEmulatorResiliencePipelines.Default.ExecuteAsync(work, TestContext.Current.CancellationToken);
+        });
+        Assert.Equal(1, attemptCount);
+    }
+}