From dd37cc5a63943e3ab79e0f565699dca5d304b545 Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sun, 17 Mar 2019 15:42:41 -0400 Subject: [PATCH 01/10] ConnectionMultiplexer: track global counts for deebugging --- src/StackExchange.Redis/ConnectionMultiplexer.cs | 10 ++++++++++ src/StackExchange.Redis/ExceptionFactory.cs | 2 ++ 2 files changed, 12 insertions(+) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index b4de3a49f..877f3c868 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -57,6 +57,11 @@ bool IInternalConnectionMultiplexer.IgnoreConnect /// internal volatile bool IgnoreConnect; + /// + /// Tracks overall connection multiplexer counts + /// + internal static int _connectAttemptCount, _connectCount, _closeCount; + /// /// Provides a way of overriding the default Task Factory. If not set, it will use the default Task.Factory. /// Useful when top level code sets it's own factory which may interfere with Redis queries. @@ -817,12 +822,14 @@ private static async Task ConnectImplAsync(object configu { muxer = CreateMultiplexer(configuration, log, out connectHandler); killMe = muxer; + Interlocked.Increment(ref _connectAttemptCount); bool configured = await muxer.ReconfigureAsync(true, false, log, null, "connect").ObserveErrors().ForAwait(); if (!configured) { throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); } killMe = null; + Interlocked.Increment(ref _connectCount); return muxer; } finally @@ -924,6 +931,7 @@ private static ConnectionMultiplexer ConnectImpl(object configuration, TextWrite muxer = CreateMultiplexer(configuration, log, out connectHandler); killMe = muxer; // note that task has timeouts internally, so it might take *just over* the regular timeout + Interlocked.Increment(ref _connectAttemptCount); var task = muxer.ReconfigureAsync(true, false, log, null, "connect"); if (!task.Wait(muxer.SyncConnectTimeout(true))) @@ -940,6 +948,7 @@ private static ConnectionMultiplexer ConnectImpl(object configuration, TextWrite } if (!task.Result) throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); killMe = null; + Interlocked.Increment(ref _connectCount); return muxer; } finally @@ -2042,6 +2051,7 @@ public void Close(bool allowCommandsToComplete = true) DisposeAndClearServers(); OnCloseReaderWriter(); OnClosing(true); + Interlocked.Increment(ref _closeCount); } partial void OnCloseReaderWriter(); diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index ef7525439..3247be17e 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -263,6 +263,8 @@ void add(string lk, string sk, string v) add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } + add("Multiplexer-Connects", "mc", $"{ConnectionMultiplexer._connectAttemptCount}/{ConnectionMultiplexer._connectCount}/{ConnectionMultiplexer._closeCount}"); + add("Version", "v", GetLibVersion()); sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); From c60bf252eb85d435ff4fec653c2e8dc99ecf018a Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sat, 21 Mar 2020 09:31:07 -0400 Subject: [PATCH 02/10] Build dammit From 2ca5b48a8d97c4de3dcd566c5d4e77d6e124192e Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sun, 22 Mar 2020 08:29:26 -0400 Subject: [PATCH 03/10] Move to per-multiplexer/add tests Note: I know mutiplexer isn't spelled right - will fix that in a follow-up to avoid noise. --- src/StackExchange.Redis/ConnectionMultiplexer.cs | 10 +++++----- src/StackExchange.Redis/ExceptionFactory.cs | 3 +-- .../StackExchange.Redis.Tests/ExceptionFactoryTests.cs | 1 + 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 771376d86..14ce37c52 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -108,7 +108,7 @@ bool IInternalConnectionMultiplexer.IgnoreConnect /// /// Tracks overall connection multiplexer counts /// - internal static int _connectAttemptCount, _connectCount, _closeCount; + internal long _connectAttemptCount, _connectCount, _closeCount; /// /// Provides a way of overriding the default Task Factory. If not set, it will use the default Task.Factory. @@ -839,14 +839,14 @@ private static async Task ConnectImplAsync(object configu { muxer = CreateMultiplexer(configuration, logProxy, out connectHandler); killMe = muxer; - Interlocked.Increment(ref _connectAttemptCount); + Interlocked.Increment(ref muxer._connectAttemptCount); bool configured = await muxer.ReconfigureAsync(true, false, logProxy, null, "connect").ObserveErrors().ForAwait(); if (!configured) { throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); } killMe = null; - Interlocked.Increment(ref _connectCount); + Interlocked.Increment(ref muxer._connectCount); return muxer; } finally @@ -999,7 +999,7 @@ private static ConnectionMultiplexer ConnectImpl(object configuration, TextWrite { muxer = CreateMultiplexer(configuration, logProxy, out connectHandler); killMe = muxer; - Interlocked.Increment(ref _connectAttemptCount); + Interlocked.Increment(ref muxer._connectAttemptCount); // note that task has timeouts internally, so it might take *just over* the regular timeout var task = muxer.ReconfigureAsync(true, false, logProxy, null, "connect"); @@ -1018,7 +1018,7 @@ private static ConnectionMultiplexer ConnectImpl(object configuration, TextWrite if (!task.Result) throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); killMe = null; - Interlocked.Increment(ref _connectCount); + Interlocked.Increment(ref muxer._connectCount); if (muxer.ServerSelectionStrategy.ServerType == ServerType.Sentinel) { diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index 4092db3fe..b69489bfc 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -273,6 +273,7 @@ void add(string lk, string sk, string v) } add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/","")); } + add("Multiplexer-Connects", "mc", $"{mutiplexer._connectAttemptCount}/{mutiplexer._connectCount}/{mutiplexer._closeCount}"); add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); add("Client-Name", "clientName", mutiplexer.ClientName); @@ -292,8 +293,6 @@ void add(string lk, string sk, string v) add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } - add("Multiplexer-Connects", "mc", $"{ConnectionMultiplexer._connectAttemptCount}/{ConnectionMultiplexer._connectCount}/{ConnectionMultiplexer._closeCount}"); - add("Version", "v", GetLibVersion()); sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); diff --git a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs index 533ce5f02..f8ed64278 100644 --- a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs +++ b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs @@ -129,6 +129,7 @@ public void TimeoutException() Assert.Contains("clientName: " + nameof(TimeoutException), ex.Message); // Ensure our pipe numbers are in place Assert.Contains("inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0", ex.Message); + Assert.Contains("mc: 1/1/0", ex.Message); Assert.Contains("serverEndpoint: " + server.EndPoint.ToString(), ex.Message); Assert.DoesNotContain("Unspecified/", ex.Message); Assert.EndsWith(" (Please take a look at this article for some common client-side issues that can cause timeouts: https://stackexchange.github.io/StackExchange.Redis/Timeouts)", ex.Message); From 31cc91ee24d51f32c04524c2dfdcd4ee215be215 Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sun, 22 Mar 2020 19:24:04 -0400 Subject: [PATCH 04/10] Fix test key names Broken since the 2.1 bump, oops --- tests/StackExchange.Redis.Tests/TestBase.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/StackExchange.Redis.Tests/TestBase.cs b/tests/StackExchange.Redis.Tests/TestBase.cs index 35f044678..b8050fda7 100644 --- a/tests/StackExchange.Redis.Tests/TestBase.cs +++ b/tests/StackExchange.Redis.Tests/TestBase.cs @@ -347,12 +347,13 @@ public static ConnectionMultiplexer CreateDefault( public static string Me([CallerFilePath] string filePath = null, [CallerMemberName] string caller = null) => #if NET462 - "net462-" + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; -#elif NETCOREAPP2_0 - "netcoreapp2.0-" + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; + "net462-" +#elif NETCOREAPP2_1 + "netcoreapp2.1-" #else - "unknown-" + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; + "unknown-" #endif + + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; protected static TimeSpan RunConcurrent(Action work, int threads, int timeout = 10000, [CallerMemberName] string caller = null) { From 5ed4191b326597178367866022e32f100140e4a4 Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sun, 22 Mar 2020 20:48:56 -0400 Subject: [PATCH 05/10] Simplify the NoConnectionAvailable static Simpifies usage for all callers. Also shares code and adds diagnostics to the "no connection" case. --- .../ConnectionMultiplexer.cs | 12 +- src/StackExchange.Redis/ExceptionFactory.cs | 232 +++++++++++------- src/StackExchange.Redis/RedisBatch.cs | 4 +- src/StackExchange.Redis/RedisServer.cs | 4 +- tests/StackExchange.Redis.Tests/AsyncTests.cs | 2 +- .../ExceptionFactoryTests.cs | 23 +- 6 files changed, 159 insertions(+), 118 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 14ce37c52..338c3f88e 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -108,7 +108,7 @@ bool IInternalConnectionMultiplexer.IgnoreConnect /// /// Tracks overall connection multiplexer counts /// - internal long _connectAttemptCount, _connectCount, _closeCount; + internal int _connectAttemptCount = 0, _connectCompletedCount = 0, _connectionCloseCount = 0; /// /// Provides a way of overriding the default Task Factory. If not set, it will use the default Task.Factory. @@ -389,7 +389,7 @@ internal void MakeMaster(ServerEndPoint server, ReplicationChangeOptions options if (server == null) throw new ArgumentNullException(nameof(server)); var srv = new RedisServer(this, server, null); - if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, IncludePerformanceCountersInExceptions, RedisCommand.SLAVEOF, null, server, GetServerSnapshot()); + if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(this, null, server, GetServerSnapshot(), command: RedisCommand.SLAVEOF); CommandMap.AssertAvailable(RedisCommand.SLAVEOF); #pragma warning disable CS0618 @@ -846,7 +846,7 @@ private static async Task ConnectImplAsync(object configu throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); } killMe = null; - Interlocked.Increment(ref muxer._connectCount); + Interlocked.Increment(ref muxer._connectCompletedCount); return muxer; } finally @@ -1018,7 +1018,7 @@ private static ConnectionMultiplexer ConnectImpl(object configuration, TextWrite if (!task.Result) throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); killMe = null; - Interlocked.Increment(ref muxer._connectCount); + Interlocked.Increment(ref muxer._connectCompletedCount); if (muxer.ServerSelectionStrategy.ServerType == ServerType.Sentinel) { @@ -2461,7 +2461,7 @@ public void Close(bool allowCommandsToComplete = true) DisposeAndClearServers(); OnCloseReaderWriter(); OnClosing(true); - Interlocked.Increment(ref _closeCount); + Interlocked.Increment(ref _connectionCloseCount); } partial void OnCloseReaderWriter(); @@ -2578,7 +2578,7 @@ internal Exception GetException(WriteResult result, Message message, ServerEndPo { case WriteResult.Success: return null; case WriteResult.NoConnectionAvailable: - return ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, IncludePerformanceCountersInExceptions, message.Command, message, server, GetServerSnapshot()); + return ExceptionFactory.NoConnectionAvailable(this, message, server); case WriteResult.TimeoutBeforeWrite: return ExceptionFactory.Timeout(this, "The timeout was reached before the message could be written to the output buffer, and it was not sent", message, server, result); case WriteResult.WriteFailure: diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index b69489bfc..45a4ee138 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -18,7 +18,7 @@ internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand c { string s = GetLabel(includeDetail, command, message); var ex = new RedisCommandException("This operation is not available unless admin mode is enabled: " + s); - if (includeDetail) AddDetail(ex, message, server, s); + if (includeDetail) AddExceptionDetail(ex, message, server, s); return ex; } @@ -33,7 +33,7 @@ internal static Exception TooManyArgs(string command, int argCount) internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint server) { var ex = new RedisConnectionException(failureType, message); - if (includeDetail) AddDetail(ex, null, server, null); + if (includeDetail) AddExceptionDetail(ex, null, server, null); return ex; } @@ -41,14 +41,14 @@ internal static Exception DatabaseNotRequired(bool includeDetail, RedisCommand c { string s = command.ToString(); var ex = new RedisCommandException("A target database is not required for " + s); - if (includeDetail) AddDetail(ex, null, null, s); + if (includeDetail) AddExceptionDetail(ex, null, null, s); return ex; } internal static Exception DatabaseOutfRange(bool includeDetail, int targetDatabase, Message message, ServerEndPoint server) { var ex = new RedisCommandException("The database does not exist on the server: " + targetDatabase); - if (includeDetail) AddDetail(ex, message, server, null); + if (includeDetail) AddExceptionDetail(ex, message, server, null); return ex; } @@ -56,7 +56,7 @@ internal static Exception DatabaseRequired(bool includeDetail, RedisCommand comm { string s = command.ToString(); var ex = new RedisCommandException("A target database is required for " + s); - if (includeDetail) AddDetail(ex, null, null, s); + if (includeDetail) AddExceptionDetail(ex, null, null, s); return ex; } @@ -64,14 +64,14 @@ internal static Exception MasterOnly(bool includeDetail, RedisCommand command, M { string s = GetLabel(includeDetail, command, message); var ex = new RedisCommandException("Command cannot be issued to a slave: " + s); - if (includeDetail) AddDetail(ex, message, server, s); + if (includeDetail) AddExceptionDetail(ex, message, server, s); return ex; } internal static Exception MultiSlot(bool includeDetail, Message message) { var ex = new RedisCommandException("Multi-key operations must involve a single slot; keys can use 'hash tags' to help this, i.e. '{/users/12345}/account' and '{/users/12345}/contacts' will always be in the same slot"); - if (includeDetail) AddDetail(ex, message, null, null); + if (includeDetail) AddExceptionDetail(ex, message, null, null); return ex; } @@ -91,9 +91,14 @@ internal static string GetInnerMostExceptionMessage(Exception e) } } - internal static Exception NoConnectionAvailable(bool includeDetail, bool includePerformanceCounters, RedisCommand command, Message message, ServerEndPoint server, ReadOnlySpan serverSnapshot) + internal static Exception NoConnectionAvailable( + ConnectionMultiplexer multiplexer, + Message message, + ServerEndPoint server, + ReadOnlySpan serverSnapshot = default, + RedisCommand command = default) { - string commandLabel = GetLabel(includeDetail, command, message); + string commandLabel = GetLabel(multiplexer.IncludeDetailInExceptions, message?.Command ?? command, message); if (server != null) { @@ -102,25 +107,51 @@ internal static Exception NoConnectionAvailable(bool includeDetail, bool include serverSnapshot = new ServerEndPoint[] { server }; } - var innerException = PopulateInnerExceptions(serverSnapshot); + var innerException = PopulateInnerExceptions(serverSnapshot == default ? multiplexer.GetServerSnapshot() : serverSnapshot); - StringBuilder exceptionmessage = new StringBuilder("No connection is available to service this operation: ").Append(commandLabel); + // Try to get a useful error message for the user. + long attempts = multiplexer._connectAttemptCount, completions = multiplexer._connectCompletedCount; + string initialMessage; + // We only need to customize the connection if we're aborting on connect fail + // The "never" case would have thrown, if this was true + if (!multiplexer.RawConfig.AbortOnConnectFail && attempts == 1 && completions == 0) + { + // Initial attempt, attempted use before an async connection completes + initialMessage = $"Connection to Redis never succeeded (1 attempt - connection likely in-progress), unable to service operation: "; + } + else if (!multiplexer.RawConfig.AbortOnConnectFail && attempts >= multiplexer.RawConfig.ConnectRetry && completions == 0) + { + // Attempted use after a full initial retry connect count # of failures + // This can happen in Azure often, where user disables abort and has the wrong config + initialMessage = $"Connection to Redis never succeeded ({attempts} attempts - check your config), unable to service operation: "; + } + else + { + // Default if we don't have a more useful error message here based on circumstances + initialMessage = "No connection is active/available to service this operation: "; + } + + StringBuilder sb = new StringBuilder(initialMessage); + sb.Append(commandLabel); string innermostExceptionstring = GetInnerMostExceptionMessage(innerException); if (!string.IsNullOrEmpty(innermostExceptionstring)) { - exceptionmessage.Append("; ").Append(innermostExceptionstring); + sb.Append("; ").Append(innermostExceptionstring); } - if (includeDetail) + // Add counters and exception data if we have it + List> data = null; + if (multiplexer.IncludeDetailInExceptions) { - exceptionmessage.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(includePerformanceCounters)); + data = new List>(); + AddCommonDetail(data, sb, message, multiplexer, server); } - - var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); - - if (includeDetail) + var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); + if (multiplexer.IncludeDetailInExceptions) { - AddDetail(ex, message, server, commandLabel); + CopyDataToException(data, ex); + sb.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(multiplexer.IncludePerformanceCountersInExceptions)); + AddExceptionDetail(ex, message, server, commandLabel); } return ex; } @@ -160,7 +191,7 @@ internal static Exception NotSupported(bool includeDetail, RedisCommand command) { string s = GetLabel(includeDetail, command, null); var ex = new RedisCommandException("Command is not available on your server: " + s); - if (includeDetail) AddDetail(ex, null, null, s); + if (includeDetail) AddExceptionDetail(ex, null, null, s); return ex; } @@ -181,7 +212,16 @@ internal static string GetLibVersion() } return _libVersion; } - internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null) + private static void Add(List> data, StringBuilder sb, string lk, string sk, string v) + { + if (v != null) + { + if (lk != null) data.Add(Tuple.Create(lk, v)); + if (sk != null) sb.Append(", ").Append(sk).Append(": ").Append(v); + } + } + + internal static Exception Timeout(ConnectionMultiplexer multiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null) { List> data = new List> { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); @@ -195,128 +235,138 @@ internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseE } else { - sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)"); - } - - void add(string lk, string sk, string v) - { - if (v != null) - { - if (lk != null) data.Add(Tuple.Create(lk, v)); - if (sk != null) sb.Append(", ").Append(sk).Append(": ").Append(v); - } + sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(multiplexer.TimeoutMilliseconds)).Append("ms)"); } // Add timeout data, if we have it if (result == WriteResult.TimeoutBeforeWrite) { - add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds)); + Add(data, sb, "Timeout", "timeout", Format.ToString(multiplexer.TimeoutMilliseconds)); try { #if DEBUG - if (message.QueuePosition >= 0) add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue - if ((int)message.ConnectionWriteState >= 0) add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue + if (message.QueuePosition >= 0) Add(data, sb, "QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue + if ((int)message.ConnectionWriteState >= 0) Add(data, sb, "WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue #endif if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta)) { - add("Write-State", null, ws.ToString()); - add("Read-State", null, rs.ToString()); + Add(data, sb, "Write-State", null, ws.ToString()); + Add(data, sb, "Read-State", null, rs.ToString()); // these might not always be available if (sentDelta >= 0) { - add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); + Add(data, sb, "OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); } if (receivedDelta >= 0) { - add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); + Add(data, sb, "InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); } } } catch { } } + AddCommonDetail(data, sb, message, multiplexer, server); + + sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); + sb.Append(timeoutHelpLink); + sb.Append(")"); + + var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) + { + HelpLink = timeoutHelpLink + }; + CopyDataToException(data, ex); + + if (multiplexer.IncludeDetailInExceptions) AddExceptionDetail(ex, message, server, null); + return ex; + } + + private static void CopyDataToException(List> data, Exception ex) + { + if (data != null) + { + var exData = ex.Data; + foreach (var kv in data) + { + exData["Redis-" + kv.Item1] = kv.Item2; + } + } + } + + private static void AddCommonDetail( + List> data, + StringBuilder sb, + Message message, + ConnectionMultiplexer multiplexer, + ServerEndPoint server + ) + { if (message != null) { message.TryGetHeadMessages(out var now, out var next); - if (now != null) add("Message-Current", "active", mutiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString()); - if (next != null) add("Message-Next", "next", mutiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString()); + if (now != null) Add(data, sb, "Message-Current", "active", multiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString()); + if (next != null) Add(data, sb, "Message-Next", "next", multiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString()); } // Add server data, if we have it - if (server != null) + if (server != null && message != null) { server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws); - switch(rs) + switch (rs) { case PhysicalConnection.ReadStatus.CompletePendingMessageAsync: case PhysicalConnection.ReadStatus.CompletePendingMessageSync: sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** "); break; } - add("OpsSinceLastHeartbeat", "inst", inst.ToString()); - add("Queue-Awaiting-Write", "qu", qu.ToString()); - add("Queue-Awaiting-Response", "qs", qs.ToString()); - add("Active-Writer", "aw", aw.ToString()); - if (qu != 0) add("Backlog-Writer", "bw", bs.ToString()); - if (rs != PhysicalConnection.ReadStatus.NA) add("Read-State", "rs", rs.ToString()); - if (ws != PhysicalConnection.WriteStatus.NA) add("Write-State", "ws", ws.ToString()); - - if (@in >= 0) add("Inbound-Bytes", "in", @in.ToString()); - if (toRead >= 0) add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString()); - if (toWrite >= 0) add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString()); - - if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0) + Add(data, sb, "OpsSinceLastHeartbeat", "inst", inst.ToString()); + Add(data, sb, "Queue-Awaiting-Write", "qu", qu.ToString()); + Add(data, sb, "Queue-Awaiting-Response", "qs", qs.ToString()); + Add(data, sb, "Active-Writer", "aw", aw.ToString()); + if (qu != 0) Add(data, sb, "Backlog-Writer", "bw", bs.ToString()); + if (rs != PhysicalConnection.ReadStatus.NA) Add(data, sb, "Read-State", "rs", rs.ToString()); + if (ws != PhysicalConnection.WriteStatus.NA) Add(data, sb, "Write-State", "ws", ws.ToString()); + + if (@in >= 0) Add(data, sb, "Inbound-Bytes", "in", @in.ToString()); + if (toRead >= 0) Add(data, sb, "Inbound-Pipe-Bytes", "in-pipe", toRead.ToString()); + if (toWrite >= 0) Add(data, sb, "Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString()); + + if (multiplexer.StormLogThreshold >= 0 && qs >= multiplexer.StormLogThreshold && Interlocked.CompareExchange(ref multiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message.Command); - if (string.IsNullOrWhiteSpace(log)) Interlocked.Exchange(ref mutiplexer.haveStormLog, 0); - else Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log); + if (string.IsNullOrWhiteSpace(log)) Interlocked.Exchange(ref multiplexer.haveStormLog, 0); + else Interlocked.Exchange(ref multiplexer.stormLogSnapshot, log); } - add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/","")); + Add(data, sb, "Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", "")); } - add("Multiplexer-Connects", "mc", $"{mutiplexer._connectAttemptCount}/{mutiplexer._connectCount}/{mutiplexer._closeCount}"); - add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); + Add(data, sb, "Multiplexer-Connects", "mc", $"{multiplexer._connectAttemptCount}/{multiplexer._connectCompletedCount}/{multiplexer._connectionCloseCount}"); + Add(data, sb, "Manager", "mgr", multiplexer.SocketManager?.GetState()); - add("Client-Name", "clientName", mutiplexer.ClientName); - var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy); - // only add keyslot if its a valid cluster key slot - if (hashSlot != ServerSelectionStrategy.NoSlot) + Add(data, sb, "Client-Name", "clientName", multiplexer.ClientName); + if (message != null) { - add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString()); + var hashSlot = message.GetHashSlot(multiplexer.ServerSelectionStrategy); + // only add keyslot if its a valid cluster key slot + if (hashSlot != ServerSelectionStrategy.NoSlot) + { + Add(data, sb, "Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(multiplexer.ServerSelectionStrategy).ToString()); + } } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker); - add("ThreadPool-IO-Completion", "IOCP", iocp); - add("ThreadPool-Workers", "WORKER", worker); + Add(data, sb, "ThreadPool-IO-Completion", "IOCP", iocp); + Add(data, sb, "ThreadPool-Workers", "WORKER", worker); data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); - if (mutiplexer.IncludePerformanceCountersInExceptions) - { - add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); - } - - add("Version", "v", GetLibVersion()); - - sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); - sb.Append(timeoutHelpLink); - sb.Append(")"); - - var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) + if (multiplexer.IncludePerformanceCountersInExceptions) { - HelpLink = timeoutHelpLink - }; - if (data != null) - { - var exData = ex.Data; - foreach (var kv in data) - { - exData["Redis-" + kv.Item1] = kv.Item2; - } + Add(data, sb, "Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } - if (mutiplexer.IncludeDetailInExceptions) AddDetail(ex, message, server, null); - return ex; + Add(data, sb, "Version", "v", GetLibVersion()); } - private static void AddDetail(Exception exception, Message message, ServerEndPoint server, string label) + private static void AddExceptionDetail(Exception exception, Message message, ServerEndPoint server, string label) { if (exception != null) { diff --git a/src/StackExchange.Redis/RedisBatch.cs b/src/StackExchange.Redis/RedisBatch.cs index 464a1bc49..33eeac76b 100644 --- a/src/StackExchange.Redis/RedisBatch.cs +++ b/src/StackExchange.Redis/RedisBatch.cs @@ -28,13 +28,13 @@ public void Execute() if (server == null) { FailNoServer(snapshot); - throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server,multiplexer.GetServerSnapshot()); + throw ExceptionFactory.NoConnectionAvailable(multiplexer, message, server); } var bridge = server.GetBridge(message.Command); if (bridge == null) { FailNoServer(snapshot); - throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot()); + throw ExceptionFactory.NoConnectionAvailable(multiplexer, message, server); } // identity a list diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index 87820c1dc..1ab799af3 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -577,7 +577,7 @@ internal override Task ExecuteAsync(Message message, ResultProcessor pr // no need to deny exec-sync here; will be complete before they see if var tcs = TaskSource.Create(asyncState); - ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot())); + ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer, message, server)); return tcs.Task; } return base.ExecuteAsync(message, processor, server); @@ -590,7 +590,7 @@ internal override T ExecuteSync(Message message, ResultProcessor processor if (!server.IsConnected) { if (message == null || message.IsFireAndForget) return default(T); - throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot()); + throw ExceptionFactory.NoConnectionAvailable(multiplexer, message, server); } return base.ExecuteSync(message, processor, server); } diff --git a/tests/StackExchange.Redis.Tests/AsyncTests.cs b/tests/StackExchange.Redis.Tests/AsyncTests.cs index b2398e973..006c6cb20 100644 --- a/tests/StackExchange.Redis.Tests/AsyncTests.cs +++ b/tests/StackExchange.Redis.Tests/AsyncTests.cs @@ -39,7 +39,7 @@ public void AsyncTasksReportFailureIfServerUnavailable() Assert.True(c.IsFaulted, "faulted"); var ex = c.Exception.InnerExceptions.Single(); Assert.IsType(ex); - Assert.StartsWith("No connection is available to service this operation: SADD " + key.ToString(), ex.Message); + Assert.StartsWith("No connection is active/available to service this operation: SADD " + key.ToString(), ex.Message); } } diff --git a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs index f8ed64278..108b037f4 100644 --- a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs +++ b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using Xunit; using Xunit.Abstractions; @@ -16,7 +15,7 @@ public void NullLastException() { var conn = muxer.GetDatabase(); Assert.Null(muxer.GetServerSnapshot()[0].LastException); - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, null); Assert.Null(ex.InnerException); } } @@ -28,20 +27,13 @@ public void CanGetVersion() Assert.Matches(@"2\.[0-9]+\.[0-9]+(\.[0-9]+)?", libVer); } - [Fact] - public void NullSnapshot() - { - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, null); - Assert.Null(ex.InnerException); - } - #if DEBUG [Fact] public void MultipleEndpointsThrowConnectionException() { try { - using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true)) + using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true, shared: false)) { var conn = muxer.GetDatabase(); muxer.AllowConnect = false; @@ -51,7 +43,7 @@ public void MultipleEndpointsThrowConnectionException() muxer.GetServer(endpoint).SimulateConnectionFailure(); } - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, null); var outer = Assert.IsType(ex); Assert.Equal(ConnectionFailureType.UnableToResolvePhysicalConnection, outer.FailureType); var inner = Assert.IsType(outer.InnerException); @@ -63,20 +55,21 @@ public void MultipleEndpointsThrowConnectionException() ClearAmbientFailures(); } } +#endif [Fact] public void ServerTakesPrecendenceOverSnapshot() { try { - using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true)) + using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true, shared: false)) { var conn = muxer.GetDatabase(); muxer.AllowConnect = false; muxer.GetServer(muxer.GetEndPoints()[0]).SimulateConnectionFailure(); - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, muxer.GetServerSnapshot()[0], muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, muxer.GetServerSnapshot()[0]); Assert.IsType(ex); Assert.IsType(ex.InnerException); Assert.Equal(ex.InnerException, muxer.GetServerSnapshot()[0].LastException); @@ -87,7 +80,6 @@ public void ServerTakesPrecendenceOverSnapshot() ClearAmbientFailures(); } } -#endif [Fact] public void NullInnerExceptionForMultipleEndpointsWithNoLastException() @@ -98,7 +90,7 @@ public void NullInnerExceptionForMultipleEndpointsWithNoLastException() { var conn = muxer.GetDatabase(); muxer.AllowConnect = false; - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, null); Assert.IsType(ex); Assert.Null(ex.InnerException); } @@ -116,7 +108,6 @@ public void TimeoutException() { using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true, shared: false) as ConnectionMultiplexer) { - var conn = muxer.GetDatabase(); var server = GetServer(muxer); muxer.AllowConnect = false; var msg = Message.Create(-1, CommandFlags.None, RedisCommand.PING); From 7ad8299720abb97aab1bed0f92256255686cf97f Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sun, 22 Mar 2020 20:49:08 -0400 Subject: [PATCH 06/10] Add tests for NoConnectionException --- .../ExceptionFactoryTests.cs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs index 108b037f4..7728c1d6c 100644 --- a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs +++ b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs @@ -132,5 +132,66 @@ public void TimeoutException() ClearAmbientFailures(); } } + + [Theory] + [InlineData(0, 0, true, "No connection is active/available to service this operation: PING")] + [InlineData(1, 0, true, "Connection to Redis never succeeded (1 attempt - connection likely in-progress), unable to service operation: PING")] + [InlineData(12, 0, true, "Connection to Redis never succeeded (12 attempts - check your config), unable to service operation: PING")] + [InlineData(0, 0, false, "No connection is active/available to service this operation: PING")] + [InlineData(1, 0, false, "Connection to Redis never succeeded (1 attempt - connection likely in-progress), unable to service operation: PING")] + [InlineData(12, 0, false, "Connection to Redis never succeeded (12 attempts - check your config), unable to service operation: PING")] + public void NoConnectionException(int connCount, int completeCount, bool hasDetail, string messageStart) + { + try + { + var options = new ConfigurationOptions() + { + AbortOnConnectFail = false, + ConnectTimeout = 500, + SyncTimeout = 500, + KeepAlive = 5000 + }; + options.EndPoints.Add($"doesnot.exist.{Guid.NewGuid():N}:6379"); + + var muxer = ConnectionMultiplexer.Connect(options); + using (muxer) + { + var server = muxer.GetServer(muxer.GetEndPoints()[0]); + muxer.AllowConnect = false; + muxer._connectAttemptCount = connCount; + muxer._connectCompletedCount = completeCount; + muxer.IncludeDetailInExceptions = hasDetail; + muxer.IncludePerformanceCountersInExceptions = hasDetail; + + var msg = Message.Create(-1, CommandFlags.None, RedisCommand.PING); + var rawEx = ExceptionFactory.NoConnectionAvailable(muxer, msg, new ServerEndPoint(muxer, server.EndPoint)); + var ex = Assert.IsType(rawEx); + Writer.WriteLine("Exception: " + ex.Message); + + // Example format: "Exception: No connection is active/available to service this operation: PING, inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0, serverEndpoint: 127.0.0.1:6379, mc: 1/1/0, mgr: 10 of 10 available, clientName: NoConnectionException, IOCP: (Busy=0,Free=1000,Min=8,Max=1000), WORKER: (Busy=2,Free=2045,Min=8,Max=2047), Local-CPU: 100%, v: 2.1.0.5"; + Assert.StartsWith(messageStart, ex.Message); + + // Ensure our pipe numbers are in place if they should be + if (hasDetail) + { + Assert.Contains("inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0", ex.Message); + Assert.Contains($"mc: {connCount}/{completeCount}/0", ex.Message); + Assert.Contains("serverEndpoint: " + server.EndPoint.ToString().Replace("Unspecified/", ""), ex.Message); + } + else + { + Assert.DoesNotContain("inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0", ex.Message); + Assert.DoesNotContain($"mc: {connCount}/{completeCount}/0", ex.Message); + Assert.DoesNotContain("serverEndpoint: " + server.EndPoint.ToString().Replace("Unspecified/", ""), ex.Message); + } + Assert.DoesNotContain("Unspecified/", ex.Message); + Assert.Null(ex.InnerException); + } + } + finally + { + ClearAmbientFailures(); + } + } } } From 904f0658e6a6ac90160909ed430f8fd9703b030e Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sun, 22 Mar 2020 20:49:32 -0400 Subject: [PATCH 07/10] Failover: fix tests and debug some SubscriptionsSurviveMasterSwitchAsync is a thorn in our side - moving to DEBUG. --- tests/StackExchange.Redis.Tests/Failover.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/StackExchange.Redis.Tests/Failover.cs b/tests/StackExchange.Redis.Tests/Failover.cs index c4530cab9..d1335b19c 100644 --- a/tests/StackExchange.Redis.Tests/Failover.cs +++ b/tests/StackExchange.Redis.Tests/Failover.cs @@ -141,7 +141,7 @@ public async Task DeslaveGoesToPrimary() Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave)); var ex = Assert.Throws(() => db.IdentifyEndpoint(key, CommandFlags.DemandSlave)); - Assert.StartsWith("No connection is available to service this operation: EXISTS " + Me(), ex.Message); + Assert.StartsWith("No connection is active/available to service this operation: EXISTS " + Me(), ex.Message); Writer.WriteLine("Invoking MakeMaster()..."); primary.MakeMaster(ReplicationChangeOptions.Broadcast | ReplicationChangeOptions.EnslaveSubordinates | ReplicationChangeOptions.SetTiebreaker, Writer); Writer.WriteLine("Finished MakeMaster() call."); @@ -191,6 +191,7 @@ public async Task DeslaveGoesToPrimary() } } +#if DEBUG [Fact] public async Task SubscriptionsSurviveMasterSwitchAsync() { @@ -329,5 +330,6 @@ public async Task SubscriptionsSurviveMasterSwitchAsync() } } } +#endif } } From 4e326fc93065d66a4806f53fca7060ceebe818fb Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Sun, 22 Mar 2020 22:07:45 -0400 Subject: [PATCH 08/10] Remove bad check Inner is irrelevant here - can be not-null depending on the connection race. --- tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs index 7728c1d6c..117f5f57a 100644 --- a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs +++ b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs @@ -185,7 +185,6 @@ public void NoConnectionException(int connCount, int completeCount, bool hasDeta Assert.DoesNotContain("serverEndpoint: " + server.EndPoint.ToString().Replace("Unspecified/", ""), ex.Message); } Assert.DoesNotContain("Unspecified/", ex.Message); - Assert.Null(ex.InnerException); } } finally From c167ca0e6f4ee3ccd9ed9517a11eba5e2b26d9f8 Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Mon, 23 Mar 2020 07:46:17 -0400 Subject: [PATCH 09/10] Update message and add more tests! --- src/StackExchange.Redis/ExceptionFactory.cs | 8 ++--- .../ExceptionFactoryTests.cs | 33 +++++++++++++------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index 45a4ee138..7faf8ee22 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -114,16 +114,16 @@ internal static Exception NoConnectionAvailable( string initialMessage; // We only need to customize the connection if we're aborting on connect fail // The "never" case would have thrown, if this was true - if (!multiplexer.RawConfig.AbortOnConnectFail && attempts == 1 && completions == 0) + if (!multiplexer.RawConfig.AbortOnConnectFail && attempts <= multiplexer.RawConfig.ConnectRetry && completions == 0) { // Initial attempt, attempted use before an async connection completes - initialMessage = $"Connection to Redis never succeeded (1 attempt - connection likely in-progress), unable to service operation: "; + initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - connection likely in-progress), unable to service operation: "; } - else if (!multiplexer.RawConfig.AbortOnConnectFail && attempts >= multiplexer.RawConfig.ConnectRetry && completions == 0) + else if (!multiplexer.RawConfig.AbortOnConnectFail && attempts > multiplexer.RawConfig.ConnectRetry && completions == 0) { // Attempted use after a full initial retry connect count # of failures // This can happen in Azure often, where user disables abort and has the wrong config - initialMessage = $"Connection to Redis never succeeded ({attempts} attempts - check your config), unable to service operation: "; + initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - check your config), unable to service operation: "; } else { diff --git a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs index 117f5f57a..a009a671a 100644 --- a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs +++ b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs @@ -134,26 +134,39 @@ public void TimeoutException() } [Theory] - [InlineData(0, 0, true, "No connection is active/available to service this operation: PING")] - [InlineData(1, 0, true, "Connection to Redis never succeeded (1 attempt - connection likely in-progress), unable to service operation: PING")] - [InlineData(12, 0, true, "Connection to Redis never succeeded (12 attempts - check your config), unable to service operation: PING")] - [InlineData(0, 0, false, "No connection is active/available to service this operation: PING")] - [InlineData(1, 0, false, "Connection to Redis never succeeded (1 attempt - connection likely in-progress), unable to service operation: PING")] - [InlineData(12, 0, false, "Connection to Redis never succeeded (12 attempts - check your config), unable to service operation: PING")] - public void NoConnectionException(int connCount, int completeCount, bool hasDetail, string messageStart) + [InlineData(false, 0, 0, true, "Connection to Redis never succeeded (attempts: 0 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 1, 0, true, "Connection to Redis never succeeded (attempts: 1 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 12, 0, true, "Connection to Redis never succeeded (attempts: 12 - check your config), unable to service operation: PING")] + [InlineData(false, 0, 0, false, "Connection to Redis never succeeded (attempts: 0 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 1, 0, false, "Connection to Redis never succeeded (attempts: 1 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 12, 0, false, "Connection to Redis never succeeded (attempts: 12 - check your config), unable to service operation: PING")] + [InlineData(true, 0, 0, true, "No connection is active/available to service this operation: PING")] + [InlineData(true, 1, 0, true, "No connection is active/available to service this operation: PING")] + [InlineData(true, 12, 0, true, "No connection is active/available to service this operation: PING")] + public void NoConnectionException(bool abortOnConnect, int connCount, int completeCount, bool hasDetail, string messageStart) { try { var options = new ConfigurationOptions() { - AbortOnConnectFail = false, + AbortOnConnectFail = abortOnConnect, ConnectTimeout = 500, SyncTimeout = 500, KeepAlive = 5000 }; - options.EndPoints.Add($"doesnot.exist.{Guid.NewGuid():N}:6379"); - var muxer = ConnectionMultiplexer.Connect(options); + ConnectionMultiplexer muxer; + if (abortOnConnect) + { + options.EndPoints.Add(TestConfig.Current.MasterServerAndPort); + muxer = ConnectionMultiplexer.Connect(options); + } + else + { + options.EndPoints.Add($"doesnot.exist.{Guid.NewGuid():N}:6379"); + muxer = ConnectionMultiplexer.Connect(options); + } + using (muxer) { var server = muxer.GetServer(muxer.GetEndPoints()[0]); From a36201ad5e4ea80ab5c901a59b24554faae30add Mon Sep 17 00:00:00 2001 From: Nick Craver Date: Mon, 23 Mar 2020 07:58:57 -0400 Subject: [PATCH 10/10] Bump pipelines to 2.1.6 --- src/StackExchange.Redis/StackExchange.Redis.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index d7331c1b0..1fbf7a9fc 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -13,7 +13,7 @@ - +