diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index c5aa16b57..9f6fceb68 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -105,6 +105,11 @@ bool IInternalConnectionMultiplexer.IgnoreConnect /// internal volatile bool IgnoreConnect; + /// + /// Tracks overall connection multiplexer counts + /// + internal int _connectAttemptCount = 0, _connectCompletedCount = 0, _connectionCloseCount = 0; + /// /// Provides a way of overriding the default Task Factory. If not set, it will use the default Task.Factory. /// Useful when top level code sets it's own factory which may interfere with Redis queries. @@ -384,7 +389,7 @@ internal void MakeMaster(ServerEndPoint server, ReplicationChangeOptions options if (server == null) throw new ArgumentNullException(nameof(server)); var srv = new RedisServer(this, server, null); - if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, IncludePerformanceCountersInExceptions, RedisCommand.SLAVEOF, null, server, GetServerSnapshot()); + if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(this, null, server, GetServerSnapshot(), command: RedisCommand.SLAVEOF); CommandMap.AssertAvailable(RedisCommand.SLAVEOF); #pragma warning disable CS0618 @@ -848,12 +853,14 @@ private static async Task ConnectImplAsync(object configu { muxer = CreateMultiplexer(configuration, logProxy, out connectHandler); killMe = muxer; + Interlocked.Increment(ref muxer._connectAttemptCount); bool configured = await muxer.ReconfigureAsync(true, false, logProxy, null, "connect").ObserveErrors().ForAwait(); if (!configured) { throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); } killMe = null; + Interlocked.Increment(ref muxer._connectCompletedCount); return muxer; } finally @@ -1006,6 +1013,7 @@ private static ConnectionMultiplexer ConnectImpl(object configuration, TextWrite { muxer = CreateMultiplexer(configuration, logProxy, out connectHandler); killMe = muxer; + Interlocked.Increment(ref muxer._connectAttemptCount); // note that task has timeouts internally, so it might take *just over* the regular timeout var task = muxer.ReconfigureAsync(true, false, logProxy, null, "connect"); @@ -1024,6 +1032,7 @@ private static ConnectionMultiplexer ConnectImpl(object configuration, TextWrite if (!task.Result) throw ExceptionFactory.UnableToConnect(muxer, muxer.failureMessage); killMe = null; + Interlocked.Increment(ref muxer._connectCompletedCount); if (muxer.ServerSelectionStrategy.ServerType == ServerType.Sentinel) { @@ -2466,6 +2475,7 @@ public void Close(bool allowCommandsToComplete = true) DisposeAndClearServers(); OnCloseReaderWriter(); OnClosing(true); + Interlocked.Increment(ref _connectionCloseCount); } partial void OnCloseReaderWriter(); @@ -2582,7 +2592,7 @@ internal Exception GetException(WriteResult result, Message message, ServerEndPo { case WriteResult.Success: return null; case WriteResult.NoConnectionAvailable: - return ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, IncludePerformanceCountersInExceptions, message.Command, message, server, GetServerSnapshot()); + return ExceptionFactory.NoConnectionAvailable(this, message, server); case WriteResult.TimeoutBeforeWrite: return ExceptionFactory.Timeout(this, "The timeout was reached before the message could be written to the output buffer, and it was not sent", message, server, result); case WriteResult.WriteFailure: diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index aa69952e4..1e000be3c 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -18,7 +18,7 @@ internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand c { string s = GetLabel(includeDetail, command, message); var ex = new RedisCommandException("This operation is not available unless admin mode is enabled: " + s); - if (includeDetail) AddDetail(ex, message, server, s); + if (includeDetail) AddExceptionDetail(ex, message, server, s); return ex; } @@ -33,7 +33,7 @@ internal static Exception TooManyArgs(string command, int argCount) internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint server) { var ex = new RedisConnectionException(failureType, message); - if (includeDetail) AddDetail(ex, null, server, null); + if (includeDetail) AddExceptionDetail(ex, null, server, null); return ex; } @@ -41,14 +41,14 @@ internal static Exception DatabaseNotRequired(bool includeDetail, RedisCommand c { string s = command.ToString(); var ex = new RedisCommandException("A target database is not required for " + s); - if (includeDetail) AddDetail(ex, null, null, s); + if (includeDetail) AddExceptionDetail(ex, null, null, s); return ex; } internal static Exception DatabaseOutfRange(bool includeDetail, int targetDatabase, Message message, ServerEndPoint server) { var ex = new RedisCommandException("The database does not exist on the server: " + targetDatabase); - if (includeDetail) AddDetail(ex, message, server, null); + if (includeDetail) AddExceptionDetail(ex, message, server, null); return ex; } @@ -56,7 +56,7 @@ internal static Exception DatabaseRequired(bool includeDetail, RedisCommand comm { string s = command.ToString(); var ex = new RedisCommandException("A target database is required for " + s); - if (includeDetail) AddDetail(ex, null, null, s); + if (includeDetail) AddExceptionDetail(ex, null, null, s); return ex; } @@ -64,14 +64,14 @@ internal static Exception MasterOnly(bool includeDetail, RedisCommand command, M { string s = GetLabel(includeDetail, command, message); var ex = new RedisCommandException("Command cannot be issued to a slave: " + s); - if (includeDetail) AddDetail(ex, message, server, s); + if (includeDetail) AddExceptionDetail(ex, message, server, s); return ex; } internal static Exception MultiSlot(bool includeDetail, Message message) { var ex = new RedisCommandException("Multi-key operations must involve a single slot; keys can use 'hash tags' to help this, i.e. '{/users/12345}/account' and '{/users/12345}/contacts' will always be in the same slot"); - if (includeDetail) AddDetail(ex, message, null, null); + if (includeDetail) AddExceptionDetail(ex, message, null, null); return ex; } @@ -91,9 +91,14 @@ internal static string GetInnerMostExceptionMessage(Exception e) } } - internal static Exception NoConnectionAvailable(bool includeDetail, bool includePerformanceCounters, RedisCommand command, Message message, ServerEndPoint server, ReadOnlySpan serverSnapshot) + internal static Exception NoConnectionAvailable( + ConnectionMultiplexer multiplexer, + Message message, + ServerEndPoint server, + ReadOnlySpan serverSnapshot = default, + RedisCommand command = default) { - string commandLabel = GetLabel(includeDetail, command, message); + string commandLabel = GetLabel(multiplexer.IncludeDetailInExceptions, message?.Command ?? command, message); if (server != null) { @@ -102,25 +107,51 @@ internal static Exception NoConnectionAvailable(bool includeDetail, bool include serverSnapshot = new ServerEndPoint[] { server }; } - var innerException = PopulateInnerExceptions(serverSnapshot); + var innerException = PopulateInnerExceptions(serverSnapshot == default ? multiplexer.GetServerSnapshot() : serverSnapshot); - StringBuilder exceptionmessage = new StringBuilder("No connection is available to service this operation: ").Append(commandLabel); + // Try to get a useful error message for the user. + long attempts = multiplexer._connectAttemptCount, completions = multiplexer._connectCompletedCount; + string initialMessage; + // We only need to customize the connection if we're aborting on connect fail + // The "never" case would have thrown, if this was true + if (!multiplexer.RawConfig.AbortOnConnectFail && attempts <= multiplexer.RawConfig.ConnectRetry && completions == 0) + { + // Initial attempt, attempted use before an async connection completes + initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - connection likely in-progress), unable to service operation: "; + } + else if (!multiplexer.RawConfig.AbortOnConnectFail && attempts > multiplexer.RawConfig.ConnectRetry && completions == 0) + { + // Attempted use after a full initial retry connect count # of failures + // This can happen in Azure often, where user disables abort and has the wrong config + initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - check your config), unable to service operation: "; + } + else + { + // Default if we don't have a more useful error message here based on circumstances + initialMessage = "No connection is active/available to service this operation: "; + } + + StringBuilder sb = new StringBuilder(initialMessage); + sb.Append(commandLabel); string innermostExceptionstring = GetInnerMostExceptionMessage(innerException); if (!string.IsNullOrEmpty(innermostExceptionstring)) { - exceptionmessage.Append("; ").Append(innermostExceptionstring); + sb.Append("; ").Append(innermostExceptionstring); } - if (includeDetail) + // Add counters and exception data if we have it + List> data = null; + if (multiplexer.IncludeDetailInExceptions) { - exceptionmessage.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(includePerformanceCounters)); + data = new List>(); + AddCommonDetail(data, sb, message, multiplexer, server); } - - var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); - - if (includeDetail) + var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); + if (multiplexer.IncludeDetailInExceptions) { - AddDetail(ex, message, server, commandLabel); + CopyDataToException(data, ex); + sb.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(multiplexer.IncludePerformanceCountersInExceptions)); + AddExceptionDetail(ex, message, server, commandLabel); } return ex; } @@ -160,7 +191,7 @@ internal static Exception NotSupported(bool includeDetail, RedisCommand command) { string s = GetLabel(includeDetail, command, null); var ex = new RedisCommandException("Command is not available on your server: " + s); - if (includeDetail) AddDetail(ex, null, null, s); + if (includeDetail) AddExceptionDetail(ex, null, null, s); return ex; } @@ -181,7 +212,16 @@ internal static string GetLibVersion() } return _libVersion; } - internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null) + private static void Add(List> data, StringBuilder sb, string lk, string sk, string v) + { + if (v != null) + { + if (lk != null) data.Add(Tuple.Create(lk, v)); + if (sk != null) sb.Append(", ").Append(sk).Append(": ").Append(v); + } + } + + internal static Exception Timeout(ConnectionMultiplexer multiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null) { List> data = new List> { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); @@ -195,127 +235,138 @@ internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseE } else { - sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)"); - } - - void add(string lk, string sk, string v) - { - if (v != null) - { - if (lk != null) data.Add(Tuple.Create(lk, v)); - if (sk != null) sb.Append(", ").Append(sk).Append(": ").Append(v); - } + sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(multiplexer.TimeoutMilliseconds)).Append("ms)"); } // Add timeout data, if we have it if (result == WriteResult.TimeoutBeforeWrite) { - add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds)); + Add(data, sb, "Timeout", "timeout", Format.ToString(multiplexer.TimeoutMilliseconds)); try { #if DEBUG - if (message.QueuePosition >= 0) add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue - if ((int)message.ConnectionWriteState >= 0) add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue + if (message.QueuePosition >= 0) Add(data, sb, "QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue + if ((int)message.ConnectionWriteState >= 0) Add(data, sb, "WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue #endif if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta)) { - add("Write-State", null, ws.ToString()); - add("Read-State", null, rs.ToString()); + Add(data, sb, "Write-State", null, ws.ToString()); + Add(data, sb, "Read-State", null, rs.ToString()); // these might not always be available if (sentDelta >= 0) { - add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); + Add(data, sb, "OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); } if (receivedDelta >= 0) { - add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); + Add(data, sb, "InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); } } } catch { } } + AddCommonDetail(data, sb, message, multiplexer, server); + + sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); + sb.Append(timeoutHelpLink); + sb.Append(")"); + + var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) + { + HelpLink = timeoutHelpLink + }; + CopyDataToException(data, ex); + + if (multiplexer.IncludeDetailInExceptions) AddExceptionDetail(ex, message, server, null); + return ex; + } + + private static void CopyDataToException(List> data, Exception ex) + { + if (data != null) + { + var exData = ex.Data; + foreach (var kv in data) + { + exData["Redis-" + kv.Item1] = kv.Item2; + } + } + } + + private static void AddCommonDetail( + List> data, + StringBuilder sb, + Message message, + ConnectionMultiplexer multiplexer, + ServerEndPoint server + ) + { if (message != null) { message.TryGetHeadMessages(out var now, out var next); - if (now != null) add("Message-Current", "active", mutiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString()); - if (next != null) add("Message-Next", "next", mutiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString()); + if (now != null) Add(data, sb, "Message-Current", "active", multiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString()); + if (next != null) Add(data, sb, "Message-Next", "next", multiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString()); } // Add server data, if we have it - if (server != null) + if (server != null && message != null) { server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws); - switch(rs) + switch (rs) { case PhysicalConnection.ReadStatus.CompletePendingMessageAsync: case PhysicalConnection.ReadStatus.CompletePendingMessageSync: sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** "); break; } - add("OpsSinceLastHeartbeat", "inst", inst.ToString()); - add("Queue-Awaiting-Write", "qu", qu.ToString()); - add("Queue-Awaiting-Response", "qs", qs.ToString()); - add("Active-Writer", "aw", aw.ToString()); - if (qu != 0) add("Backlog-Writer", "bw", bs.ToString()); - if (rs != PhysicalConnection.ReadStatus.NA) add("Read-State", "rs", rs.ToString()); - if (ws != PhysicalConnection.WriteStatus.NA) add("Write-State", "ws", ws.ToString()); - - if (@in >= 0) add("Inbound-Bytes", "in", @in.ToString()); - if (toRead >= 0) add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString()); - if (toWrite >= 0) add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString()); - - if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0) + Add(data, sb, "OpsSinceLastHeartbeat", "inst", inst.ToString()); + Add(data, sb, "Queue-Awaiting-Write", "qu", qu.ToString()); + Add(data, sb, "Queue-Awaiting-Response", "qs", qs.ToString()); + Add(data, sb, "Active-Writer", "aw", aw.ToString()); + if (qu != 0) Add(data, sb, "Backlog-Writer", "bw", bs.ToString()); + if (rs != PhysicalConnection.ReadStatus.NA) Add(data, sb, "Read-State", "rs", rs.ToString()); + if (ws != PhysicalConnection.WriteStatus.NA) Add(data, sb, "Write-State", "ws", ws.ToString()); + + if (@in >= 0) Add(data, sb, "Inbound-Bytes", "in", @in.ToString()); + if (toRead >= 0) Add(data, sb, "Inbound-Pipe-Bytes", "in-pipe", toRead.ToString()); + if (toWrite >= 0) Add(data, sb, "Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString()); + + if (multiplexer.StormLogThreshold >= 0 && qs >= multiplexer.StormLogThreshold && Interlocked.CompareExchange(ref multiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message.Command); - if (string.IsNullOrWhiteSpace(log)) Interlocked.Exchange(ref mutiplexer.haveStormLog, 0); - else Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log); + if (string.IsNullOrWhiteSpace(log)) Interlocked.Exchange(ref multiplexer.haveStormLog, 0); + else Interlocked.Exchange(ref multiplexer.stormLogSnapshot, log); } - add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/","")); + Add(data, sb, "Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", "")); } - add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); + Add(data, sb, "Multiplexer-Connects", "mc", $"{multiplexer._connectAttemptCount}/{multiplexer._connectCompletedCount}/{multiplexer._connectionCloseCount}"); + Add(data, sb, "Manager", "mgr", multiplexer.SocketManager?.GetState()); - add("Client-Name", "clientName", mutiplexer.ClientName); - var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy); - // only add keyslot if its a valid cluster key slot - if (hashSlot != ServerSelectionStrategy.NoSlot) + Add(data, sb, "Client-Name", "clientName", multiplexer.ClientName); + if (message != null) { - add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString()); + var hashSlot = message.GetHashSlot(multiplexer.ServerSelectionStrategy); + // only add keyslot if its a valid cluster key slot + if (hashSlot != ServerSelectionStrategy.NoSlot) + { + Add(data, sb, "Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(multiplexer.ServerSelectionStrategy).ToString()); + } } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker); - add("ThreadPool-IO-Completion", "IOCP", iocp); - add("ThreadPool-Workers", "WORKER", worker); + Add(data, sb, "ThreadPool-IO-Completion", "IOCP", iocp); + Add(data, sb, "ThreadPool-Workers", "WORKER", worker); data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); - if (mutiplexer.IncludePerformanceCountersInExceptions) - { - add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); - } - - add("Version", "v", GetLibVersion()); - - sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); - sb.Append(timeoutHelpLink); - sb.Append(")"); - - var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) + if (multiplexer.IncludePerformanceCountersInExceptions) { - HelpLink = timeoutHelpLink - }; - if (data != null) - { - var exData = ex.Data; - foreach (var kv in data) - { - exData["Redis-" + kv.Item1] = kv.Item2; - } + Add(data, sb, "Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } - if (mutiplexer.IncludeDetailInExceptions) AddDetail(ex, message, server, null); - return ex; + Add(data, sb, "Version", "v", GetLibVersion()); } - private static void AddDetail(Exception exception, Message message, ServerEndPoint server, string label) + private static void AddExceptionDetail(Exception exception, Message message, ServerEndPoint server, string label) { if (exception != null) { diff --git a/src/StackExchange.Redis/RedisBatch.cs b/src/StackExchange.Redis/RedisBatch.cs index 464a1bc49..33eeac76b 100644 --- a/src/StackExchange.Redis/RedisBatch.cs +++ b/src/StackExchange.Redis/RedisBatch.cs @@ -28,13 +28,13 @@ public void Execute() if (server == null) { FailNoServer(snapshot); - throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server,multiplexer.GetServerSnapshot()); + throw ExceptionFactory.NoConnectionAvailable(multiplexer, message, server); } var bridge = server.GetBridge(message.Command); if (bridge == null) { FailNoServer(snapshot); - throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot()); + throw ExceptionFactory.NoConnectionAvailable(multiplexer, message, server); } // identity a list diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index 87820c1dc..1ab799af3 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -577,7 +577,7 @@ internal override Task ExecuteAsync(Message message, ResultProcessor pr // no need to deny exec-sync here; will be complete before they see if var tcs = TaskSource.Create(asyncState); - ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot())); + ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer, message, server)); return tcs.Task; } return base.ExecuteAsync(message, processor, server); @@ -590,7 +590,7 @@ internal override T ExecuteSync(Message message, ResultProcessor processor if (!server.IsConnected) { if (message == null || message.IsFireAndForget) return default(T); - throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, multiplexer.IncludePerformanceCountersInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot()); + throw ExceptionFactory.NoConnectionAvailable(multiplexer, message, server); } return base.ExecuteSync(message, processor, server); } diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index d7331c1b0..1fbf7a9fc 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -13,7 +13,7 @@ - + diff --git a/tests/StackExchange.Redis.Tests/AsyncTests.cs b/tests/StackExchange.Redis.Tests/AsyncTests.cs index b2398e973..006c6cb20 100644 --- a/tests/StackExchange.Redis.Tests/AsyncTests.cs +++ b/tests/StackExchange.Redis.Tests/AsyncTests.cs @@ -39,7 +39,7 @@ public void AsyncTasksReportFailureIfServerUnavailable() Assert.True(c.IsFaulted, "faulted"); var ex = c.Exception.InnerExceptions.Single(); Assert.IsType(ex); - Assert.StartsWith("No connection is available to service this operation: SADD " + key.ToString(), ex.Message); + Assert.StartsWith("No connection is active/available to service this operation: SADD " + key.ToString(), ex.Message); } } diff --git a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs index 389a3f130..14673c049 100644 --- a/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs +++ b/tests/StackExchange.Redis.Tests/ExceptionFactoryTests.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using Xunit; using Xunit.Abstractions; @@ -16,7 +15,7 @@ public void NullLastException() { var conn = muxer.GetDatabase(); Assert.Null(muxer.GetServerSnapshot()[0].LastException); - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, null); Assert.Null(ex.InnerException); } } @@ -28,13 +27,7 @@ public void CanGetVersion() Assert.Matches(@"2\.[0-9]+\.[0-9]+(\.[0-9]+)?", libVer); } - [Fact] - public void NullSnapshot() - { - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, null); - Assert.Null(ex.InnerException); - } - +#if DEBUG [Fact] public void MultipleEndpointsThrowConnectionException() { @@ -50,7 +43,7 @@ public void MultipleEndpointsThrowConnectionException() muxer.GetServer(endpoint).SimulateConnectionFailure(); } - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, null); var outer = Assert.IsType(ex); Assert.Equal(ConnectionFailureType.UnableToResolvePhysicalConnection, outer.FailureType); var inner = Assert.IsType(outer.InnerException); @@ -63,6 +56,7 @@ public void MultipleEndpointsThrowConnectionException() ClearAmbientFailures(); } } +#endif [Fact] public void ServerTakesPrecendenceOverSnapshot() @@ -76,7 +70,7 @@ public void ServerTakesPrecendenceOverSnapshot() muxer.GetServer(muxer.GetEndPoints()[0]).SimulateConnectionFailure(); - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, muxer.GetServerSnapshot()[0], muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, muxer.GetServerSnapshot()[0]); Assert.IsType(ex); Assert.IsType(ex.InnerException); Assert.Equal(ex.InnerException, muxer.GetServerSnapshot()[0].LastException); @@ -97,7 +91,7 @@ public void NullInnerExceptionForMultipleEndpointsWithNoLastException() { var conn = muxer.GetDatabase(); muxer.AllowConnect = false; - var ex = ExceptionFactory.NoConnectionAvailable(true, true, new RedisCommand(), null, null, muxer.GetServerSnapshot()); + var ex = ExceptionFactory.NoConnectionAvailable(muxer as ConnectionMultiplexer, null, null); Assert.IsType(ex); Assert.Null(ex.InnerException); } @@ -115,7 +109,6 @@ public void TimeoutException() { using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true, shared: false) as ConnectionMultiplexer) { - var conn = muxer.GetDatabase(); var server = GetServer(muxer); muxer.AllowConnect = false; var msg = Message.Create(-1, CommandFlags.None, RedisCommand.PING); @@ -128,6 +121,7 @@ public void TimeoutException() Assert.Contains("clientName: " + nameof(TimeoutException), ex.Message); // Ensure our pipe numbers are in place Assert.Contains("inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0", ex.Message); + Assert.Contains("mc: 1/1/0", ex.Message); Assert.Contains("serverEndpoint: " + server.EndPoint.ToString(), ex.Message); Assert.DoesNotContain("Unspecified/", ex.Message); Assert.EndsWith(" (Please take a look at this article for some common client-side issues that can cause timeouts: https://stackexchange.github.io/StackExchange.Redis/Timeouts)", ex.Message); @@ -139,5 +133,78 @@ public void TimeoutException() ClearAmbientFailures(); } } + + [Theory] + [InlineData(false, 0, 0, true, "Connection to Redis never succeeded (attempts: 0 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 1, 0, true, "Connection to Redis never succeeded (attempts: 1 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 12, 0, true, "Connection to Redis never succeeded (attempts: 12 - check your config), unable to service operation: PING")] + [InlineData(false, 0, 0, false, "Connection to Redis never succeeded (attempts: 0 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 1, 0, false, "Connection to Redis never succeeded (attempts: 1 - connection likely in-progress), unable to service operation: PING")] + [InlineData(false, 12, 0, false, "Connection to Redis never succeeded (attempts: 12 - check your config), unable to service operation: PING")] + [InlineData(true, 0, 0, true, "No connection is active/available to service this operation: PING")] + [InlineData(true, 1, 0, true, "No connection is active/available to service this operation: PING")] + [InlineData(true, 12, 0, true, "No connection is active/available to service this operation: PING")] + public void NoConnectionException(bool abortOnConnect, int connCount, int completeCount, bool hasDetail, string messageStart) + { + try + { + var options = new ConfigurationOptions() + { + AbortOnConnectFail = abortOnConnect, + ConnectTimeout = 500, + SyncTimeout = 500, + KeepAlive = 5000 + }; + + ConnectionMultiplexer muxer; + if (abortOnConnect) + { + options.EndPoints.Add(TestConfig.Current.MasterServerAndPort); + muxer = ConnectionMultiplexer.Connect(options); + } + else + { + options.EndPoints.Add($"doesnot.exist.{Guid.NewGuid():N}:6379"); + muxer = ConnectionMultiplexer.Connect(options); + } + + using (muxer) + { + var server = muxer.GetServer(muxer.GetEndPoints()[0]); + muxer.AllowConnect = false; + muxer._connectAttemptCount = connCount; + muxer._connectCompletedCount = completeCount; + muxer.IncludeDetailInExceptions = hasDetail; + muxer.IncludePerformanceCountersInExceptions = hasDetail; + + var msg = Message.Create(-1, CommandFlags.None, RedisCommand.PING); + var rawEx = ExceptionFactory.NoConnectionAvailable(muxer, msg, new ServerEndPoint(muxer, server.EndPoint)); + var ex = Assert.IsType(rawEx); + Writer.WriteLine("Exception: " + ex.Message); + + // Example format: "Exception: No connection is active/available to service this operation: PING, inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0, serverEndpoint: 127.0.0.1:6379, mc: 1/1/0, mgr: 10 of 10 available, clientName: NoConnectionException, IOCP: (Busy=0,Free=1000,Min=8,Max=1000), WORKER: (Busy=2,Free=2045,Min=8,Max=2047), Local-CPU: 100%, v: 2.1.0.5"; + Assert.StartsWith(messageStart, ex.Message); + + // Ensure our pipe numbers are in place if they should be + if (hasDetail) + { + Assert.Contains("inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0", ex.Message); + Assert.Contains($"mc: {connCount}/{completeCount}/0", ex.Message); + Assert.Contains("serverEndpoint: " + server.EndPoint.ToString().Replace("Unspecified/", ""), ex.Message); + } + else + { + Assert.DoesNotContain("inst: 0, qu: 0, qs: 0, aw: False, in: 0, in-pipe: 0, out-pipe: 0", ex.Message); + Assert.DoesNotContain($"mc: {connCount}/{completeCount}/0", ex.Message); + Assert.DoesNotContain("serverEndpoint: " + server.EndPoint.ToString().Replace("Unspecified/", ""), ex.Message); + } + Assert.DoesNotContain("Unspecified/", ex.Message); + } + } + finally + { + ClearAmbientFailures(); + } + } } } diff --git a/tests/StackExchange.Redis.Tests/Failover.cs b/tests/StackExchange.Redis.Tests/Failover.cs index 860ef058d..f0ef444ef 100644 --- a/tests/StackExchange.Redis.Tests/Failover.cs +++ b/tests/StackExchange.Redis.Tests/Failover.cs @@ -143,7 +143,7 @@ public async Task DeslaveGoesToPrimary() Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave)); var ex = Assert.Throws(() => db.IdentifyEndpoint(key, CommandFlags.DemandSlave)); - Assert.StartsWith("No connection is available to service this operation: EXISTS " + Me(), ex.Message); + Assert.StartsWith("No connection is active/available to service this operation: EXISTS " + Me(), ex.Message); Writer.WriteLine("Invoking MakeMaster()..."); primary.MakeMaster(ReplicationChangeOptions.Broadcast | ReplicationChangeOptions.EnslaveSubordinates | ReplicationChangeOptions.SetTiebreaker, Writer); Writer.WriteLine("Finished MakeMaster() call."); @@ -195,6 +195,7 @@ public async Task DeslaveGoesToPrimary() } } +#if DEBUG [Fact] public async Task SubscriptionsSurviveMasterSwitchAsync() { @@ -353,5 +354,6 @@ public async Task SubscriptionsSurviveMasterSwitchAsync() } } } +#endif } } diff --git a/tests/StackExchange.Redis.Tests/TestBase.cs b/tests/StackExchange.Redis.Tests/TestBase.cs index 301c0c9c5..86b38e372 100644 --- a/tests/StackExchange.Redis.Tests/TestBase.cs +++ b/tests/StackExchange.Redis.Tests/TestBase.cs @@ -356,12 +356,13 @@ public static ConnectionMultiplexer CreateDefault( public static string Me([CallerFilePath] string filePath = null, [CallerMemberName] string caller = null) => #if NET462 - "net462-" + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; -#elif NETCOREAPP2_0 - "netcoreapp2.0-" + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; + "net462-" +#elif NETCOREAPP2_1 + "netcoreapp2.1-" #else - "unknown-" + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; + "unknown-" #endif + + Path.GetFileNameWithoutExtension(filePath) + "-" + caller; protected static TimeSpan RunConcurrent(Action work, int threads, int timeout = 10000, [CallerMemberName] string caller = null) {