From e0d3bbfa0cee59be0eba6e665edb6a55c66bc86f Mon Sep 17 00:00:00 2001 From: David Fowler Date: Tue, 22 Mar 2022 23:19:19 -0700 Subject: [PATCH 01/77] WIP --- .../src/Internal/Http2/Http2Connection.cs | 109 +++++- .../src/Internal/Http2/Http2FrameWriter.cs | 275 ++++++++++++--- .../src/Internal/Http2/Http2OutputProducer.cs | 315 +++++++++++++----- .../Core/src/Internal/Http2/Http2Stream.cs | 3 +- src/Shared/Http2cat/Http2Utilities.cs | 4 +- 5 files changed, 556 insertions(+), 150 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 5f05596a101e..94a9a4cdfea5 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -126,7 +126,9 @@ public Http2Connection(HttpConnectionContext context) _scheduleInline = context.ServiceContext.Scheduler == PipeScheduler.Inline; _inputTask = CopyPipeAsync(_context.Transport.Input, _input.Writer); - _outputTask = CopyPipeAsync(_output.Reader, _context.Transport.Output); + _outputTask = CopyOutputPipeAsync(_output.Reader, _context.Transport.Output); + + _window = _outputFlowControl.Available; } public string ConnectionId => _context.ConnectionId; @@ -1000,7 +1002,7 @@ private Task ProcessWindowUpdateFrameAsync() if (_incomingFrame.StreamId == 0) { - if (!_frameWriter.TryUpdateConnectionWindow(_incomingFrame.WindowUpdateSizeIncrement)) + if (!TryUpdateConnectionWindow(_incomingFrame.WindowUpdateSizeIncrement)) { throw new Http2ConnectionErrorException(CoreStrings.Http2ErrorWindowUpdateSizeInvalid, Http2ErrorCode.FLOW_CONTROL_ERROR); } @@ -1694,6 +1696,104 @@ private PipeOptions GetOutputPipeOptions() useSynchronizationContext: false); } + private readonly object _windowUpdateLock = new(); + private long _window; + private ManualResetValueTaskSource _waitForMoreWindow; + + internal (long, long) ConsumeWindow(long bytes) + { + lock (_windowUpdateLock) + { + var actual = Math.Min(bytes, _window); + var remaining = _window -= actual; + + if (remaining == 0) + { + // Reset the awaitable if we've drained the connection window, it means we can't write any more just yet + _waitForMoreWindow ??= new(); + _waitForMoreWindow.Reset(); + } + + return (actual, remaining); + } + } + + private bool TryUpdateConnectionWindow(int windowUpdateSizeIncrement) + { + ManualResetValueTaskSource? tcs = null; + + lock (_windowUpdateLock) + { + tcs = _waitForMoreWindow; + + _window += windowUpdateSizeIncrement; + } + + // Resume the connection copy loop + tcs?.TrySetResult(null); + return true; + } + + private async Task CopyOutputPipeAsync(PipeReader reader, PipeWriter writer) + { + Exception? error = null; + try + { + while (true) + { + var readResult = await reader.ReadAsync(); + + if ((readResult.IsCompleted && readResult.Buffer.Length == 0) || readResult.IsCanceled) + { + // FIN + break; + } + var buffer = readResult.Buffer; + + var (actual, remaining) = ConsumeWindow(buffer.Length); + + if (actual < buffer.Length) + { + buffer = buffer.Slice(0, actual); + } + + var outputBuffer = writer.GetMemory(_minAllocBufferSize); + + var copyAmount = (int)Math.Min(outputBuffer.Length, buffer.Length); + var bufferSlice = buffer.Slice(0, copyAmount); + + bufferSlice.CopyTo(outputBuffer.Span); + + writer.Advance(copyAmount); + + var result = await writer.FlushAsync(); + + reader.AdvanceTo(bufferSlice.End); + + if (result.IsCompleted || result.IsCanceled) + { + // flushResult should not be canceled. + break; + } + + if (remaining == 0) + { + await new ValueTask(_waitForMoreWindow, _waitForMoreWindow.Version); + } + } + } + catch (Exception ex) + { + // Don't rethrow the exception. It should be handled by the Pipeline consumer. + error = ex; + } + finally + { + await reader.CompleteAsync(error); + await writer.CompleteAsync(error); + } + } + private async Task CopyPipeAsync(PipeReader reader, PipeWriter writer) { Exception? error = null; @@ -1708,11 +1808,12 @@ private async Task CopyPipeAsync(PipeReader reader, PipeWriter writer) // FIN break; } + var buffer = readResult.Buffer; var outputBuffer = writer.GetMemory(_minAllocBufferSize); - var copyAmount = (int)Math.Min(outputBuffer.Length, readResult.Buffer.Length); - var bufferSlice = readResult.Buffer.Slice(0, copyAmount); + var copyAmount = (int)Math.Min(outputBuffer.Length, buffer.Length); + var bufferSlice = buffer.Slice(0, copyAmount); bufferSlice.CopyTo(outputBuffer.Span); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index cec7e6d53566..cabe9db139d2 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -6,11 +6,13 @@ using System.Diagnostics; using System.IO.Pipelines; using System.Net.Http.HPack; +using System.Threading.Channels; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.FlowControl; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure.PipeWriterHelpers; +using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; @@ -33,6 +35,7 @@ internal class Http2FrameWriter private readonly MinDataRate? _minResponseDataRate; private readonly TimingPipeFlusher _flusher; private readonly DynamicHPackEncoder _hpackEncoder; + private readonly Channel _channel; // This is only set to true by tests. private readonly bool _scheduleInline; @@ -72,6 +75,104 @@ public Http2FrameWriter( _scheduleInline = serviceContext.Scheduler == PipeScheduler.Inline; _hpackEncoder = new DynamicHPackEncoder(serviceContext.ServerOptions.AllowResponseHeaderCompression); + _channel = Channel.CreateBounded(serviceContext.ServerOptions.Limits.Http2.MaxStreamsPerConnection); + + _ = WriteToOutputPipe(); + } + + public void Schedule(Http2OutputProducer producer) + { + _channel.Writer.TryWrite(producer); + } + + private async Task WriteToOutputPipe() + { + await foreach (var producer in _channel.Reader.ReadAllAsync()) + { + var reader = producer.PipeReader; + var stream = producer.Stream; + + var hasData = reader.TryRead(out var readResult); + var buffer = readResult.Buffer; + + // This shouldn't be called when there's no data + Debug.Assert(hasData); + + var (actual, remaining) = producer.ConsumeWindow(buffer.Length); + + // Write what we can + if (actual < buffer.Length) + { + buffer = buffer.Slice(0, actual); + } + + FlushResult flushResult = default; + + if (readResult.IsCanceled) + { + // Response body is aborted, break and complete reader. + // break; + } + else if (readResult.IsCompleted && stream.ResponseTrailers?.Count > 0) + { + // Output is ending and there are trailers to write + // Write any remaining content then write trailers + + stream.ResponseTrailers.SetReadOnly(); + stream.DecrementActiveClientStreamCount(); + + // TBD: Trailers + //if (readResult.Buffer.Length > 0) + //{ + // // It is faster to write data and trailers together. Locking once reduces lock contention. + // flushResult = await WriteDataAndTrailersAsync(stream.StreamId, _flowControl, readResult.Buffer, producer.FirstWrite, stream.ResponseTrailers); + //} + //else + //{ + // flushResult = await WriteResponseTrailersAsync(stream.StreamId, stream.ResponseTrailers); + //} + } + else if (readResult.IsCompleted && producer.StreamEnded) + { + if (readResult.Buffer.Length != 0) + { + // TODO: Use the right logger here. + // _log.LogCritical(nameof(Http2OutputProducer) + "." + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); + } + + // Headers have already been written and there is no other content to write + flushResult = await FlushAsync(outputAborter: null, cancellationToken: default); + } + else + { + var endStream = readResult.IsCompleted; + + if (endStream) + { + stream.DecrementActiveClientStreamCount(); + } + + flushResult = await WriteDataAsync(stream.StreamId, buffer, actual, endStream, producer.FirstWrite); + } + + producer.FirstWrite = false; + + var hasModeData = producer.Dequeue(buffer.Length); + + reader.AdvanceTo(buffer.End); + + if (readResult.IsCompleted || readResult.IsCanceled) + { + producer.CompleteResponse(flushResult); + } + // We're not going to schedule this again if there's no remaining window. + // When the window update is sent, the producer will be re-queued if needed. + else if (hasModeData && remaining > 0) + { + // Move this stream to the back of the queue so we're being fair to the other streams that have data + Schedule(producer); + } + } } public void UpdateMaxHeaderTableSize(uint maxHeaderTableSize) @@ -172,19 +273,8 @@ public ValueTask Write100ContinueAsync(int streamId) | Padding (*) ... +---------------------------------------------------------------+ */ - public void WriteResponseHeaders(Http2Stream stream, int statusCode, bool endStream, HttpResponseHeaders headers) + public void WriteResponseHeaders(int streamId, int statusCode, Http2HeadersFrameFlags headerFrameFlags, HttpResponseHeaders headers) { - Http2HeadersFrameFlags headerFrameFlags; - if (endStream) - { - headerFrameFlags = Http2HeadersFrameFlags.END_STREAM; - stream.DecrementActiveClientStreamCount(); - } - else - { - headerFrameFlags = Http2HeadersFrameFlags.NONE; - } - lock (_writeLock) { if (_completed) @@ -195,26 +285,24 @@ public void WriteResponseHeaders(Http2Stream stream, int statusCode, bool endStr try { _headersEnumerator.Initialize(headers); - _outgoingFrame.PrepareHeaders(headerFrameFlags, stream.StreamId); + _outgoingFrame.PrepareHeaders(headerFrameFlags, streamId); var buffer = _headerEncodingBuffer.AsSpan(); var done = HPackHeaderWriter.BeginEncodeHeaders(statusCode, _hpackEncoder, _headersEnumerator, buffer, out var payloadLength); - FinishWritingHeaders(stream.StreamId, payloadLength, done); + FinishWritingHeaders(streamId, payloadLength, done); } // Any exception from the HPack encoder can leave the dynamic table in a corrupt state. // Since we allow custom header encoders we don't know what type of exceptions to expect. catch (Exception ex) { - _log.HPackEncodingError(_connectionId, stream.StreamId, ex); + _log.HPackEncodingError(_connectionId, streamId, ex); _http2Connection.Abort(new ConnectionAbortedException(ex.Message, ex)); throw new InvalidOperationException(ex.Message, ex); // Report the error to the user if this was the first write. } } } - public ValueTask WriteResponseTrailersAsync(Http2Stream stream, HttpResponseTrailers headers) + private ValueTask WriteResponseTrailersAsync(int streamId, HttpResponseTrailers headers) { - stream.DecrementActiveClientStreamCount(); - lock (_writeLock) { if (_completed) @@ -225,16 +313,16 @@ public ValueTask WriteResponseTrailersAsync(Http2Stream stream, Htt try { _headersEnumerator.Initialize(headers); - _outgoingFrame.PrepareHeaders(Http2HeadersFrameFlags.END_STREAM, stream.StreamId); + _outgoingFrame.PrepareHeaders(Http2HeadersFrameFlags.END_STREAM, streamId); var buffer = _headerEncodingBuffer.AsSpan(); var done = HPackHeaderWriter.BeginEncodeHeaders(_hpackEncoder, _headersEnumerator, buffer, out var payloadLength); - FinishWritingHeaders(stream.StreamId, payloadLength, done); + FinishWritingHeaders(streamId, payloadLength, done); } // Any exception from the HPack encoder can leave the dynamic table in a corrupt state. // Since we allow custom header encoders we don't know what type of exceptions to expect. catch (Exception ex) { - _log.HPackEncodingError(_connectionId, stream.StreamId, ex); + _log.HPackEncodingError(_connectionId, streamId, ex); _http2Connection.Abort(new ConnectionAbortedException(ex.Message, ex)); } @@ -271,7 +359,7 @@ private void FinishWritingHeaders(int streamId, int payloadLength, bool done) } } - public ValueTask WriteDataAsync(Http2Stream stream, StreamOutputFlowControl flowControl, in ReadOnlySequence data, bool endStream, bool firstWrite, bool forceFlush) + private ValueTask WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, in ReadOnlySequence data, bool endStream, bool firstWrite, bool forceFlush) { // Logic in this method is replicated in WriteDataAndTrailersAsync. // Changes here may need to be mirrored in WriteDataAndTrailersAsync. @@ -290,12 +378,12 @@ public ValueTask WriteDataAsync(Http2Stream stream, StreamOutputFlo // https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.1 if (dataLength != 0 && dataLength > flowControl.Available) { - return WriteDataAsync(stream, flowControl, data, dataLength, endStream, firstWrite); + return WriteDataAsync(streamId, flowControl, data, dataLength, endStream, firstWrite); } // This cast is safe since if dataLength would overflow an int, it's guaranteed to be greater than the available flow control window. flowControl.Advance((int)dataLength); - WriteDataUnsynchronized(stream, data, dataLength, endStream); + WriteDataUnsynchronized(streamId, data, dataLength, endStream); if (forceFlush) { @@ -306,7 +394,7 @@ public ValueTask WriteDataAsync(Http2Stream stream, StreamOutputFlo } } - public ValueTask WriteDataAndTrailersAsync(Http2Stream stream, StreamOutputFlowControl flowControl, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) + public ValueTask WriteDataAndTrailersAsync(int streamId, StreamOutputFlowControl flowControl, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) { // This method combines WriteDataAsync and WriteResponseTrailers. // Changes here may need to be mirrored in WriteDataAsync. @@ -325,21 +413,21 @@ public ValueTask WriteDataAndTrailersAsync(Http2Stream stream, Stre // https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.1 if (dataLength != 0 && dataLength > flowControl.Available) { - return WriteDataAndTrailersAsyncCore(this, stream, flowControl, data, dataLength, firstWrite, headers); + return WriteDataAndTrailersAsyncCore(this, streamId, flowControl, data, dataLength, firstWrite, headers); } // This cast is safe since if dataLength would overflow an int, it's guaranteed to be greater than the available flow control window. flowControl.Advance((int)dataLength); - WriteDataUnsynchronized(stream, data, dataLength, endStream: false); + WriteDataUnsynchronized(streamId, data, dataLength, endStream: false); - return WriteResponseTrailersAsync(stream, headers); + return WriteResponseTrailersAsync(streamId, headers); } - static async ValueTask WriteDataAndTrailersAsyncCore(Http2FrameWriter writer, Http2Stream stream, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool firstWrite, HttpResponseTrailers headers) + static async ValueTask WriteDataAndTrailersAsyncCore(Http2FrameWriter writer, int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool firstWrite, HttpResponseTrailers headers) { - await writer.WriteDataAsync(stream, flowControl, data, dataLength, endStream: false, firstWrite); + await writer.WriteDataAsync(streamId, flowControl, data, dataLength, endStream: false, firstWrite); - return await writer.WriteResponseTrailersAsync(stream, headers); + return await writer.WriteResponseTrailersAsync(streamId, headers); } } @@ -352,12 +440,12 @@ static async ValueTask WriteDataAndTrailersAsyncCore(Http2FrameWrit | Padding (*) ... +---------------------------------------------------------------+ */ - private void WriteDataUnsynchronized(Http2Stream stream, in ReadOnlySequence data, long dataLength, bool endStream) + private void WriteDataUnsynchronized(int streamId, in ReadOnlySequence data, long dataLength, bool endStream) { Debug.Assert(dataLength == data.Length); // Note padding is not implemented - _outgoingFrame.PrepareData(stream.StreamId); + _outgoingFrame.PrepareData(streamId); if (dataLength > _maxFrameSize) // Minus padding { @@ -365,7 +453,16 @@ private void WriteDataUnsynchronized(Http2Stream stream, in ReadOnlySequence data, long dataLen do { var currentData = remainingData.Slice(0, dataPayloadLength); + _outgoingFrame.PayloadLength = dataPayloadLength; // Plus padding - WriteDataUnsynchronizedCore(stream, endStream: false, dataPayloadLength, currentData); + WriteHeaderUnsynchronized(); + + foreach (var buffer in currentData) + { + _outputWriter.Write(buffer.Span); + } // Plus padding dataLength -= dataPayloadLength; @@ -391,37 +494,101 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen } while (dataLength > dataPayloadLength); - WriteDataUnsynchronizedCore(stream, endStream, dataLength, remainingData); + if (endStream) + { + _outgoingFrame.DataFlags |= Http2DataFrameFlags.END_STREAM; + } + + _outgoingFrame.PayloadLength = (int)dataLength; // Plus padding + + WriteHeaderUnsynchronized(); + + foreach (var buffer in remainingData) + { + _outputWriter.Write(buffer.Span); + } // Plus padding } + } - void WriteDataUnsynchronizedCore(Http2Stream stream, bool endStream, long dataLength, in ReadOnlySequence data) - { - Debug.Assert(dataLength == data.Length); + private async ValueTask WriteDataAsync(int streamId, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) + { + FlushResult flushResult = default; - if (endStream) + var writeTask = default(ValueTask); + + lock (_writeLock) + { + if (_completed) { - _outgoingFrame.DataFlags |= Http2DataFrameFlags.END_STREAM; + return flushResult; + } - // When writing data, must decrement active stream count after flow control availability is checked. - // If active stream count becomes zero while a graceful shutdown is in progress then the input side of connection is closed. - // This is a problem if a large amount of data is being written. The server must keep processing incoming WINDOW_UPDATE frames. - // No WINDOW_UPDATE frames means response write could hit flow control and hang. - // Decrement also has to happen before writing END_STREAM to client to avoid race over active stream count. - stream.DecrementActiveClientStreamCount(); + var shouldFlush = false; + + WriteDataUnsynchronized(streamId, data, dataLength, endStream); + + //if (actual > 0) + //{ + // if (actual < dataLength) + // { + // WriteDataUnsynchronized(streamId, data.Slice(0, actual), actual, endStream: false); + // data = data.Slice(actual); + // dataLength -= actual; + // } + // else + // { + + // } + + // // Don't call FlushAsync() with the min data rate, since we time this write while also accounting for + // // flow control induced backpressure below. + // shouldFlush = true; + //} + //else if (firstWrite) + //{ + // // If we're facing flow control induced backpressure on the first write for a given stream's response body, + // // we make sure to flush the response headers immediately. + // shouldFlush = true; + //} + + //if (shouldFlush) + //{ + + //} + + if (_minResponseDataRate != null) + { + // Call BytesWrittenToBuffer before FlushAsync() to make testing easier, otherwise the Flush can cause test code to run before the timeout + // control updates and if the test checks for a timeout it can fail + _timeoutControl.BytesWrittenToBuffer(_minResponseDataRate, _unflushedBytes); } - // It can be expensive to get length from ROS. Use already available value. - _outgoingFrame.PayloadLength = (int)dataLength; // Plus padding + _unflushedBytes = 0; - WriteHeaderUnsynchronized(); + writeTask = _flusher.FlushAsync(); + + //firstWrite = false; + } + + // REVIEW: This will include flow control waiting as of right now. We need to handle that elsewhere. + if (_minResponseDataRate != null) + { + _timeoutControl.StartTimingWrite(); + } + + flushResult = await writeTask; - data.CopyTo(_outputWriter); + if (_minResponseDataRate != null) + { + _timeoutControl.StopTimingWrite(); } + + return flushResult; } - private async ValueTask WriteDataAsync(Http2Stream stream, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) + private async ValueTask WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) { FlushResult flushResult = default; @@ -446,13 +613,13 @@ private async ValueTask WriteDataAsync(Http2Stream stream, StreamOu { if (actual < dataLength) { - WriteDataUnsynchronized(stream, data.Slice(0, actual), actual, endStream: false); + WriteDataUnsynchronized(streamId, data.Slice(0, actual), actual, endStream: false); data = data.Slice(actual); dataLength -= actual; } else { - WriteDataUnsynchronized(stream, data, actual, endStream); + WriteDataUnsynchronized(streamId, data, actual, endStream); dataLength = 0; } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 0cabbd36bd36..fe7bd182a078 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -43,6 +43,11 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV // Internal for testing internal Task _dataWriteProcessingTask; internal bool _disposed; + internal long _unconsumedBytes; + internal long _window; + + // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state + bool _enqueuedForObservation; /// The core logic for the IValueTaskSource implementation. private ManualResetValueTaskSourceCore _responseCompleteTaskSource = new ManualResetValueTaskSourceCore { RunContinuationsAsynchronously = true }; // mutable struct, do not make this readonly @@ -72,10 +77,84 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, Strea // The minimum output data rate is enforced at the connection level by Http2FrameWriter. _flusher = new TimingPipeFlusher(timeoutControl: null, _log); _flusher.Initialize(_pipeWriter); + _window = flowControl.Available; + + // _dataWriteProcessingTask = ProcessDataWrites(); + } + + public Http2Stream Stream => _stream; + public PipeReader PipeReader => _pipeReader; + + public bool FirstWrite { get; set; } = true; + + public bool StreamEnded => _streamEnded; + + // Added bytes to the queue. + // Returns a bool that represents whether we should schedule this producer to write + // the enqueued bytes + private bool Enqueue(long bytes) + { + lock (_dataWriterLock) + { + var wasEmpty = _unconsumedBytes == 0; + _unconsumedBytes += bytes; + return wasEmpty; + } + } + + // Determines if we should schedule this producer to observe + // any state changes made. + private bool EnqueueForObservation() + { + lock (_dataWriterLock) + { + var wasEnqueuedForObservation = _enqueuedForObservation; + _enqueuedForObservation = true; + return _unconsumedBytes == 0 && !wasEnqueuedForObservation; + } + } + + // Removes consumed bytes from the queue. + // Returns a bool that represents whether we should schedule this producer to write + // the remaining bytes. + internal bool Dequeue(long bytes) + { + lock (_dataWriterLock) + { + var wasEnqueuedForObservation = _enqueuedForObservation; + _enqueuedForObservation = false; + _unconsumedBytes -= bytes; + return _unconsumedBytes > 0 || wasEnqueuedForObservation; + } + } - _dataWriteProcessingTask = ProcessDataWrites(); + // Consumes bytes from the stream's window and returns the remaining bytes and actual bytes consumed + internal (long, long) ConsumeWindow(long bytes) + { + lock (_dataWriterLock) + { + var actual = Math.Min(bytes, _window); + var remaining = _window -= actual; + return (actual, remaining); + } } + // Adds more bytes to the stream's window + // Returns a bool that represents whether we should schedule this producer to write + // the remaining bytes. + private bool UpdateWindow(long bytes) + { + lock (_dataWriterLock) + { + var wasEmpty = _window == 0; + + _window += bytes; + + return wasEmpty && _unconsumedBytes > 0; + } + } + + public void StreamReset() { // Data background task must still be running. @@ -135,7 +214,7 @@ void IHttpOutputAborter.Abort(ConnectionAbortedException abortReason) void IHttpOutputAborter.OnInputOrOutputCompleted() { - _stream.ResetAndAbort(new ConnectionAbortedException($"{nameof(Http2OutputProducer)}.{nameof(ProcessDataWrites)} has completed."), Http2ErrorCode.INTERNAL_ERROR); + _stream.ResetAndAbort(new ConnectionAbortedException($"{nameof(Http2OutputProducer)} has completed."), Http2ErrorCode.INTERNAL_ERROR); } public ValueTask FlushAsync(CancellationToken cancellationToken) @@ -201,11 +280,20 @@ public void WriteResponseHeaders(int statusCode, string? reasonPhrase, HttpRespo // The headers will be the final frame if: // 1. There is no content // 2. There is no trailing HEADERS frame. - _streamEnded = appCompleted - && !_startedWritingDataFrames - && (_stream.ResponseTrailers == null || _stream.ResponseTrailers.Count == 0); + Http2HeadersFrameFlags http2HeadersFrame; - _frameWriter.WriteResponseHeaders(_stream, statusCode, _streamEnded, responseHeaders); + if (appCompleted && !_startedWritingDataFrames && (_stream.ResponseTrailers == null || _stream.ResponseTrailers.Count == 0)) + { + _streamEnded = true; + _stream.DecrementActiveClientStreamCount(); + http2HeadersFrame = Http2HeadersFrameFlags.END_STREAM; + } + else + { + http2HeadersFrame = Http2HeadersFrameFlags.NONE; + } + + _frameWriter.WriteResponseHeaders(StreamId, statusCode, http2HeadersFrame, responseHeaders); } } @@ -230,7 +318,16 @@ public Task WriteDataAsync(ReadOnlySpan data, CancellationToken cancellati _startedWritingDataFrames = true; _pipeWriter.Write(data); - return _flusher.FlushAsync(this, cancellationToken).GetAsTask(); + + var enqueue = Enqueue(data.Length); + var task = _flusher.FlushAsync(this, cancellationToken).GetAsTask(); + + if (enqueue) + { + _frameWriter.Schedule(this); + } + + return task; } } @@ -343,7 +440,16 @@ public ValueTask WriteDataToPipeAsync(ReadOnlySpan data, Canc _startedWritingDataFrames = true; _pipeWriter.Write(data); - return _flusher.FlushAsync(this, cancellationToken); + + var enqueue = Enqueue(data.Length); + var task = _flusher.FlushAsync(this, cancellationToken); + + if (enqueue) + { + _frameWriter.Schedule(this); + } + + return task; } } @@ -378,9 +484,17 @@ public void Stop() _streamCompleted = true; + var enqueue = EnqueueForObservation(); + _pipeReader.CancelPendingRead(); - _frameWriter.AbortPendingStreamDataWrites(_flowControl); + if (enqueue) + { + // We need to make sure the cancellation is observed by the code + _frameWriter.Schedule(this); + } + + // _frameWriter.AbortPendingStreamDataWrites(_flowControl); } } @@ -388,89 +502,101 @@ public void Reset() { } - private async Task ProcessDataWrites() + internal void CompleteResponse(in FlushResult flushResult) { - // ProcessDataWrites runs for the lifetime of the Http2OutputProducer, and is designed to be reused by multiple streams. - // When Http2OutputProducer is no longer used (e.g. a stream is aborted and will no longer be used, or the connection is closed) - // it should be disposed so ProcessDataWrites exits. Not disposing won't cause a memory leak in release builds, but in debug - // builds active tasks are rooted on Task.s_currentActiveTasks. Dispose could be removed in the future when active tasks are - // tracked by a weak reference. See https://github.com/dotnet/runtime/issues/26565 - do - { - FlushResult flushResult = default; - ReadResult readResult = default; - try - { - do - { - var firstWrite = true; - - readResult = await _pipeReader.ReadAsync(); - - if (readResult.IsCanceled) - { - // Response body is aborted, break and complete reader. - break; - } - else if (readResult.IsCompleted && _stream.ResponseTrailers?.Count > 0) - { - // Output is ending and there are trailers to write - // Write any remaining content then write trailers - - _stream.ResponseTrailers.SetReadOnly(); - - if (readResult.Buffer.Length > 0) - { - // It is faster to write data and trailers together. Locking once reduces lock contention. - flushResult = await _frameWriter.WriteDataAndTrailersAsync(_stream, _flowControl, readResult.Buffer, firstWrite, _stream.ResponseTrailers); - } - else - { - flushResult = await _frameWriter.WriteResponseTrailersAsync(_stream, _stream.ResponseTrailers); - } - } - else if (readResult.IsCompleted && _streamEnded) - { - if (readResult.Buffer.Length != 0) - { - ThrowUnexpectedState(); - } - - // Headers have already been written and there is no other content to write - flushResult = await _frameWriter.FlushAsync(outputAborter: null, cancellationToken: default); - } - else - { - var endStream = readResult.IsCompleted; - flushResult = await _frameWriter.WriteDataAsync(_stream, _flowControl, readResult.Buffer, endStream, firstWrite, forceFlush: true); - } - - firstWrite = false; - _pipeReader.AdvanceTo(readResult.Buffer.End); - } while (!readResult.IsCompleted); - } - catch (Exception ex) - { - _log.LogCritical(ex, nameof(Http2OutputProducer) + "." + nameof(ProcessDataWrites) + " observed an unexpected exception."); - } - - await _pipeReader.CompleteAsync(); - - // Signal via WriteStreamSuffixAsync to the stream that output has finished. - // Stream state will move to RequestProcessingStatus.ResponseCompleted - _responseCompleteTaskSource.SetResult(flushResult); - - // Wait here for the stream to be reset or disposed. - await new ValueTask(_resetAwaitable, _resetAwaitable.Version); - _resetAwaitable.Reset(); - } while (!_disposed); - - static void ThrowUnexpectedState() - { - throw new InvalidOperationException(nameof(Http2OutputProducer) + "." + nameof(ProcessDataWrites) + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); - } + _responseCompleteTaskSource.SetResult(flushResult); } + //private async Task ProcessDataWrites() + //{ + // // ProcessDataWrites runs for the lifetime of the Http2OutputProducer, and is designed to be reused by multiple streams. + // // When Http2OutputProducer is no longer used (e.g. a stream is aborted and will no longer be used, or the connection is closed) + // // it should be disposed so ProcessDataWrites exits. Not disposing won't cause a memory leak in release builds, but in debug + // // builds active tasks are rooted on Task.s_currentActiveTasks. Dispose could be removed in the future when active tasks are + // // tracked by a weak reference. See https://github.com/dotnet/runtime/issues/26565 + // do + // { + // FlushResult flushResult = default; + // ReadResult readResult = default; + // try + // { + // do + // { + // var firstWrite = true; + + // readResult = await _pipeReader.ReadAsync(); + + // if (readResult.IsCanceled) + // { + // // Response body is aborted, break and complete reader. + // break; + // } + // else if (readResult.IsCompleted && _stream.ResponseTrailers?.Count > 0) + // { + // // Output is ending and there are trailers to write + // // Write any remaining content then write trailers + + // _stream.ResponseTrailers.SetReadOnly(); + // _stream.DecrementActiveClientStreamCount(); + + // if (readResult.Buffer.Length > 0) + // { + // // It is faster to write data and trailers together. Locking once reduces lock contention. + // flushResult = await _frameWriter.WriteDataAndTrailersAsync(StreamId, _flowControl, readResult.Buffer, firstWrite, _stream.ResponseTrailers); + // } + // else + // { + // flushResult = await _frameWriter.WriteResponseTrailersAsync(StreamId, _stream.ResponseTrailers); + // } + // } + // else if (readResult.IsCompleted && _streamEnded) + // { + // if (readResult.Buffer.Length != 0) + // { + // ThrowUnexpectedState(); + // } + + // // Headers have already been written and there is no other content to write + // flushResult = await _frameWriter.FlushAsync(outputAborter: null, cancellationToken: default); + // } + // else + // { + // var endStream = readResult.IsCompleted; + + // if (endStream) + // { + // _stream.DecrementActiveClientStreamCount(); + // } + + // flushResult = await _frameWriter.WriteDataAsync(StreamId, _flowControl, readResult.Buffer, endStream, firstWrite, forceFlush: true); + // } + + // firstWrite = false; + // _pipeReader.AdvanceTo(readResult.Buffer.End); + // } while (!readResult.IsCompleted); + // } + // catch (Exception ex) + // { + // _log.LogCritical(ex, nameof(Http2OutputProducer) + "." + nameof(ProcessDataWrites) + " observed an unexpected exception."); + // } + + // await _pipeReader.CompleteAsync(); + + // // Signal via WriteStreamSuffixAsync to the stream that output has finished. + // // Stream state will move to RequestProcessingStatus.ResponseCompleted + // _responseCompleteTaskSource.SetResult(flushResult); + + // // Wait here for the stream to be reset or disposed. + // await new ValueTask(_resetAwaitable, _resetAwaitable.Version); + // _resetAwaitable.Reset(); + // } while (!_disposed); + + // static void ThrowUnexpectedState() + // { + // throw new InvalidOperationException(nameof(Http2OutputProducer) + "." + nameof(ProcessDataWrites) + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); + // } + //} + internal Memory GetFakeMemory(int minSize) { // Try to reuse _fakeMemoryOwner @@ -515,6 +641,17 @@ internal Memory GetFakeMemory(int minSize) } } + // REVIEW: When does this fail? When the connection is aborted? + public bool TryUpdateStreamWindow(int bytes) + { + if (UpdateWindow(bytes)) + { + _frameWriter.Schedule(this); + } + + return true; + } + [StackTraceHidden] private void ThrowIfSuffixSentOrCompleted() { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index 54f4e14a0c5e..29d6b94d5a30 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -506,7 +506,8 @@ public void OnDataRead(int bytesRead) public bool TryUpdateOutputWindow(int bytes) { - return _context.FrameWriter.TryUpdateStreamWindow(_outputFlowControl, bytes); + return _http2Output.TryUpdateStreamWindow(bytes); + // return _context.FrameWriter.TryUpdateStreamWindow(_outputFlowControl, bytes); } public void AbortRstStreamReceived() diff --git a/src/Shared/Http2cat/Http2Utilities.cs b/src/Shared/Http2cat/Http2Utilities.cs index c5479147e1ed..6b73ea63c931 100644 --- a/src/Shared/Http2cat/Http2Utilities.cs +++ b/src/Shared/Http2cat/Http2Utilities.cs @@ -393,7 +393,7 @@ public Task SendAsync(ReadOnlySpan span) public static async Task FlushAsync(PipeWriter writableBuffer) { - await writableBuffer.FlushAsync().AsTask().DefaultTimeout(); + await writableBuffer.FlushAsync().AsTask(); } public Task SendPreambleAsync() => SendAsync(ClientPreface); @@ -827,7 +827,7 @@ internal async Task ReceiveFrameAsync(uint maxFrameSize = while (true) { - var result = await _pair.Application.Input.ReadAsync().AsTask().DefaultTimeout(); + var result = await _pair.Application.Input.ReadAsync().AsTask();//.DefaultTimeout(); var buffer = result.Buffer; var consumed = buffer.Start; var examined = buffer.Start; From f478542b8023d18564d46bab6e6ff0dfc1f728c3 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 00:08:15 -0700 Subject: [PATCH 02/77] Make more things work --- .../src/Internal/Http2/Http2FrameWriter.cs | 2 + .../src/Internal/Http2/Http2OutputProducer.cs | 123 +++++------------- .../PipeWriterHelpers/ConcurrentPipeWriter.cs | 4 + .../Kestrel/samples/Http2SampleApp/Startup.cs | 2 +- 4 files changed, 37 insertions(+), 94 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index cabe9db139d2..fdd59df2fd6a 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -163,6 +163,8 @@ private async Task WriteToOutputPipe() if (readResult.IsCompleted || readResult.IsCanceled) { + await reader.CompleteAsync(); + producer.CompleteResponse(flushResult); } // We're not going to schedule this again if there's no remaining window. diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index fe7bd182a078..b4d9cc64de05 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -98,7 +98,7 @@ private bool Enqueue(long bytes) { var wasEmpty = _unconsumedBytes == 0; _unconsumedBytes += bytes; - return wasEmpty; + return wasEmpty && _unconsumedBytes > 0; } } @@ -158,7 +158,7 @@ private bool UpdateWindow(long bytes) public void StreamReset() { // Data background task must still be running. - Debug.Assert(!_dataWriteProcessingTask.IsCompleted); + // Debug.Assert(!_dataWriteProcessingTask.IsCompleted); // Response should have been completed. Debug.Assert(_responseCompleteTaskSource.GetStatus(_responseCompleteTaskSource.Version) == ValueTaskSourceStatus.Succeeded); @@ -171,6 +171,10 @@ public void StreamReset() _pipeWriter.Reset(); _responseCompleteTaskSource.Reset(); + _window = _flowControl.Available; + _unconsumedBytes = 0; + _enqueuedForObservation = false; + // Trigger the data process task to resume _resetAwaitable.SetResult(null); } @@ -188,9 +192,15 @@ public void Complete() Stop(); + var enqueue = EnqueueForObservation(); // Make sure the writing side is completed. _pipeWriter.Complete(); + if (enqueue) + { + _frameWriter.Schedule(this); + } + if (_fakeMemoryOwner != null) { _fakeMemoryOwner.Dispose(); @@ -234,9 +244,19 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) if (_startedWritingDataFrames) { + Debug.Assert(_pipeWriter.CanGetUnflushedBytes); + + var enqueue = Enqueue(_pipeWriter.UnflushedBytes); // If there's already been response data written to the stream, just wait for that. Any header // should be in front of the data frames in the connection pipe. Trailers could change things. - return _flusher.FlushAsync(this, cancellationToken); + var task = _flusher.FlushAsync(this, cancellationToken); + + if (enqueue) + { + _frameWriter.Schedule(this); + } + + return task; } else { @@ -343,7 +363,14 @@ public ValueTask WriteStreamSuffixAsync() _streamCompleted = true; _suffixSent = true; + var enqueue = EnqueueForObservation(); _pipeWriter.Complete(); + + if (enqueue) + { + _frameWriter.Schedule(this); + } + return GetWaiterTask(); } } @@ -507,96 +534,6 @@ internal void CompleteResponse(in FlushResult flushResult) _responseCompleteTaskSource.SetResult(flushResult); } - //private async Task ProcessDataWrites() - //{ - // // ProcessDataWrites runs for the lifetime of the Http2OutputProducer, and is designed to be reused by multiple streams. - // // When Http2OutputProducer is no longer used (e.g. a stream is aborted and will no longer be used, or the connection is closed) - // // it should be disposed so ProcessDataWrites exits. Not disposing won't cause a memory leak in release builds, but in debug - // // builds active tasks are rooted on Task.s_currentActiveTasks. Dispose could be removed in the future when active tasks are - // // tracked by a weak reference. See https://github.com/dotnet/runtime/issues/26565 - // do - // { - // FlushResult flushResult = default; - // ReadResult readResult = default; - // try - // { - // do - // { - // var firstWrite = true; - - // readResult = await _pipeReader.ReadAsync(); - - // if (readResult.IsCanceled) - // { - // // Response body is aborted, break and complete reader. - // break; - // } - // else if (readResult.IsCompleted && _stream.ResponseTrailers?.Count > 0) - // { - // // Output is ending and there are trailers to write - // // Write any remaining content then write trailers - - // _stream.ResponseTrailers.SetReadOnly(); - // _stream.DecrementActiveClientStreamCount(); - - // if (readResult.Buffer.Length > 0) - // { - // // It is faster to write data and trailers together. Locking once reduces lock contention. - // flushResult = await _frameWriter.WriteDataAndTrailersAsync(StreamId, _flowControl, readResult.Buffer, firstWrite, _stream.ResponseTrailers); - // } - // else - // { - // flushResult = await _frameWriter.WriteResponseTrailersAsync(StreamId, _stream.ResponseTrailers); - // } - // } - // else if (readResult.IsCompleted && _streamEnded) - // { - // if (readResult.Buffer.Length != 0) - // { - // ThrowUnexpectedState(); - // } - - // // Headers have already been written and there is no other content to write - // flushResult = await _frameWriter.FlushAsync(outputAborter: null, cancellationToken: default); - // } - // else - // { - // var endStream = readResult.IsCompleted; - - // if (endStream) - // { - // _stream.DecrementActiveClientStreamCount(); - // } - - // flushResult = await _frameWriter.WriteDataAsync(StreamId, _flowControl, readResult.Buffer, endStream, firstWrite, forceFlush: true); - // } - - // firstWrite = false; - // _pipeReader.AdvanceTo(readResult.Buffer.End); - // } while (!readResult.IsCompleted); - // } - // catch (Exception ex) - // { - // _log.LogCritical(ex, nameof(Http2OutputProducer) + "." + nameof(ProcessDataWrites) + " observed an unexpected exception."); - // } - - // await _pipeReader.CompleteAsync(); - - // // Signal via WriteStreamSuffixAsync to the stream that output has finished. - // // Stream state will move to RequestProcessingStatus.ResponseCompleted - // _responseCompleteTaskSource.SetResult(flushResult); - - // // Wait here for the stream to be reset or disposed. - // await new ValueTask(_resetAwaitable, _resetAwaitable.Version); - // _resetAwaitable.Reset(); - // } while (!_disposed); - - // static void ThrowUnexpectedState() - // { - // throw new InvalidOperationException(nameof(Http2OutputProducer) + "." + nameof(ProcessDataWrites) + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); - // } - //} - internal Memory GetFakeMemory(int minSize) { // Try to reuse _fakeMemoryOwner diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs index 0256e7830a23..d1683adb54dc 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs @@ -57,6 +57,10 @@ public ConcurrentPipeWriter(PipeWriter innerPipeWriter, MemoryPool pool, o _sync = sync; } + public override bool CanGetUnflushedBytes => true; + + public override long UnflushedBytes => _currentFlushTcs is null && _head is null ? _innerPipeWriter.UnflushedBytes : _bytesBuffered; + public void Reset() { Debug.Assert(_currentFlushTcs == null, "There should not be a pending flush."); diff --git a/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs b/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs index 93e1f2fda79e..a5dc4500fecd 100644 --- a/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs +++ b/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs @@ -17,7 +17,7 @@ public void ConfigureServices(IServiceCollection services) public void Configure(IApplicationBuilder app) { - app.UseTimingMiddleware(); + // app.UseTimingMiddleware(); app.Run(context => { return context.Response.WriteAsync("Hello World! " + context.Request.Protocol); From d2a5eb599c2b2061f81db73165da9ebcfec9bcdb Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 01:24:23 -0700 Subject: [PATCH 03/77] No more crashes! - Fix double complete --- .../src/Internal/Http2/Http2FrameWriter.cs | 141 +++++++++--------- .../src/Internal/Http2/Http2OutputProducer.cs | 19 ++- 2 files changed, 85 insertions(+), 75 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index fdd59df2fd6a..f62195fbb60b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -77,7 +77,7 @@ public Http2FrameWriter( _hpackEncoder = new DynamicHPackEncoder(serviceContext.ServerOptions.AllowResponseHeaderCompression); _channel = Channel.CreateBounded(serviceContext.ServerOptions.Limits.Http2.MaxStreamsPerConnection); - _ = WriteToOutputPipe(); + _ = Task.Run(WriteToOutputPipe); } public void Schedule(Http2OutputProducer producer) @@ -89,90 +89,97 @@ private async Task WriteToOutputPipe() { await foreach (var producer in _channel.Reader.ReadAllAsync()) { - var reader = producer.PipeReader; - var stream = producer.Stream; - - var hasData = reader.TryRead(out var readResult); - var buffer = readResult.Buffer; - - // This shouldn't be called when there's no data - Debug.Assert(hasData); - - var (actual, remaining) = producer.ConsumeWindow(buffer.Length); - - // Write what we can - if (actual < buffer.Length) + try { - buffer = buffer.Slice(0, actual); - } + var reader = producer.PipeReader; + var stream = producer.Stream; - FlushResult flushResult = default; + var hasData = reader.TryRead(out var readResult); + var buffer = readResult.Buffer; - if (readResult.IsCanceled) - { - // Response body is aborted, break and complete reader. - // break; - } - else if (readResult.IsCompleted && stream.ResponseTrailers?.Count > 0) - { - // Output is ending and there are trailers to write - // Write any remaining content then write trailers + // This shouldn't be called when there's no data + Debug.Assert(hasData); - stream.ResponseTrailers.SetReadOnly(); - stream.DecrementActiveClientStreamCount(); + var (actual, remaining) = producer.ConsumeWindow(buffer.Length); - // TBD: Trailers - //if (readResult.Buffer.Length > 0) - //{ - // // It is faster to write data and trailers together. Locking once reduces lock contention. - // flushResult = await WriteDataAndTrailersAsync(stream.StreamId, _flowControl, readResult.Buffer, producer.FirstWrite, stream.ResponseTrailers); - //} - //else - //{ - // flushResult = await WriteResponseTrailersAsync(stream.StreamId, stream.ResponseTrailers); - //} - } - else if (readResult.IsCompleted && producer.StreamEnded) - { - if (readResult.Buffer.Length != 0) + // Write what we can + if (actual < buffer.Length) { - // TODO: Use the right logger here. - // _log.LogCritical(nameof(Http2OutputProducer) + "." + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); + buffer = buffer.Slice(0, actual); } - // Headers have already been written and there is no other content to write - flushResult = await FlushAsync(outputAborter: null, cancellationToken: default); - } - else - { - var endStream = readResult.IsCompleted; + FlushResult flushResult = default; - if (endStream) + if (readResult.IsCanceled) { + // Response body is aborted, break and complete reader. + // break; + } + else if (readResult.IsCompleted && stream.ResponseTrailers?.Count > 0) + { + // Output is ending and there are trailers to write + // Write any remaining content then write trailers + + stream.ResponseTrailers.SetReadOnly(); stream.DecrementActiveClientStreamCount(); + + // TBD: Trailers + //if (readResult.Buffer.Length > 0) + //{ + // // It is faster to write data and trailers together. Locking once reduces lock contention. + // flushResult = await WriteDataAndTrailersAsync(stream.StreamId, _flowControl, readResult.Buffer, producer.FirstWrite, stream.ResponseTrailers); + //} + //else + //{ + // flushResult = await WriteResponseTrailersAsync(stream.StreamId, stream.ResponseTrailers); + //} + } + else if (readResult.IsCompleted && producer.StreamEnded) + { + if (readResult.Buffer.Length != 0) + { + // TODO: Use the right logger here. + // _log.LogCritical(nameof(Http2OutputProducer) + "." + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); + } + + // Headers have already been written and there is no other content to write + flushResult = await FlushAsync(outputAborter: null, cancellationToken: default); } + else + { + var endStream = readResult.IsCompleted; - flushResult = await WriteDataAsync(stream.StreamId, buffer, actual, endStream, producer.FirstWrite); - } + if (endStream) + { + stream.DecrementActiveClientStreamCount(); + } + + flushResult = await WriteDataAsync(stream.StreamId, buffer, actual, endStream, producer.FirstWrite); + } - producer.FirstWrite = false; + producer.FirstWrite = false; - var hasModeData = producer.Dequeue(buffer.Length); + var hasModeData = producer.Dequeue(buffer.Length); - reader.AdvanceTo(buffer.End); + reader.AdvanceTo(buffer.End); - if (readResult.IsCompleted || readResult.IsCanceled) - { - await reader.CompleteAsync(); + if (readResult.IsCompleted || readResult.IsCanceled) + { + await reader.CompleteAsync(); - producer.CompleteResponse(flushResult); + producer.CompleteResponse(flushResult); + } + // We're not going to schedule this again if there's no remaining window. + // When the window update is sent, the producer will be re-queued if needed. + else if (hasModeData && remaining > 0) + { + // Move this stream to the back of the queue so we're being fair to the other streams that have data + Schedule(producer); + } } - // We're not going to schedule this again if there's no remaining window. - // When the window update is sent, the producer will be re-queued if needed. - else if (hasModeData && remaining > 0) + catch (Exception ex) { - // Move this stream to the back of the queue so we're being fair to the other streams that have data - Schedule(producer); + _log.LogCritical(ex, "The event loop in connection {ConnectionId} failed unexpectedly", _connectionId); } } } @@ -541,7 +548,7 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen // } // else // { - + // } // // Don't call FlushAsync() with the min data rate, since we time this write while also accounting for @@ -557,7 +564,7 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen //if (shouldFlush) //{ - + //} if (_minResponseDataRate != null) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index b4d9cc64de05..144b42cf547d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -176,7 +176,7 @@ public void StreamReset() _enqueuedForObservation = false; // Trigger the data process task to resume - _resetAwaitable.SetResult(null); + // _resetAwaitable.SetResult(null); } public void Complete() @@ -192,13 +192,16 @@ public void Complete() Stop(); - var enqueue = EnqueueForObservation(); - // Make sure the writing side is completed. - _pipeWriter.Complete(); - - if (enqueue) + if (!_streamCompleted) { - _frameWriter.Schedule(this); + var enqueue = EnqueueForObservation(); + // Make sure the writing side is completed. + _pipeWriter.Complete(); + + if (enqueue) + { + _frameWriter.Schedule(this); + } } if (_fakeMemoryOwner != null) @@ -636,6 +639,6 @@ public void Dispose() _disposed = true; // Set awaitable after disposed is true to ensure ProcessDataWrites exits successfully. - _resetAwaitable.SetResult(null); + // _resetAwaitable.SetResult(null); } } From 6f07604d32fab9ab95a1da1a45f7aa2939acf478 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 08:27:19 -0700 Subject: [PATCH 04/77] Added trailers --- .../src/Internal/Http2/Http2FrameWriter.cs | 68 +++++++++++++++---- .../Kestrel/samples/Http2SampleApp/Startup.cs | 2 +- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index f62195fbb60b..721754bedb87 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -123,27 +123,28 @@ private async Task WriteToOutputPipe() stream.ResponseTrailers.SetReadOnly(); stream.DecrementActiveClientStreamCount(); - // TBD: Trailers - //if (readResult.Buffer.Length > 0) - //{ - // // It is faster to write data and trailers together. Locking once reduces lock contention. - // flushResult = await WriteDataAndTrailersAsync(stream.StreamId, _flowControl, readResult.Buffer, producer.FirstWrite, stream.ResponseTrailers); - //} - //else - //{ - // flushResult = await WriteResponseTrailersAsync(stream.StreamId, stream.ResponseTrailers); - //} + if (buffer.Length > 0) + { + // It is faster to write data and trailers together. Locking once reduces lock contention. + flushResult = await WriteDataAndTrailersAsync(stream.StreamId, buffer, producer.FirstWrite, stream.ResponseTrailers); + } + else + { + flushResult = await WriteResponseTrailersAsync(stream.StreamId, stream.ResponseTrailers); + } } else if (readResult.IsCompleted && producer.StreamEnded) { - if (readResult.Buffer.Length != 0) + if (buffer.Length != 0) { // TODO: Use the right logger here. - // _log.LogCritical(nameof(Http2OutputProducer) + "." + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); + _log.LogCritical(nameof(Http2OutputProducer) + "." + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); + } + else + { + // Headers have already been written and there is no other content to write + flushResult = await FlushAsync(outputAborter: null, cancellationToken: default); } - - // Headers have already been written and there is no other content to write - flushResult = await FlushAsync(outputAborter: null, cancellationToken: default); } else { @@ -403,6 +404,43 @@ private ValueTask WriteDataAsync(int streamId, StreamOutputFlowCont } } + public ValueTask WriteDataAndTrailersAsync(int streamId, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) + { + // This method combines WriteDataAsync and WriteResponseTrailers. + // Changes here may need to be mirrored in WriteDataAsync. + + // The Length property of a ReadOnlySequence can be expensive, so we cache the value. + var dataLength = data.Length; + + lock (_writeLock) + { + if (_completed) + { + return default; + } + + //// Zero-length data frames are allowed to be sent immediately even if there is no space available in the flow control window. + //// https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.1 + //if (dataLength != 0 && dataLength > flowControl.Available) + //{ + // return WriteDataAndTrailersAsyncCore(this, streamId, flowControl, data, dataLength, firstWrite, headers); + //} + + // This cast is safe since if dataLength would overflow an int, it's guaranteed to be greater than the available flow control window. + // flowControl.Advance((int)dataLength); + WriteDataUnsynchronized(streamId, data, dataLength, endStream: false); + + return WriteResponseTrailersAsync(streamId, headers); + } + + //static async ValueTask WriteDataAndTrailersAsyncCore(Http2FrameWriter writer, int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool firstWrite, HttpResponseTrailers headers) + //{ + // await writer.WriteDataAsync(streamId, flowControl, data, dataLength, endStream: false, firstWrite); + + // return await writer.WriteResponseTrailersAsync(streamId, headers); + //} + } + public ValueTask WriteDataAndTrailersAsync(int streamId, StreamOutputFlowControl flowControl, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) { // This method combines WriteDataAsync and WriteResponseTrailers. diff --git a/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs b/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs index a5dc4500fecd..93e1f2fda79e 100644 --- a/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs +++ b/src/Servers/Kestrel/samples/Http2SampleApp/Startup.cs @@ -17,7 +17,7 @@ public void ConfigureServices(IServiceCollection services) public void Configure(IApplicationBuilder app) { - // app.UseTimingMiddleware(); + app.UseTimingMiddleware(); app.Run(context => { return context.Response.WriteAsync("Hello World! " + context.Request.Protocol); From e70cb393c1dee42876de47a9428dd60c10d184e7 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 09:04:18 -0700 Subject: [PATCH 05/77] Delete more code so it's obvious where changes are in the diff --- .../src/Internal/Http2/Http2Connection.cs | 26 +++++++++++++++---- .../src/Internal/Http2/Http2FrameWriter.cs | 3 +-- .../src/Internal/Http2/Http2OutputProducer.cs | 11 -------- .../Http2/Http2ConnectionTests.cs | 4 +-- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 94a9a4cdfea5..933eb2d8166b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -75,6 +75,10 @@ internal partial class Http2Connection : IHttp2StreamLifetimeHandler, IHttpStrea internal readonly Dictionary _streams = new Dictionary(); internal PooledStreamStack StreamPool; + private readonly object _windowUpdateLock = new(); + private long _window; + private ManualResetValueTaskSource? _waitForMoreWindow; + public Http2Connection(HttpConnectionContext context) { var httpLimits = context.ServiceContext.ServerOptions.Limits; @@ -371,10 +375,12 @@ public async Task ProcessRequestsAsync(IHttpApplication appl TimeoutControl.StartDrainTimeout(Limits.MinResponseDataRate, Limits.MaxResponseBufferSize); _frameWriter.Complete(); + AbortFlowControl(); } catch { _frameWriter.Abort(connectionError); + AbortFlowControl(); throw; } finally @@ -1696,10 +1702,6 @@ private PipeOptions GetOutputPipeOptions() useSynchronizationContext: false); } - private readonly object _windowUpdateLock = new(); - private long _window; - private ManualResetValueTaskSource _waitForMoreWindow; - internal (long, long) ConsumeWindow(long bytes) { lock (_windowUpdateLock) @@ -1734,6 +1736,19 @@ private bool TryUpdateConnectionWindow(int windowUpdateSizeIncrement) return true; } + private void AbortFlowControl() + { + ManualResetValueTaskSource? tcs = null; + + lock (_windowUpdateLock) + { + tcs = _waitForMoreWindow; + } + + // Resume the connection copy loop + tcs?.TrySetResult(null); + } + private async Task CopyOutputPipeAsync(PipeReader reader, PipeWriter writer) { Exception? error = null; @@ -1778,7 +1793,8 @@ private async Task CopyOutputPipeAsync(PipeReader reader, PipeWriter writer) if (remaining == 0) { - await new ValueTask(_waitForMoreWindow, _waitForMoreWindow.Version); + // This shouldn't be null here + await new ValueTask(_waitForMoreWindow!, _waitForMoreWindow!.Version); } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 721754bedb87..77eb82ea50e4 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -215,7 +215,6 @@ public void Complete() } _completed = true; - _connectionOutputFlowControl.Abort(); _outputWriter.Abort(); } } @@ -572,7 +571,7 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen return flushResult; } - var shouldFlush = false; + // var shouldFlush = false; WriteDataUnsynchronized(streamId, data, dataLength, endStream); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 144b42cf547d..9c2273226113 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -31,7 +31,6 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV private readonly Pipe _pipe; private readonly ConcurrentPipeWriter _pipeWriter; private readonly PipeReader _pipeReader; - private readonly ManualResetValueTaskSource _resetAwaitable = new ManualResetValueTaskSource(); private IMemoryOwner? _fakeMemoryOwner; private byte[]? _fakeMemory; private bool _startedWritingDataFrames; @@ -41,7 +40,6 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV private bool _writerComplete; // Internal for testing - internal Task _dataWriteProcessingTask; internal bool _disposed; internal long _unconsumedBytes; internal long _window; @@ -78,8 +76,6 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, Strea _flusher = new TimingPipeFlusher(timeoutControl: null, _log); _flusher.Initialize(_pipeWriter); _window = flowControl.Available; - - // _dataWriteProcessingTask = ProcessDataWrites(); } public Http2Stream Stream => _stream; @@ -157,8 +153,6 @@ private bool UpdateWindow(long bytes) public void StreamReset() { - // Data background task must still be running. - // Debug.Assert(!_dataWriteProcessingTask.IsCompleted); // Response should have been completed. Debug.Assert(_responseCompleteTaskSource.GetStatus(_responseCompleteTaskSource.Version) == ValueTaskSourceStatus.Succeeded); @@ -174,9 +168,6 @@ public void StreamReset() _window = _flowControl.Available; _unconsumedBytes = 0; _enqueuedForObservation = false; - - // Trigger the data process task to resume - // _resetAwaitable.SetResult(null); } public void Complete() @@ -638,7 +629,5 @@ public void Dispose() } _disposed = true; - // Set awaitable after disposed is true to ensure ProcessDataWrites exits successfully. - // _resetAwaitable.SetResult(null); } } diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index 55fafea7826e..2f517f239dde 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -426,7 +426,7 @@ await ExpectAsync(Http2FrameType.HEADERS, await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); var output = (Http2OutputProducer)stream.Output; - await output._dataWriteProcessingTask.DefaultTimeout(); + // await output._dataWriteProcessingTask.DefaultTimeout(); } [Fact] @@ -577,7 +577,7 @@ await InitializeConnectionAsync(async context => var output = (Http2OutputProducer)stream.Output; Assert.True(output._disposed); - await output._dataWriteProcessingTask.DefaultTimeout(); + // await output._dataWriteProcessingTask.DefaultTimeout(); // Stream is not returned to the pool Assert.Equal(0, _connection.StreamPool.Count); From 1d5ba975b391111b3763ae1d4dd0ebd5fc25de61 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 09:09:52 -0700 Subject: [PATCH 06/77] Complete the channel in Complete --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 77eb82ea50e4..81dc964e2396 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -216,6 +216,7 @@ public void Complete() _completed = true; _outputWriter.Abort(); + _channel.Writer.TryComplete(); } } From c60d11367c0929ab638db8a8a8962ba73781b0cd Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 13:49:40 -0700 Subject: [PATCH 07/77] Move to flow control to the FrameWriter instead - We only want to throttle data frames --- .../src/Internal/Http2/Http2Connection.cs | 120 +------- .../src/Internal/Http2/Http2FrameWriter.cs | 283 ++++-------------- .../src/Internal/Http2/Http2OutputProducer.cs | 2 - 3 files changed, 54 insertions(+), 351 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 933eb2d8166b..bc68d911c7bd 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -75,10 +75,6 @@ internal partial class Http2Connection : IHttp2StreamLifetimeHandler, IHttpStrea internal readonly Dictionary _streams = new Dictionary(); internal PooledStreamStack StreamPool; - private readonly object _windowUpdateLock = new(); - private long _window; - private ManualResetValueTaskSource? _waitForMoreWindow; - public Http2Connection(HttpConnectionContext context) { var httpLimits = context.ServiceContext.ServerOptions.Limits; @@ -130,9 +126,7 @@ public Http2Connection(HttpConnectionContext context) _scheduleInline = context.ServiceContext.Scheduler == PipeScheduler.Inline; _inputTask = CopyPipeAsync(_context.Transport.Input, _input.Writer); - _outputTask = CopyOutputPipeAsync(_output.Reader, _context.Transport.Output); - - _window = _outputFlowControl.Available; + _outputTask = CopyPipeAsync(_output.Reader, _context.Transport.Output); } public string ConnectionId => _context.ConnectionId; @@ -375,12 +369,10 @@ public async Task ProcessRequestsAsync(IHttpApplication appl TimeoutControl.StartDrainTimeout(Limits.MinResponseDataRate, Limits.MaxResponseBufferSize); _frameWriter.Complete(); - AbortFlowControl(); } catch { _frameWriter.Abort(connectionError); - AbortFlowControl(); throw; } finally @@ -1008,7 +1000,7 @@ private Task ProcessWindowUpdateFrameAsync() if (_incomingFrame.StreamId == 0) { - if (!TryUpdateConnectionWindow(_incomingFrame.WindowUpdateSizeIncrement)) + if (!_frameWriter.TryUpdateConnectionWindow(_incomingFrame.WindowUpdateSizeIncrement)) { throw new Http2ConnectionErrorException(CoreStrings.Http2ErrorWindowUpdateSizeInvalid, Http2ErrorCode.FLOW_CONTROL_ERROR); } @@ -1702,114 +1694,6 @@ private PipeOptions GetOutputPipeOptions() useSynchronizationContext: false); } - internal (long, long) ConsumeWindow(long bytes) - { - lock (_windowUpdateLock) - { - var actual = Math.Min(bytes, _window); - var remaining = _window -= actual; - - if (remaining == 0) - { - // Reset the awaitable if we've drained the connection window, it means we can't write any more just yet - _waitForMoreWindow ??= new(); - _waitForMoreWindow.Reset(); - } - - return (actual, remaining); - } - } - - private bool TryUpdateConnectionWindow(int windowUpdateSizeIncrement) - { - ManualResetValueTaskSource? tcs = null; - - lock (_windowUpdateLock) - { - tcs = _waitForMoreWindow; - - _window += windowUpdateSizeIncrement; - } - - // Resume the connection copy loop - tcs?.TrySetResult(null); - return true; - } - - private void AbortFlowControl() - { - ManualResetValueTaskSource? tcs = null; - - lock (_windowUpdateLock) - { - tcs = _waitForMoreWindow; - } - - // Resume the connection copy loop - tcs?.TrySetResult(null); - } - - private async Task CopyOutputPipeAsync(PipeReader reader, PipeWriter writer) - { - Exception? error = null; - try - { - while (true) - { - var readResult = await reader.ReadAsync(); - - if ((readResult.IsCompleted && readResult.Buffer.Length == 0) || readResult.IsCanceled) - { - // FIN - break; - } - var buffer = readResult.Buffer; - - var (actual, remaining) = ConsumeWindow(buffer.Length); - - if (actual < buffer.Length) - { - buffer = buffer.Slice(0, actual); - } - - var outputBuffer = writer.GetMemory(_minAllocBufferSize); - - var copyAmount = (int)Math.Min(outputBuffer.Length, buffer.Length); - var bufferSlice = buffer.Slice(0, copyAmount); - - bufferSlice.CopyTo(outputBuffer.Span); - - writer.Advance(copyAmount); - - var result = await writer.FlushAsync(); - - reader.AdvanceTo(bufferSlice.End); - - if (result.IsCompleted || result.IsCanceled) - { - // flushResult should not be canceled. - break; - } - - if (remaining == 0) - { - // This shouldn't be null here - await new ValueTask(_waitForMoreWindow!, _waitForMoreWindow!.Version); - } - } - } - catch (Exception ex) - { - // Don't rethrow the exception. It should be handled by the Pipeline consumer. - error = ex; - } - finally - { - await reader.CompleteAsync(error); - await writer.CompleteAsync(error); - } - } - private async Task CopyPipeAsync(PipeReader reader, PipeWriter writer) { Exception? error = null; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 81dc964e2396..b3f16609fd25 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -47,6 +47,10 @@ internal class Http2FrameWriter private bool _completed; private bool _aborted; + private readonly object _windowUpdateLock = new(); + private long _window; + private ManualResetValueTaskSource? _waitForMoreWindow; + public Http2FrameWriter( PipeWriter outputPipeWriter, BaseConnectionContext connectionContext, @@ -76,6 +80,7 @@ public Http2FrameWriter( _hpackEncoder = new DynamicHPackEncoder(serviceContext.ServerOptions.AllowResponseHeaderCompression); _channel = Channel.CreateBounded(serviceContext.ServerOptions.Limits.Http2.MaxStreamsPerConnection); + _window = connectionOutputFlowControl.Available; _ = Task.Run(WriteToOutputPipe); } @@ -94,13 +99,17 @@ private async Task WriteToOutputPipe() var reader = producer.PipeReader; var stream = producer.Stream; - var hasData = reader.TryRead(out var readResult); + var hasResult = reader.TryRead(out var readResult); var buffer = readResult.Buffer; // This shouldn't be called when there's no data - Debug.Assert(hasData); + Debug.Assert(hasResult); + + // Check the stream window + var (actual, remainingStream) = producer.ConsumeWindow(buffer.Length); - var (actual, remaining) = producer.ConsumeWindow(buffer.Length); + // Now check the connection window + (actual, var remainingConnection) = ConsumeWindow(actual); // Write what we can if (actual < buffer.Length) @@ -155,7 +164,7 @@ private async Task WriteToOutputPipe() stream.DecrementActiveClientStreamCount(); } - flushResult = await WriteDataAsync(stream.StreamId, buffer, actual, endStream, producer.FirstWrite); + flushResult = await WriteDataAsync(stream.StreamId, buffer, buffer.Length, endStream, producer.FirstWrite); } producer.FirstWrite = false; @@ -172,11 +181,17 @@ private async Task WriteToOutputPipe() } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. - else if (hasModeData && remaining > 0) + else if (hasModeData && remainingStream > 0) { // Move this stream to the back of the queue so we're being fair to the other streams that have data Schedule(producer); } + + if (remainingConnection == 0) + { + // This shouldn't be null here + await new ValueTask(_waitForMoreWindow!, _waitForMoreWindow!.Version); + } } catch (Exception ex) { @@ -217,6 +232,7 @@ public void Complete() _completed = true; _outputWriter.Abort(); _channel.Writer.TryComplete(); + AbortFlowControl(); } } @@ -231,6 +247,7 @@ public void Abort(ConnectionAbortedException error) _aborted = true; _connectionContext.Abort(error); + AbortFlowControl(); Complete(); } @@ -369,41 +386,6 @@ private void FinishWritingHeaders(int streamId, int payloadLength, bool done) } } - private ValueTask WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, in ReadOnlySequence data, bool endStream, bool firstWrite, bool forceFlush) - { - // Logic in this method is replicated in WriteDataAndTrailersAsync. - // Changes here may need to be mirrored in WriteDataAndTrailersAsync. - - // The Length property of a ReadOnlySequence can be expensive, so we cache the value. - var dataLength = data.Length; - - lock (_writeLock) - { - if (_completed || flowControl.IsAborted) - { - return default; - } - - // Zero-length data frames are allowed to be sent immediately even if there is no space available in the flow control window. - // https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.1 - if (dataLength != 0 && dataLength > flowControl.Available) - { - return WriteDataAsync(streamId, flowControl, data, dataLength, endStream, firstWrite); - } - - // This cast is safe since if dataLength would overflow an int, it's guaranteed to be greater than the available flow control window. - flowControl.Advance((int)dataLength); - WriteDataUnsynchronized(streamId, data, dataLength, endStream); - - if (forceFlush) - { - return TimeFlushUnsynchronizedAsync(); - } - - return default; - } - } - public ValueTask WriteDataAndTrailersAsync(int streamId, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) { // This method combines WriteDataAsync and WriteResponseTrailers. @@ -419,63 +401,10 @@ public ValueTask WriteDataAndTrailersAsync(int streamId, in ReadOnl return default; } - //// Zero-length data frames are allowed to be sent immediately even if there is no space available in the flow control window. - //// https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.1 - //if (dataLength != 0 && dataLength > flowControl.Available) - //{ - // return WriteDataAndTrailersAsyncCore(this, streamId, flowControl, data, dataLength, firstWrite, headers); - //} - - // This cast is safe since if dataLength would overflow an int, it's guaranteed to be greater than the available flow control window. - // flowControl.Advance((int)dataLength); - WriteDataUnsynchronized(streamId, data, dataLength, endStream: false); - - return WriteResponseTrailersAsync(streamId, headers); - } - - //static async ValueTask WriteDataAndTrailersAsyncCore(Http2FrameWriter writer, int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool firstWrite, HttpResponseTrailers headers) - //{ - // await writer.WriteDataAsync(streamId, flowControl, data, dataLength, endStream: false, firstWrite); - - // return await writer.WriteResponseTrailersAsync(streamId, headers); - //} - } - - public ValueTask WriteDataAndTrailersAsync(int streamId, StreamOutputFlowControl flowControl, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) - { - // This method combines WriteDataAsync and WriteResponseTrailers. - // Changes here may need to be mirrored in WriteDataAsync. - - // The Length property of a ReadOnlySequence can be expensive, so we cache the value. - var dataLength = data.Length; - - lock (_writeLock) - { - if (_completed || flowControl.IsAborted) - { - return default; - } - - // Zero-length data frames are allowed to be sent immediately even if there is no space available in the flow control window. - // https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.1 - if (dataLength != 0 && dataLength > flowControl.Available) - { - return WriteDataAndTrailersAsyncCore(this, streamId, flowControl, data, dataLength, firstWrite, headers); - } - - // This cast is safe since if dataLength would overflow an int, it's guaranteed to be greater than the available flow control window. - flowControl.Advance((int)dataLength); WriteDataUnsynchronized(streamId, data, dataLength, endStream: false); return WriteResponseTrailersAsync(streamId, headers); } - - static async ValueTask WriteDataAndTrailersAsyncCore(Http2FrameWriter writer, int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool firstWrite, HttpResponseTrailers headers) - { - await writer.WriteDataAsync(streamId, flowControl, data, dataLength, endStream: false, firstWrite); - - return await writer.WriteResponseTrailersAsync(streamId, headers); - } } /* Padding is not implemented @@ -572,39 +501,8 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen return flushResult; } - // var shouldFlush = false; - WriteDataUnsynchronized(streamId, data, dataLength, endStream); - //if (actual > 0) - //{ - // if (actual < dataLength) - // { - // WriteDataUnsynchronized(streamId, data.Slice(0, actual), actual, endStream: false); - // data = data.Slice(actual); - // dataLength -= actual; - // } - // else - // { - - // } - - // // Don't call FlushAsync() with the min data rate, since we time this write while also accounting for - // // flow control induced backpressure below. - // shouldFlush = true; - //} - //else if (firstWrite) - //{ - // // If we're facing flow control induced backpressure on the first write for a given stream's response body, - // // we make sure to flush the response headers immediately. - // shouldFlush = true; - //} - - //if (shouldFlush) - //{ - - //} - if (_minResponseDataRate != null) { // Call BytesWrittenToBuffer before FlushAsync() to make testing easier, otherwise the Flush can cause test code to run before the timeout @@ -615,8 +513,6 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen _unflushedBytes = 0; writeTask = _flusher.FlushAsync(); - - //firstWrite = false; } // REVIEW: This will include flow control waiting as of right now. We need to handle that elsewhere. @@ -635,104 +531,6 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen return flushResult; } - private async ValueTask WriteDataAsync(int streamId, StreamOutputFlowControl flowControl, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) - { - FlushResult flushResult = default; - - while (dataLength > 0) - { - ValueTask availabilityTask; - var writeTask = default(ValueTask); - - lock (_writeLock) - { - if (_completed || flowControl.IsAborted) - { - break; - } - - // Observe HTTP/2 backpressure - var actual = flowControl.AdvanceUpToAndWait(dataLength, out availabilityTask); - - var shouldFlush = false; - - if (actual > 0) - { - if (actual < dataLength) - { - WriteDataUnsynchronized(streamId, data.Slice(0, actual), actual, endStream: false); - data = data.Slice(actual); - dataLength -= actual; - } - else - { - WriteDataUnsynchronized(streamId, data, actual, endStream); - dataLength = 0; - } - - // Don't call FlushAsync() with the min data rate, since we time this write while also accounting for - // flow control induced backpressure below. - shouldFlush = true; - } - else if (firstWrite) - { - // If we're facing flow control induced backpressure on the first write for a given stream's response body, - // we make sure to flush the response headers immediately. - shouldFlush = true; - } - - if (shouldFlush) - { - if (_minResponseDataRate != null) - { - // Call BytesWrittenToBuffer before FlushAsync() to make testing easier, otherwise the Flush can cause test code to run before the timeout - // control updates and if the test checks for a timeout it can fail - _timeoutControl.BytesWrittenToBuffer(_minResponseDataRate, _unflushedBytes); - } - - _unflushedBytes = 0; - - writeTask = _flusher.FlushAsync(); - } - - firstWrite = false; - } - - // Avoid timing writes that are already complete. This is likely to happen during the last iteration. - if (availabilityTask.IsCompleted && writeTask.IsCompleted) - { - continue; - } - - if (_minResponseDataRate != null) - { - _timeoutControl.StartTimingWrite(); - } - - // This awaitable releases continuations in FIFO order when the window updates. - // It should be very rare for a continuation to run without any availability. - if (!availabilityTask.IsCompleted) - { - await availabilityTask; - } - - flushResult = await writeTask; - - if (_minResponseDataRate != null) - { - _timeoutControl.StopTimingWrite(); - } - } - - if (!_scheduleInline) - { - // Ensure that the application continuation isn't executed inline by ProcessWindowUpdateFrameAsync. - await ThreadPoolAwaitable.Instance; - } - - return flushResult; - } - /* https://tools.ietf.org/html/rfc7540#section-6.9 +-+-------------------------------------------------------------+ |R| Window Size Increment (31) | @@ -939,27 +737,50 @@ private ValueTask TimeFlushUnsynchronizedAsync() return _flusher.FlushAsync(_minResponseDataRate, bytesWritten); } - public bool TryUpdateConnectionWindow(int bytes) + internal (long, long) ConsumeWindow(long bytes) { - lock (_writeLock) + lock (_windowUpdateLock) { - return _connectionOutputFlowControl.TryUpdateWindow(bytes); + var actual = Math.Min(bytes, _window); + var remaining = _window -= actual; + + if (remaining == 0) + { + // Reset the awaitable if we've drained the connection window, it means we can't write any more just yet + _waitForMoreWindow ??= new(); + _waitForMoreWindow.Reset(); + } + + return (actual, remaining); } } - public bool TryUpdateStreamWindow(StreamOutputFlowControl flowControl, int bytes) + private void AbortFlowControl() { + ManualResetValueTaskSource? tcs = null; + lock (_writeLock) { - return flowControl.TryUpdateWindow(bytes); + tcs = _waitForMoreWindow; } + + // Resume the connection copy loop + tcs?.TrySetResult(null); } - public void AbortPendingStreamDataWrites(StreamOutputFlowControl flowControl) + public bool TryUpdateConnectionWindow(int bytes) { + ManualResetValueTaskSource? tcs = null; + lock (_writeLock) { - flowControl.Abort(); + tcs = _waitForMoreWindow; + + _window += bytes; } + + // Resume the connection copy loop + tcs?.TrySetResult(null); + return true; } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 9c2273226113..8ab7eae64b6d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -11,7 +11,6 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.FlowControl; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure.PipeWriterHelpers; -using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; @@ -150,7 +149,6 @@ private bool UpdateWindow(long bytes) } } - public void StreamReset() { // Response should have been completed. From b14cf57691bab4ac8086e7de0d5491adf891932c Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 13:54:10 -0700 Subject: [PATCH 08/77] Use correct lock, add logs --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index b3f16609fd25..a53138562279 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -121,8 +121,7 @@ private async Task WriteToOutputPipe() if (readResult.IsCanceled) { - // Response body is aborted, break and complete reader. - // break; + // Response body is aborted, complete reader for this output producer. } else if (readResult.IsCompleted && stream.ResponseTrailers?.Count > 0) { @@ -198,6 +197,8 @@ private async Task WriteToOutputPipe() _log.LogCritical(ex, "The event loop in connection {ConnectionId} failed unexpectedly", _connectionId); } } + + _log.LogDebug("The connection processing loop for {ConnectionId} ended gracefully", _connectionId); } public void UpdateMaxHeaderTableSize(uint maxHeaderTableSize) @@ -759,7 +760,7 @@ private void AbortFlowControl() { ManualResetValueTaskSource? tcs = null; - lock (_writeLock) + lock (_windowUpdateLock) { tcs = _waitForMoreWindow; } @@ -772,7 +773,7 @@ public bool TryUpdateConnectionWindow(int bytes) { ManualResetValueTaskSource? tcs = null; - lock (_writeLock) + lock (_windowUpdateLock) { tcs = _waitForMoreWindow; From 8a59b464a7bebbb5cd21f6cd65d7fba24df1c69a Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 13:56:21 -0700 Subject: [PATCH 09/77] Abort flow control in a single place --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index a53138562279..13467bd9f11a 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -231,9 +231,9 @@ public void Complete() } _completed = true; + AbortFlowControl(); _outputWriter.Abort(); _channel.Writer.TryComplete(); - AbortFlowControl(); } } @@ -248,7 +248,6 @@ public void Abort(ConnectionAbortedException error) _aborted = true; _connectionContext.Abort(error); - AbortFlowControl(); Complete(); } From 01905ee986c0181c6022c7fbf92c4bc0cfe3621a Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 14:02:18 -0700 Subject: [PATCH 10/77] Remove dead code --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index 29d6b94d5a30..80c6e11a824a 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -507,7 +507,6 @@ public void OnDataRead(int bytesRead) public bool TryUpdateOutputWindow(int bytes) { return _http2Output.TryUpdateStreamWindow(bytes); - // return _context.FrameWriter.TryUpdateStreamWindow(_outputFlowControl, bytes); } public void AbortRstStreamReceived() From 30ef07d0f32d1720cd1d1ee8cb76be1f62db226d Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 14:10:14 -0700 Subject: [PATCH 11/77] Undo chanes --- .../Kestrel/Core/src/Internal/Http2/Http2Connection.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index bc68d911c7bd..5f05596a101e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -1708,12 +1708,11 @@ private async Task CopyPipeAsync(PipeReader reader, PipeWriter writer) // FIN break; } - var buffer = readResult.Buffer; var outputBuffer = writer.GetMemory(_minAllocBufferSize); - var copyAmount = (int)Math.Min(outputBuffer.Length, buffer.Length); - var bufferSlice = buffer.Slice(0, copyAmount); + var copyAmount = (int)Math.Min(outputBuffer.Length, readResult.Buffer.Length); + var bufferSlice = readResult.Buffer.Slice(0, copyAmount); bufferSlice.CopyTo(outputBuffer.Span); From 3c3d4f82ad5d66bc29e81034259d91892bf8e19f Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 14:23:56 -0700 Subject: [PATCH 12/77] Remove call --- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 8ab7eae64b6d..26b7451d5822 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -512,8 +512,6 @@ public void Stop() // We need to make sure the cancellation is observed by the code _frameWriter.Schedule(this); } - - // _frameWriter.AbortPendingStreamDataWrites(_flowControl); } } From 086c627f5664df3b2df5cad7be098401568f9e86 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 22:08:13 -0700 Subject: [PATCH 13/77] Made a few tests pass --- .../src/Internal/Http2/Http2FrameWriter.cs | 77 +++++++++++-------- .../src/Internal/Http2/Http2OutputProducer.cs | 21 +++-- .../Core/src/Internal/Http2/Http2Stream.cs | 2 +- 3 files changed, 61 insertions(+), 39 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 13467bd9f11a..560d210d925f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -49,7 +49,7 @@ internal class Http2FrameWriter private readonly object _windowUpdateLock = new(); private long _window; - private ManualResetValueTaskSource? _waitForMoreWindow; + private readonly Queue _waitingForMoreWindow = new(); public Http2FrameWriter( PipeWriter outputPipeWriter, @@ -180,16 +180,19 @@ private async Task WriteToOutputPipe() } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. - else if (hasModeData && remainingStream > 0) + else if (hasModeData) { - // Move this stream to the back of the queue so we're being fair to the other streams that have data - Schedule(producer); - } - - if (remainingConnection == 0) - { - // This shouldn't be null here - await new ValueTask(_waitForMoreWindow!, _waitForMoreWindow!.Version); + // We have no more connection window, put this producer in a queue waiting for it to + // a window update to resume the connection. + if (remainingConnection == 0) + { + _waitingForMoreWindow.Enqueue(producer); + } + else if (remainingStream > 0) + { + // Move this stream to the back of the queue so we're being fair to the other streams that have data + Schedule(producer); + } } } catch (Exception ex) @@ -496,12 +499,23 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen lock (_writeLock) { + var shouldFlush = false; + if (_completed) { return flushResult; } - WriteDataUnsynchronized(streamId, data, dataLength, endStream); + if (dataLength > 0 || endStream) + { + WriteDataUnsynchronized(streamId, data, dataLength, endStream); + + shouldFlush = true; + } + else if (firstWrite) + { + shouldFlush = true; + } if (_minResponseDataRate != null) { @@ -510,9 +524,12 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen _timeoutControl.BytesWrittenToBuffer(_minResponseDataRate, _unflushedBytes); } - _unflushedBytes = 0; + if (shouldFlush) + { + _unflushedBytes = 0; - writeTask = _flusher.FlushAsync(); + writeTask = _flusher.FlushAsync(); + } } // REVIEW: This will include flow control waiting as of right now. We need to handle that elsewhere. @@ -744,43 +761,39 @@ private ValueTask TimeFlushUnsynchronizedAsync() var actual = Math.Min(bytes, _window); var remaining = _window -= actual; - if (remaining == 0) - { - // Reset the awaitable if we've drained the connection window, it means we can't write any more just yet - _waitForMoreWindow ??= new(); - _waitForMoreWindow.Reset(); - } - return (actual, remaining); } } private void AbortFlowControl() { - ManualResetValueTaskSource? tcs = null; - lock (_windowUpdateLock) { - tcs = _waitForMoreWindow; + while (_waitingForMoreWindow.TryDequeue(out var producer)) + { + Schedule(producer); + } } - - // Resume the connection copy loop - tcs?.TrySetResult(null); } public bool TryUpdateConnectionWindow(int bytes) { - ManualResetValueTaskSource? tcs = null; - lock (_windowUpdateLock) { - tcs = _waitForMoreWindow; + var maxUpdate = Http2PeerSettings.MaxWindowSize - _window; + + if (bytes > maxUpdate) + { + return false; + } _window += bytes; - } - // Resume the connection copy loop - tcs?.TrySetResult(null); + while (_waitingForMoreWindow.TryDequeue(out var producer)) + { + Schedule(producer); + } + } return true; } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 26b7451d5822..77651731d2a1 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -74,7 +74,7 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, Strea // The minimum output data rate is enforced at the connection level by Http2FrameWriter. _flusher = new TimingPipeFlusher(timeoutControl: null, _log); _flusher.Initialize(_pipeWriter); - _window = flowControl.Available; + _window = context.ClientPeerSettings.InitialWindowSize; } public Http2Stream Stream => _stream; @@ -141,15 +141,15 @@ private bool UpdateWindow(long bytes) { lock (_dataWriterLock) { - var wasEmpty = _window == 0; + var wasDepleted = _window <= 0; _window += bytes; - return wasEmpty && _unconsumedBytes > 0; + return wasDepleted && _window > 0 && _unconsumedBytes > 0; } } - public void StreamReset() + public void StreamReset(uint initialWindowSize) { // Response should have been completed. Debug.Assert(_responseCompleteTaskSource.GetStatus(_responseCompleteTaskSource.Version) == ValueTaskSourceStatus.Succeeded); @@ -163,7 +163,7 @@ public void StreamReset() _pipeWriter.Reset(); _responseCompleteTaskSource.Reset(); - _window = _flowControl.Available; + _window = initialWindowSize; _unconsumedBytes = 0; _enqueuedForObservation = false; } @@ -568,9 +568,18 @@ internal Memory GetFakeMemory(int minSize) } } - // REVIEW: When does this fail? When the connection is aborted? public bool TryUpdateStreamWindow(int bytes) { + lock (_dataWriterLock) + { + var maxUpdate = Http2PeerSettings.MaxWindowSize - _window; + + if (bytes > maxUpdate) + { + return false; + } + } + if (UpdateWindow(bytes)) { _frameWriter.Schedule(this); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index 80c6e11a824a..ad8fdb1797b0 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -70,7 +70,7 @@ public void Initialize(Http2StreamContext context) { _inputFlowControl.Reset(); _outputFlowControl.Reset(context.ClientPeerSettings.InitialWindowSize); - _http2Output.StreamReset(); + _http2Output.StreamReset(context.ClientPeerSettings.InitialWindowSize); RequestBodyPipe.Reset(); } } From c72680be118350d6d18cd0278478ac0738b5546a Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 22:52:08 -0700 Subject: [PATCH 14/77] "Fixed" more tests --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 6 +++++- .../Http2/Http2ConnectionTests.cs | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 560d210d925f..d8ad9aa4ec8a 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -186,7 +186,10 @@ private async Task WriteToOutputPipe() // a window update to resume the connection. if (remainingConnection == 0) { - _waitingForMoreWindow.Enqueue(producer); + lock (_windowUpdateLock) + { + _waitingForMoreWindow.Enqueue(producer); + } } else if (remainingStream > 0) { @@ -769,6 +772,7 @@ private void AbortFlowControl() { lock (_windowUpdateLock) { + // REVIEW: What should this be doing? while (_waitingForMoreWindow.TryDequeue(out var producer)) { Schedule(producer); diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index 2f517f239dde..dc0d764a31d3 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -4720,6 +4720,10 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 1); + // Kick the input loop so it sees that we have no more active streams. This is a bug in the graceful shutdown logic + // but since we wake up the loop pretty often it doesn't show up in practice + _pair.Transport.Input.CancelPendingRead(); + await _closedStateReached.Task.DefaultTimeout(); VerifyGoAway(await ReceiveFrameAsync(), 1, Http2ErrorCode.NO_ERROR); } @@ -4740,7 +4744,7 @@ public async Task AcceptNewStreamsDuringClosingConnection() await StartStreamAsync(3, _browserRequestHeaders, endStream: false); await SendDataAsync(1, _helloBytes, true); - var f = await ExpectAsync(Http2FrameType.HEADERS, + await ExpectAsync(Http2FrameType.HEADERS, withLength: 32, withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS, withStreamId: 1); @@ -4766,6 +4770,10 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 3); + // Kick the input loop so it sees that we have no more active streams. This is a bug in the graceful shutdown logic + // but since we wake up the loop pretty often it doesn't show up in practice + _pair.Transport.Input.CancelPendingRead(); + await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); } From 6257f2662e5728c79c3e2b75e512c3b8afd25945 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 23 Mar 2022 23:17:41 -0700 Subject: [PATCH 15/77] Put back some timeouts --- src/Shared/Http2cat/Http2Utilities.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Shared/Http2cat/Http2Utilities.cs b/src/Shared/Http2cat/Http2Utilities.cs index 6b73ea63c931..c5479147e1ed 100644 --- a/src/Shared/Http2cat/Http2Utilities.cs +++ b/src/Shared/Http2cat/Http2Utilities.cs @@ -393,7 +393,7 @@ public Task SendAsync(ReadOnlySpan span) public static async Task FlushAsync(PipeWriter writableBuffer) { - await writableBuffer.FlushAsync().AsTask(); + await writableBuffer.FlushAsync().AsTask().DefaultTimeout(); } public Task SendPreambleAsync() => SendAsync(ClientPreface); @@ -827,7 +827,7 @@ internal async Task ReceiveFrameAsync(uint maxFrameSize = while (true) { - var result = await _pair.Application.Input.ReadAsync().AsTask();//.DefaultTimeout(); + var result = await _pair.Application.Input.ReadAsync().AsTask().DefaultTimeout(); var buffer = result.Buffer; var consumed = buffer.Start; var examined = buffer.Start; From 82fe5cd7651456f2ac4099eba9f63bc4b464a104 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Fri, 25 Mar 2022 14:34:39 -0700 Subject: [PATCH 16/77] Made more test fixes - Allow scheduling the output for aborts and completions if we're waiting for window updates. --- .../src/Internal/Http2/Http2FrameWriter.cs | 23 +++++++-- .../src/Internal/Http2/Http2OutputProducer.cs | 47 +++++++++++++++---- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index d8ad9aa4ec8a..f7478dea05eb 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -79,7 +79,12 @@ public Http2FrameWriter( _scheduleInline = serviceContext.Scheduler == PipeScheduler.Inline; _hpackEncoder = new DynamicHPackEncoder(serviceContext.ServerOptions.AllowResponseHeaderCompression); - _channel = Channel.CreateBounded(serviceContext.ServerOptions.Limits.Http2.MaxStreamsPerConnection); + + _channel = Channel.CreateBounded(new BoundedChannelOptions(serviceContext.ServerOptions.Limits.Http2.MaxStreamsPerConnection) + { + AllowSynchronousContinuations = _scheduleInline + }); + _window = connectionOutputFlowControl.Available; _ = Task.Run(WriteToOutputPipe); @@ -186,6 +191,8 @@ private async Task WriteToOutputPipe() // a window update to resume the connection. if (remainingConnection == 0) { + producer.MarkWaitingForWindowUpdates(); + lock (_windowUpdateLock) { _waitingForMoreWindow.Enqueue(producer); @@ -196,6 +203,10 @@ private async Task WriteToOutputPipe() // Move this stream to the back of the queue so we're being fair to the other streams that have data Schedule(producer); } + else + { + producer.MarkWaitingForWindowUpdates(); + } } } catch (Exception ex) @@ -775,7 +786,10 @@ private void AbortFlowControl() // REVIEW: What should this be doing? while (_waitingForMoreWindow.TryDequeue(out var producer)) { - Schedule(producer); + if (!producer.StreamCompleted) + { + Schedule(producer); + } } } } @@ -795,7 +809,10 @@ public bool TryUpdateConnectionWindow(int bytes) while (_waitingForMoreWindow.TryDequeue(out var producer)) { - Schedule(producer); + if (!producer.StreamCompleted) + { + Schedule(producer); + } } } return true; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 77651731d2a1..ccf27f83d6a3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -45,6 +45,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state bool _enqueuedForObservation; + bool _waitingForWindowUpdates; /// The core logic for the IValueTaskSource implementation. private ManualResetValueTaskSourceCore _responseCompleteTaskSource = new ManualResetValueTaskSourceCore { RunContinuationsAsynchronously = true }; // mutable struct, do not make this readonly @@ -84,6 +85,17 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, Strea public bool StreamEnded => _streamEnded; + public bool StreamCompleted + { + get + { + lock (_dataWriterLock) + { + return _streamCompleted; + } + } + } + // Added bytes to the queue. // Returns a bool that represents whether we should schedule this producer to write // the enqueued bytes @@ -105,7 +117,7 @@ private bool EnqueueForObservation() { var wasEnqueuedForObservation = _enqueuedForObservation; _enqueuedForObservation = true; - return _unconsumedBytes == 0 && !wasEnqueuedForObservation; + return (_unconsumedBytes == 0 || _waitingForWindowUpdates) && !wasEnqueuedForObservation; } } @@ -189,7 +201,7 @@ public void Complete() if (enqueue) { - _frameWriter.Schedule(this); + Schedule(); } } @@ -245,7 +257,7 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) if (enqueue) { - _frameWriter.Schedule(this); + Schedule(); } return task; @@ -259,6 +271,25 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) } } + public void MarkWaitingForWindowUpdates() + { + lock (_dataWriterLock) + { + _waitingForWindowUpdates = true; + } + } + + private void Schedule() + { + lock (_dataWriterLock) + { + // Lock here + _waitingForWindowUpdates = false; + } + + _frameWriter.Schedule(this); + } + public ValueTask Write100ContinueAsync() { lock (_dataWriterLock) @@ -336,7 +367,7 @@ public Task WriteDataAsync(ReadOnlySpan data, CancellationToken cancellati if (enqueue) { - _frameWriter.Schedule(this); + Schedule(); } return task; @@ -360,7 +391,7 @@ public ValueTask WriteStreamSuffixAsync() if (enqueue) { - _frameWriter.Schedule(this); + Schedule(); } return GetWaiterTask(); @@ -465,7 +496,7 @@ public ValueTask WriteDataToPipeAsync(ReadOnlySpan data, Canc if (enqueue) { - _frameWriter.Schedule(this); + Schedule(); } return task; @@ -510,7 +541,7 @@ public void Stop() if (enqueue) { // We need to make sure the cancellation is observed by the code - _frameWriter.Schedule(this); + Schedule(); } } } @@ -582,7 +613,7 @@ public bool TryUpdateStreamWindow(int bytes) if (UpdateWindow(bytes)) { - _frameWriter.Schedule(this); + Schedule(); } return true; From 59b51e4adf45359830cc73d94fedb32cab7f1a1c Mon Sep 17 00:00:00 2001 From: David Fowler Date: Fri, 25 Mar 2022 15:49:47 -0700 Subject: [PATCH 17/77] Fix how we stop streams when flow control is being aborted --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 11 ++++++++--- .../Core/src/Internal/Http2/Http2OutputProducer.cs | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index f7478dea05eb..0eca63497406 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -191,7 +191,8 @@ private async Task WriteToOutputPipe() // a window update to resume the connection. if (remainingConnection == 0) { - producer.MarkWaitingForWindowUpdates(); + // Mark the output as waiting for a window upate to resume writing (there's still data) + producer.MarkWaitingForWindowUpdates(true); lock (_windowUpdateLock) { @@ -205,7 +206,8 @@ private async Task WriteToOutputPipe() } else { - producer.MarkWaitingForWindowUpdates(); + // Mark the output as waiting for a window upate to resume writing (there's still data) + producer.MarkWaitingForWindowUpdates(true); } } } @@ -788,7 +790,8 @@ private void AbortFlowControl() { if (!producer.StreamCompleted) { - Schedule(producer); + // Stop the output + producer.Stop(); } } } @@ -811,6 +814,8 @@ public bool TryUpdateConnectionWindow(int bytes) { if (!producer.StreamCompleted) { + // We're no longer waiting for the update + producer.MarkWaitingForWindowUpdates(false); Schedule(producer); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index ccf27f83d6a3..cf4263efb95f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -271,12 +271,12 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) } } - public void MarkWaitingForWindowUpdates() + public void MarkWaitingForWindowUpdates(bool waitingForUpdates) { lock (_dataWriterLock) { - _waitingForWindowUpdates = true; - } + _waitingForWindowUpdates = waitingForUpdates; + } } private void Schedule() From da543962fba8a5714b1e950741762ccb9032c366 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Fri, 25 Mar 2022 16:17:40 -0700 Subject: [PATCH 18/77] Always clean up the pipe writer --- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index cf4263efb95f..e8023354e4de 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -204,6 +204,11 @@ public void Complete() Schedule(); } } + else + { + // Make sure the writing side is completed. + _pipeWriter.Complete(); + } if (_fakeMemoryOwner != null) { @@ -276,7 +281,7 @@ public void MarkWaitingForWindowUpdates(bool waitingForUpdates) lock (_dataWriterLock) { _waitingForWindowUpdates = waitingForUpdates; - } + } } private void Schedule() From 8f841d4a93d8c06105854647d32f22216fe19043 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Fri, 25 Mar 2022 16:41:42 -0700 Subject: [PATCH 19/77] Fixed more tests --- .../Http2/Http2ConnectionTests.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index dc0d764a31d3..0e63a4e43cbd 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -3804,7 +3804,11 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 3); + // Force input loop to check connection state + TriggerTick(); + await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); + await _closedStateReached.Task.DefaultTimeout(); } @@ -4168,7 +4172,7 @@ await InitializeConnectionAsync(async context => for (var i = 0; i < expectedFullFrameCountBeforeBackpressure; i++) { await expectingDataSem.WaitAsync(); - Assert.True(context.Response.Body.WriteAsync(_maxData, 0, _maxData.Length).IsCompleted); + await context.Response.Body.WriteAsync(_maxData, 0, _maxData.Length); } await expectingDataSem.WaitAsync(); @@ -4722,7 +4726,7 @@ await ExpectAsync(Http2FrameType.DATA, // Kick the input loop so it sees that we have no more active streams. This is a bug in the graceful shutdown logic // but since we wake up the loop pretty often it doesn't show up in practice - _pair.Transport.Input.CancelPendingRead(); + TriggerTick(); await _closedStateReached.Task.DefaultTimeout(); VerifyGoAway(await ReceiveFrameAsync(), 1, Http2ErrorCode.NO_ERROR); @@ -4772,7 +4776,7 @@ await ExpectAsync(Http2FrameType.DATA, // Kick the input loop so it sees that we have no more active streams. This is a bug in the graceful shutdown logic // but since we wake up the loop pretty often it doesn't show up in practice - _pair.Transport.Input.CancelPendingRead(); + TriggerTick(); await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); } From 645479a39b019128a4b4563dd137c22810b890f6 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 07:58:57 -0700 Subject: [PATCH 20/77] Make inline scheduling work better - Schedule before flushing so we can take advantage of the pipe scheduler --- .../src/Internal/Http2/Http2FrameWriter.cs | 5 +---- .../src/Internal/Http2/Http2OutputProducer.cs | 20 +++++++++---------- .../Http2/Http2ConnectionTests.cs | 14 +------------ 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 0eca63497406..e2efd4b25755 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -104,12 +104,9 @@ private async Task WriteToOutputPipe() var reader = producer.PipeReader; var stream = producer.Stream; - var hasResult = reader.TryRead(out var readResult); + var readResult = await reader.ReadAsync(); var buffer = readResult.Buffer; - // This shouldn't be called when there's no data - Debug.Assert(hasResult); - // Check the stream window var (actual, remainingStream) = producer.ConsumeWindow(buffer.Length); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index e8023354e4de..c0d4d46a1bdf 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -40,8 +40,9 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV // Internal for testing internal bool _disposed; - internal long _unconsumedBytes; - internal long _window; + + private long _unconsumedBytes; + private long _window; // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state bool _enqueuedForObservation; @@ -178,6 +179,7 @@ public void StreamReset(uint initialWindowSize) _window = initialWindowSize; _unconsumedBytes = 0; _enqueuedForObservation = false; + _waitingForWindowUpdates = false; } public void Complete() @@ -256,16 +258,15 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) Debug.Assert(_pipeWriter.CanGetUnflushedBytes); var enqueue = Enqueue(_pipeWriter.UnflushedBytes); - // If there's already been response data written to the stream, just wait for that. Any header - // should be in front of the data frames in the connection pipe. Trailers could change things. - var task = _flusher.FlushAsync(this, cancellationToken); if (enqueue) { Schedule(); } - return task; + // If there's already been response data written to the stream, just wait for that. Any header + // should be in front of the data frames in the connection pipe. Trailers could change things. + return _flusher.FlushAsync(this, cancellationToken); } else { @@ -368,14 +369,13 @@ public Task WriteDataAsync(ReadOnlySpan data, CancellationToken cancellati _pipeWriter.Write(data); var enqueue = Enqueue(data.Length); - var task = _flusher.FlushAsync(this, cancellationToken).GetAsTask(); if (enqueue) { Schedule(); } - return task; + return _flusher.FlushAsync(this, cancellationToken).GetAsTask(); } } @@ -497,14 +497,13 @@ public ValueTask WriteDataToPipeAsync(ReadOnlySpan data, Canc _pipeWriter.Write(data); var enqueue = Enqueue(data.Length); - var task = _flusher.FlushAsync(this, cancellationToken); if (enqueue) { Schedule(); } - return task; + return _flusher.FlushAsync(this, cancellationToken); } } @@ -669,6 +668,5 @@ public void Dispose() return; } _disposed = true; - } } diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index 0e63a4e43cbd..f5d06e04c2c9 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -577,7 +577,6 @@ await InitializeConnectionAsync(async context => var output = (Http2OutputProducer)stream.Output; Assert.True(output._disposed); - // await output._dataWriteProcessingTask.DefaultTimeout(); // Stream is not returned to the pool Assert.Equal(0, _connection.StreamPool.Count); @@ -3804,9 +3803,6 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 3); - // Force input loop to check connection state - TriggerTick(); - await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); await _closedStateReached.Task.DefaultTimeout(); @@ -4172,7 +4168,7 @@ await InitializeConnectionAsync(async context => for (var i = 0; i < expectedFullFrameCountBeforeBackpressure; i++) { await expectingDataSem.WaitAsync(); - await context.Response.Body.WriteAsync(_maxData, 0, _maxData.Length); + Assert.True(context.Response.Body.WriteAsync(_maxData, 0, _maxData.Length).IsCompleted); } await expectingDataSem.WaitAsync(); @@ -4724,10 +4720,6 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 1); - // Kick the input loop so it sees that we have no more active streams. This is a bug in the graceful shutdown logic - // but since we wake up the loop pretty often it doesn't show up in practice - TriggerTick(); - await _closedStateReached.Task.DefaultTimeout(); VerifyGoAway(await ReceiveFrameAsync(), 1, Http2ErrorCode.NO_ERROR); } @@ -4774,10 +4766,6 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 3); - // Kick the input loop so it sees that we have no more active streams. This is a bug in the graceful shutdown logic - // but since we wake up the loop pretty often it doesn't show up in practice - TriggerTick(); - await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); } From 033c54db22de562682a08d87301b4861d155906b Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 08:23:47 -0700 Subject: [PATCH 21/77] Enqueue unflushed bytes before setting observation flag --- .../src/Internal/Http2/Http2OutputProducer.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index c0d4d46a1bdf..270e0583688e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -197,14 +197,15 @@ public void Complete() if (!_streamCompleted) { - var enqueue = EnqueueForObservation(); - // Make sure the writing side is completed. - _pipeWriter.Complete(); + var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); if (enqueue) { Schedule(); } + + // Make sure the writing side is completed. + _pipeWriter.Complete(); } else { @@ -391,14 +392,16 @@ public ValueTask WriteStreamSuffixAsync() _streamCompleted = true; _suffixSent = true; - var enqueue = EnqueueForObservation(); - _pipeWriter.Complete(); + // Try to enqueue any unflushed bytes + var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); if (enqueue) { Schedule(); } + _pipeWriter.Complete(); + return GetWaiterTask(); } } @@ -538,15 +541,15 @@ public void Stop() _streamCompleted = true; - var enqueue = EnqueueForObservation(); - - _pipeReader.CancelPendingRead(); + var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); if (enqueue) { // We need to make sure the cancellation is observed by the code Schedule(); } + + _pipeReader.CancelPendingRead(); } } From 78a1ea96ff2eaebf9ac50913033fa0f15dd55830 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 11:58:31 -0700 Subject: [PATCH 22/77] Moar fixes --- .../src/Internal/Http2/Http2Connection.cs | 41 +------------------ .../src/Internal/Http2/Http2FrameWriter.cs | 18 ++++---- .../src/Internal/Http2/Http2OutputProducer.cs | 18 ++++---- 3 files changed, 22 insertions(+), 55 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 5f05596a101e..1178a0b402e3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -36,9 +36,7 @@ internal partial class Http2Connection : IHttp2StreamLifetimeHandler, IHttpStrea private readonly HttpConnectionContext _context; private readonly Http2FrameWriter _frameWriter; private readonly Pipe _input; - private readonly Pipe _output; private readonly Task _inputTask; - private readonly Task _outputTask; private readonly int _minAllocBufferSize; private readonly HPackDecoder _hpackDecoder; private readonly InputFlowControl _inputFlowControl; @@ -86,10 +84,9 @@ public Http2Connection(HttpConnectionContext context) _context.InitialExecutionContext = ExecutionContext.Capture(); _input = new Pipe(GetInputPipeOptions()); - _output = new Pipe(GetOutputPipeOptions()); _frameWriter = new Http2FrameWriter( - _output.Writer, + context.Transport.Output, context.ConnectionContext, this, _outputFlowControl, @@ -126,7 +123,7 @@ public Http2Connection(HttpConnectionContext context) _scheduleInline = context.ServiceContext.Scheduler == PipeScheduler.Inline; _inputTask = CopyPipeAsync(_context.Transport.Input, _input.Writer); - _outputTask = CopyPipeAsync(_output.Reader, _context.Transport.Output); + } public string ConnectionId => _context.ConnectionId; @@ -378,10 +375,8 @@ public async Task ProcessRequestsAsync(IHttpApplication appl finally { Input.Complete(); - _output.Writer.Complete(); _context.Transport.Input.CancelPendingRead(); await _inputTask; - await _outputTask; } } } @@ -1662,38 +1657,6 @@ public void DecrementActiveClientStreamCount() minimumSegmentSize: _context.MemoryPool.GetMinimumSegmentSize(), useSynchronizationContext: false); - private PipeOptions GetOutputPipeOptions() - { - // Never write inline because we do not want to hold Http2FramerWriter._writeLock for potentially expensive TLS - // write operations. This essentially doubles the MaxResponseBufferSize for HTTP/2 connections compared to - // HTTP/1.x. This seems reasonable given HTTP/2's support for many concurrent streams per connection. We don't - // want every write to return an incomplete ValueTask now that we're dispatching TLS write operations which - // would likely happen with a pauseWriterThreshold of 1, but we still need to respect connection back pressure. - var pauseWriterThreshold = _context.ServiceContext.ServerOptions.Limits.MaxResponseBufferSize switch - { - // null means that we have no back pressure - null => 0, - // 0 = no buffering so we need to configure the pipe so the writer waits on the reader directly - 0 => 1, - long limit => limit, - }; - - var resumeWriterThreshold = pauseWriterThreshold switch - { - // The resumeWriterThreshold must be at least 1 to ever resume after pausing. - 1 => 1, - long limit => limit / 2, - }; - - return new PipeOptions(pool: _context.MemoryPool, - readerScheduler: _context.ServiceContext.Scheduler, - writerScheduler: PipeScheduler.Inline, - pauseWriterThreshold: pauseWriterThreshold, - resumeWriterThreshold: resumeWriterThreshold, - minimumSegmentSize: _context.MemoryPool.GetMinimumSegmentSize(), - useSynchronizationContext: false); - } - private async Task CopyPipeAsync(PipeReader reader, PipeWriter writer) { Exception? error = null; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index e2efd4b25755..1d616d8dc071 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -119,16 +119,19 @@ private async Task WriteToOutputPipe() buffer = buffer.Slice(0, actual); } + var (hasModeData, scheduleAgain) = producer.Dequeue(buffer.Length); + FlushResult flushResult = default; if (readResult.IsCanceled) { // Response body is aborted, complete reader for this output producer. } - else if (readResult.IsCompleted && stream.ResponseTrailers?.Count > 0) + else if (readResult.IsCompleted && stream.ResponseTrailers is { Count: > 0 } && !hasModeData) { // Output is ending and there are trailers to write - // Write any remaining content then write trailers + // Write any remaining content then write trailers and there's no + // flow control back pressure being applied (hasMoreData) stream.ResponseTrailers.SetReadOnly(); stream.DecrementActiveClientStreamCount(); @@ -158,7 +161,7 @@ private async Task WriteToOutputPipe() } else { - var endStream = readResult.IsCompleted; + var endStream = readResult.IsCompleted && !hasModeData; if (endStream) { @@ -170,11 +173,9 @@ private async Task WriteToOutputPipe() producer.FirstWrite = false; - var hasModeData = producer.Dequeue(buffer.Length); - reader.AdvanceTo(buffer.End); - if (readResult.IsCompleted || readResult.IsCanceled) + if ((readResult.IsCompleted && !hasModeData) || readResult.IsCanceled) { await reader.CompleteAsync(); @@ -207,6 +208,10 @@ private async Task WriteToOutputPipe() producer.MarkWaitingForWindowUpdates(true); } } + else if (scheduleAgain) + { + Schedule(producer); + } } catch (Exception ex) { @@ -249,7 +254,6 @@ public void Complete() _completed = true; AbortFlowControl(); _outputWriter.Abort(); - _channel.Writer.TryComplete(); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 270e0583688e..c96ecf91200f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -125,14 +125,14 @@ private bool EnqueueForObservation() // Removes consumed bytes from the queue. // Returns a bool that represents whether we should schedule this producer to write // the remaining bytes. - internal bool Dequeue(long bytes) + internal (bool, bool) Dequeue(long bytes) { lock (_dataWriterLock) { var wasEnqueuedForObservation = _enqueuedForObservation; _enqueuedForObservation = false; _unconsumedBytes -= bytes; - return _unconsumedBytes > 0 || wasEnqueuedForObservation; + return (_unconsumedBytes > 0, wasEnqueuedForObservation); } } @@ -199,13 +199,13 @@ public void Complete() { var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); + // Make sure the writing side is completed. + _pipeWriter.Complete(); + if (enqueue) { Schedule(); } - - // Make sure the writing side is completed. - _pipeWriter.Complete(); } else { @@ -395,13 +395,13 @@ public ValueTask WriteStreamSuffixAsync() // Try to enqueue any unflushed bytes var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); + _pipeWriter.Complete(); + if (enqueue) { Schedule(); } - _pipeWriter.Complete(); - return GetWaiterTask(); } } @@ -543,13 +543,13 @@ public void Stop() var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); + _pipeReader.CancelPendingRead(); + if (enqueue) { // We need to make sure the cancellation is observed by the code Schedule(); } - - _pipeReader.CancelPendingRead(); } } From 4c88f2fe5e007aa6e9d5a93da485bf18d6f3b634 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 18:34:41 -0700 Subject: [PATCH 23/77] Cleaned up some tests --- .../InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index f5d06e04c2c9..1df14e24fe4c 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -424,9 +424,6 @@ await ExpectAsync(Http2FrameType.HEADERS, withStreamId: 1); await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); - - var output = (Http2OutputProducer)stream.Output; - // await output._dataWriteProcessingTask.DefaultTimeout(); } [Fact] @@ -3804,7 +3801,6 @@ await ExpectAsync(Http2FrameType.DATA, withStreamId: 3); await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); - await _closedStateReached.Task.DefaultTimeout(); } From f57ce6789f3ca749d241500caae6170711ad9c56 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 18:35:26 -0700 Subject: [PATCH 24/77] Remove space --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 1178a0b402e3..976f10745863 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -123,7 +123,6 @@ public Http2Connection(HttpConnectionContext context) _scheduleInline = context.ServiceContext.Scheduler == PipeScheduler.Inline; _inputTask = CopyPipeAsync(_context.Transport.Input, _input.Writer); - } public string ConnectionId => _context.ConnectionId; From 4dfe1358fc718dc0b9e3223de2bd8367e5aa0107 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 18:40:42 -0700 Subject: [PATCH 25/77] More code cleanup --- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index c96ecf91200f..d4a04fa32149 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -45,8 +45,8 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV private long _window; // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state - bool _enqueuedForObservation; - bool _waitingForWindowUpdates; + private bool _enqueuedForObservation; + private bool _waitingForWindowUpdates; /// The core logic for the IValueTaskSource implementation. private ManualResetValueTaskSourceCore _responseCompleteTaskSource = new ManualResetValueTaskSourceCore { RunContinuationsAsynchronously = true }; // mutable struct, do not make this readonly @@ -155,9 +155,7 @@ private bool UpdateWindow(long bytes) lock (_dataWriterLock) { var wasDepleted = _window <= 0; - _window += bytes; - return wasDepleted && _window > 0 && _unconsumedBytes > 0; } } From 4d521adcb120f4aa864757910cae7f549f4c5e02 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 22:07:34 -0700 Subject: [PATCH 26/77] Made the timeout tests work --- .../src/Internal/Http2/Http2FrameWriter.cs | 27 +++++++++++++++---- .../src/Internal/Http2/Http2OutputProducer.cs | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 1d616d8dc071..f86a927d2c42 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -171,6 +171,11 @@ private async Task WriteToOutputPipe() flushResult = await WriteDataAsync(stream.StreamId, buffer, buffer.Length, endStream, producer.FirstWrite); } + if (producer.IsTimingWrite) + { + _timeoutControl.StopTimingWrite(); + } + producer.FirstWrite = false; reader.AdvanceTo(buffer.End); @@ -196,6 +201,13 @@ private async Task WriteToOutputPipe() { _waitingForMoreWindow.Enqueue(producer); } + + // Include waiting for window updates in timing writes + if (_minResponseDataRate != null) + { + producer.IsTimingWrite = true; + _timeoutControl.StartTimingWrite(); + } } else if (remainingStream > 0) { @@ -206,6 +218,13 @@ private async Task WriteToOutputPipe() { // Mark the output as waiting for a window upate to resume writing (there's still data) producer.MarkWaitingForWindowUpdates(true); + + // Include waiting for window updates in timing writes + if (_minResponseDataRate != null) + { + producer.IsTimingWrite = true; + _timeoutControl.StartTimingWrite(); + } } } else if (scheduleAgain) @@ -510,8 +529,6 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen private async ValueTask WriteDataAsync(int streamId, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) { - FlushResult flushResult = default; - var writeTask = default(ValueTask); lock (_writeLock) @@ -520,7 +537,7 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen if (_completed) { - return flushResult; + return new FlushResult(isCanceled: false, isCompleted: true); } if (dataLength > 0 || endStream) @@ -549,13 +566,12 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen } } - // REVIEW: This will include flow control waiting as of right now. We need to handle that elsewhere. if (_minResponseDataRate != null) { _timeoutControl.StartTimingWrite(); } - flushResult = await writeTask; + var flushResult = await writeTask; if (_minResponseDataRate != null) { @@ -817,6 +833,7 @@ public bool TryUpdateConnectionWindow(int bytes) { // We're no longer waiting for the update producer.MarkWaitingForWindowUpdates(false); + Schedule(producer); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index d4a04fa32149..dc3198a5677f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -82,6 +82,8 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, Strea public Http2Stream Stream => _stream; public PipeReader PipeReader => _pipeReader; + public bool IsTimingWrite { get; set; } + public bool FirstWrite { get; set; } = true; public bool StreamEnded => _streamEnded; From af84669e5873e75b0c6b158f7adbafdbe590ad7d Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 26 Mar 2022 22:14:30 -0700 Subject: [PATCH 27/77] Tests pass! --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index f86a927d2c42..ed20cad795ad 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -85,7 +85,8 @@ public Http2FrameWriter( AllowSynchronousContinuations = _scheduleInline }); - _window = connectionOutputFlowControl.Available; + // This is null in tests sometimes + _window = connectionOutputFlowControl?.Available ?? 0; _ = Task.Run(WriteToOutputPipe); } From 533b6a6cb2f71583865be4fc1a84bc8b0ad55f9f Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sun, 27 Mar 2022 07:50:34 -0700 Subject: [PATCH 28/77] Move header writing to the processing loop - Reverted the schedule before run pattern as it results in multiple dispatches for headers and body. - Fixed tests to work with this pattern. --- .../src/Internal/Http2/Http2Connection.cs | 2 + .../src/Internal/Http2/Http2FrameWriter.cs | 65 ++++++++++++++-- .../src/Internal/Http2/Http2OutputProducer.cs | 77 +++++++++++-------- .../Http2/Http2ConnectionTests.cs | 12 ++- .../Http2/Http2StreamTests.cs | 6 +- .../Http2/Http2TimeoutTests.cs | 1 - 6 files changed, 119 insertions(+), 44 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 976f10745863..81e9e99d2a04 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -72,6 +72,7 @@ internal partial class Http2Connection : IHttp2StreamLifetimeHandler, IHttpStrea internal readonly Http2KeepAlive? _keepAlive; internal readonly Dictionary _streams = new Dictionary(); internal PooledStreamStack StreamPool; + internal Action? _onStreamCompleted; public Http2Connection(HttpConnectionContext context) { @@ -1230,6 +1231,7 @@ void IHttp2StreamLifetimeHandler.OnStreamCompleted(Http2Stream stream) { _completedStreams.Enqueue(stream); _streamCompletionAwaitable.Complete(); + _onStreamCompleted?.Invoke(stream); } private void UpdateCompletedStreams() diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index ed20cad795ad..661e9638f93f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -105,9 +105,37 @@ private async Task WriteToOutputPipe() var reader = producer.PipeReader; var stream = producer.Stream; - var readResult = await reader.ReadAsync(); + var observed = Http2OutputProducer.State.None; + + if (!reader.TryRead(out var readResult)) + { + if (producer.WriteHeaders) + { + // Flush headers, we have nothing to look at on the pipe + } + else + { + readResult = await reader.ReadAsync(); + } + } + var buffer = readResult.Buffer; + if (producer.WriteHeaders) + { + observed |= Http2OutputProducer.State.FlushHeaders; + } + + if (readResult.IsCanceled) + { + observed |= Http2OutputProducer.State.Cancelled; + } + + if (readResult.IsCompleted) + { + observed |= Http2OutputProducer.State.Completed; + } + // Check the stream window var (actual, remainingStream) = producer.ConsumeWindow(buffer.Length); @@ -120,13 +148,18 @@ private async Task WriteToOutputPipe() buffer = buffer.Slice(0, actual); } - var (hasModeData, scheduleAgain) = producer.Dequeue(buffer.Length); + var (hasModeData, scheduleAgain) = producer.Dequeue(buffer.Length, observed); FlushResult flushResult = default; if (readResult.IsCanceled) { // Response body is aborted, complete reader for this output producer. + if (producer.WriteHeaders) + { + // write headers + WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + } } else if (readResult.IsCompleted && stream.ResponseTrailers is { Count: > 0 } && !hasModeData) { @@ -137,10 +170,17 @@ private async Task WriteToOutputPipe() stream.ResponseTrailers.SetReadOnly(); stream.DecrementActiveClientStreamCount(); + // TODO: Combine headers, data and trailers + if (producer.WriteHeaders) + { + // write headers + WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + } + if (buffer.Length > 0) { // It is faster to write data and trailers together. Locking once reduces lock contention. - flushResult = await WriteDataAndTrailersAsync(stream.StreamId, buffer, producer.FirstWrite, stream.ResponseTrailers); + flushResult = await WriteDataAndTrailersAsync(stream.StreamId, buffer, producer.WriteHeaders, stream.ResponseTrailers); } else { @@ -156,6 +196,14 @@ private async Task WriteToOutputPipe() } else { + stream.DecrementActiveClientStreamCount(); + + if (producer.WriteHeaders) + { + // write headers + WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.END_STREAM, (HttpResponseHeaders)stream.ResponseHeaders); + } + // Headers have already been written and there is no other content to write flushResult = await FlushAsync(outputAborter: null, cancellationToken: default); } @@ -169,7 +217,13 @@ private async Task WriteToOutputPipe() stream.DecrementActiveClientStreamCount(); } - flushResult = await WriteDataAsync(stream.StreamId, buffer, buffer.Length, endStream, producer.FirstWrite); + if (producer.WriteHeaders) + { + // We need to write headers + WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + } + + flushResult = await WriteDataAsync(stream.StreamId, buffer, buffer.Length, endStream, producer.WriteHeaders); } if (producer.IsTimingWrite) @@ -177,7 +231,7 @@ private async Task WriteToOutputPipe() _timeoutControl.StopTimingWrite(); } - producer.FirstWrite = false; + producer.WriteHeaders = false; reader.AdvanceTo(buffer.End); @@ -363,7 +417,6 @@ public void WriteResponseHeaders(int streamId, int statusCode, Http2HeadersFrame { _log.HPackEncodingError(_connectionId, streamId, ex); _http2Connection.Abort(new ConnectionAbortedException(ex.Message, ex)); - throw new InvalidOperationException(ex.Message, ex); // Report the error to the user if this was the first write. } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index dc3198a5677f..035181fff76e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -45,7 +45,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV private long _window; // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state - private bool _enqueuedForObservation; + private State _observationState; private bool _waitingForWindowUpdates; /// The core logic for the IValueTaskSource implementation. @@ -84,7 +84,7 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, Strea public bool IsTimingWrite { get; set; } - public bool FirstWrite { get; set; } = true; + public bool WriteHeaders { get; set; } public bool StreamEnded => _streamEnded; @@ -99,6 +99,9 @@ public bool StreamCompleted } } + // Useful for debugging the scheduling state in the debugger + internal (int, long, State, long, bool) SchedulingState => (Stream.StreamId, _unconsumedBytes, _observationState, _window, _waitingForWindowUpdates); + // Added bytes to the queue. // Returns a bool that represents whether we should schedule this producer to write // the enqueued bytes @@ -108,18 +111,18 @@ private bool Enqueue(long bytes) { var wasEmpty = _unconsumedBytes == 0; _unconsumedBytes += bytes; - return wasEmpty && _unconsumedBytes > 0; + return wasEmpty && _unconsumedBytes > 0 && _observationState == State.None; } } // Determines if we should schedule this producer to observe // any state changes made. - private bool EnqueueForObservation() + private bool EnqueueForObservation(State state) { lock (_dataWriterLock) { - var wasEnqueuedForObservation = _enqueuedForObservation; - _enqueuedForObservation = true; + var wasEnqueuedForObservation = _observationState != State.None; + _observationState |= state; return (_unconsumedBytes == 0 || _waitingForWindowUpdates) && !wasEnqueuedForObservation; } } @@ -127,14 +130,13 @@ private bool EnqueueForObservation() // Removes consumed bytes from the queue. // Returns a bool that represents whether we should schedule this producer to write // the remaining bytes. - internal (bool, bool) Dequeue(long bytes) + internal (bool, bool) Dequeue(long bytes, State state) { lock (_dataWriterLock) { - var wasEnqueuedForObservation = _enqueuedForObservation; - _enqueuedForObservation = false; + _observationState &= ~state; _unconsumedBytes -= bytes; - return (_unconsumedBytes > 0, wasEnqueuedForObservation); + return (_unconsumedBytes > 0, _observationState != State.None); } } @@ -178,8 +180,10 @@ public void StreamReset(uint initialWindowSize) _window = initialWindowSize; _unconsumedBytes = 0; - _enqueuedForObservation = false; + _observationState = State.None; _waitingForWindowUpdates = false; + WriteHeaders = false; + IsTimingWrite = false; } public void Complete() @@ -197,7 +201,7 @@ public void Complete() if (!_streamCompleted) { - var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); + var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Completed); // Make sure the writing side is completed. _pipeWriter.Complete(); @@ -260,20 +264,27 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) var enqueue = Enqueue(_pipeWriter.UnflushedBytes); + // If there's already been response data written to the stream, just wait for that. Any header + // should be in front of the data frames in the connection pipe. Trailers could change things. + var task = _flusher.FlushAsync(this, cancellationToken); + if (enqueue) { Schedule(); } - // If there's already been response data written to the stream, just wait for that. Any header - // should be in front of the data frames in the connection pipe. Trailers could change things. - return _flusher.FlushAsync(this, cancellationToken); + return task; } else { - // Flushing the connection pipe ensures headers already in the pipe are flushed even if no data - // frames have been written. - return _frameWriter.FlushAsync(this, cancellationToken); + var enqueue = EnqueueForObservation(State.FlushHeaders); + + if (enqueue) + { + Schedule(); + } + + return default; } } } @@ -330,20 +341,12 @@ public void WriteResponseHeaders(int statusCode, string? reasonPhrase, HttpRespo // The headers will be the final frame if: // 1. There is no content // 2. There is no trailing HEADERS frame. - Http2HeadersFrameFlags http2HeadersFrame; - if (appCompleted && !_startedWritingDataFrames && (_stream.ResponseTrailers == null || _stream.ResponseTrailers.Count == 0)) { _streamEnded = true; - _stream.DecrementActiveClientStreamCount(); - http2HeadersFrame = Http2HeadersFrameFlags.END_STREAM; - } - else - { - http2HeadersFrame = Http2HeadersFrameFlags.NONE; } - _frameWriter.WriteResponseHeaders(StreamId, statusCode, http2HeadersFrame, responseHeaders); + WriteHeaders = true; } } @@ -371,12 +374,14 @@ public Task WriteDataAsync(ReadOnlySpan data, CancellationToken cancellati var enqueue = Enqueue(data.Length); + var task = _flusher.FlushAsync(this, cancellationToken).GetAsTask(); + if (enqueue) { Schedule(); } - return _flusher.FlushAsync(this, cancellationToken).GetAsTask(); + return task; } } @@ -393,7 +398,7 @@ public ValueTask WriteStreamSuffixAsync() _suffixSent = true; // Try to enqueue any unflushed bytes - var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); + var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Completed); _pipeWriter.Complete(); @@ -500,13 +505,14 @@ public ValueTask WriteDataToPipeAsync(ReadOnlySpan data, Canc _pipeWriter.Write(data); var enqueue = Enqueue(data.Length); + var task = _flusher.FlushAsync(this, cancellationToken); if (enqueue) { Schedule(); } - return _flusher.FlushAsync(this, cancellationToken); + return task; } } @@ -541,7 +547,7 @@ public void Stop() _streamCompleted = true; - var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(); + var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Cancelled); _pipeReader.CancelPendingRead(); @@ -672,4 +678,13 @@ public void Dispose() } _disposed = true; } + + [Flags] + public enum State + { + None = 0, + FlushHeaders = 1, + Cancelled = 2, + Completed = 4 + } } diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index 1df14e24fe4c..3a9ac6fc49f2 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -564,11 +564,17 @@ await InitializeConnectionAsync(async context => throw new InvalidOperationException("Put the stream into an invalid state by throwing after writing to response."); }); + var streamCompletedTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _connection._onStreamCompleted = _ => streamCompletedTcs.TrySetResult(); + await StartStreamAsync(1, _browserRequestHeaders, endStream: true); var stream = _connection._streams[1]; serverTcs.SetResult(); + // Wait for the stream to be completed + await streamCompletedTcs.Task; + // TriggerTick will trigger the stream to be returned to the pool so we can assert it TriggerTick(); @@ -3800,6 +3806,7 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 3); + TriggerTick(); await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); await _closedStateReached.Task.DefaultTimeout(); } @@ -4164,7 +4171,7 @@ await InitializeConnectionAsync(async context => for (var i = 0; i < expectedFullFrameCountBeforeBackpressure; i++) { await expectingDataSem.WaitAsync(); - Assert.True(context.Response.Body.WriteAsync(_maxData, 0, _maxData.Length).IsCompleted); + await context.Response.Body.WriteAsync(_maxData, 0, _maxData.Length); } await expectingDataSem.WaitAsync(); @@ -4716,6 +4723,7 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 1); + TriggerTick(); await _closedStateReached.Task.DefaultTimeout(); VerifyGoAway(await ReceiveFrameAsync(), 1, Http2ErrorCode.NO_ERROR); } @@ -4762,6 +4770,8 @@ await ExpectAsync(Http2FrameType.DATA, withFlags: (byte)Http2DataFrameFlags.END_STREAM, withStreamId: 3); + TriggerTick(); + await WaitForConnectionStopAsync(expectedLastStreamId: 3, ignoreNonGoAwayFrames: false); } diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs index 52569848c45a..d1262d61c1e5 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs @@ -3103,15 +3103,11 @@ public async Task ResponseWithHeadersTooLarge_AbortsConnection() await InitializeConnectionAsync(async context => { context.Response.Headers["too_long"] = new string('a', (int)Http2PeerSettings.DefaultMaxFrameSize); - var ex = await Assert.ThrowsAsync(() => context.Response.WriteAsync("Hello World")).DefaultTimeout(); - appFinished.TrySetResult(ex.InnerException.Message); + await context.Response.WriteAsync("Hello World"); }); await StartStreamAsync(1, _browserRequestHeaders, endStream: true); - var message = await appFinished.Task.DefaultTimeout(); - Assert.Equal(SR.net_http_hpack_encode_failure, message); - // Just the StatusCode gets written before aborting in the continuation frame await ExpectAsync(Http2FrameType.HEADERS, withLength: 32, diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs index dca2d0321b1b..7c47189caf86 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs @@ -314,7 +314,6 @@ private class EchoAppWithNotification public async Task RunApp(HttpContext context) { - await context.Response.Body.FlushAsync(); var buffer = new byte[Http2PeerSettings.MinAllowedMaxFrameSize]; int received; From 1a3d0f048293c44768b2a66fa9bb1428e94dd466 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sun, 27 Mar 2022 07:56:02 -0700 Subject: [PATCH 29/77] Small clean up --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 7 ++++--- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 661e9638f93f..84372fe1330f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -583,6 +583,8 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen private async ValueTask WriteDataAsync(int streamId, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) { + FlushResult flushResult = default; + var writeTask = default(ValueTask); lock (_writeLock) @@ -591,7 +593,7 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen if (_completed) { - return new FlushResult(isCanceled: false, isCompleted: true); + return flushResult; } if (dataLength > 0 || endStream) @@ -625,7 +627,7 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen _timeoutControl.StartTimingWrite(); } - var flushResult = await writeTask; + flushResult = await writeTask; if (_minResponseDataRate != null) { @@ -856,7 +858,6 @@ private void AbortFlowControl() { lock (_windowUpdateLock) { - // REVIEW: What should this be doing? while (_waitingForMoreWindow.TryDequeue(out var producer)) { if (!producer.StreamCompleted) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 035181fff76e..5babc35a2ad0 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -70,6 +70,7 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, Strea _pipe = CreateDataPipe(_memoryPool); _pipeWriter = new ConcurrentPipeWriter(_pipe.Writer, _memoryPool, _dataWriterLock); + Debug.Assert(_pipeWriter.CanGetUnflushedBytes); _pipeReader = _pipe.Reader; // No need to pass in timeoutControl here, since no minDataRates are passed to the TimingPipeFlusher. @@ -260,8 +261,6 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) if (_startedWritingDataFrames) { - Debug.Assert(_pipeWriter.CanGetUnflushedBytes); - var enqueue = Enqueue(_pipeWriter.UnflushedBytes); // If there's already been response data written to the stream, just wait for that. Any header From 17c9cc67ed3b66d969690439c3ed80e216428809 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sun, 27 Mar 2022 12:41:51 -0700 Subject: [PATCH 30/77] Merged the locks for header writing when possible --- .../src/Internal/Http2/Http2FrameWriter.cs | 89 ++++++++++--------- 1 file changed, 48 insertions(+), 41 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 84372fe1330f..f14b149f71d6 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -170,21 +170,14 @@ private async Task WriteToOutputPipe() stream.ResponseTrailers.SetReadOnly(); stream.DecrementActiveClientStreamCount(); - // TODO: Combine headers, data and trailers - if (producer.WriteHeaders) - { - // write headers - WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); - } - if (buffer.Length > 0) { // It is faster to write data and trailers together. Locking once reduces lock contention. - flushResult = await WriteDataAndTrailersAsync(stream.StreamId, buffer, producer.WriteHeaders, stream.ResponseTrailers); + flushResult = await WriteDataAndTrailersAsync(stream, buffer, producer.WriteHeaders, stream.ResponseTrailers); } else { - flushResult = await WriteResponseTrailersAsync(stream.StreamId, stream.ResponseTrailers); + flushResult = await WriteResponseTrailersAsync(stream, producer.WriteHeaders, stream.ResponseTrailers); } } else if (readResult.IsCompleted && producer.StreamEnded) @@ -217,13 +210,7 @@ private async Task WriteToOutputPipe() stream.DecrementActiveClientStreamCount(); } - if (producer.WriteHeaders) - { - // We need to write headers - WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); - } - - flushResult = await WriteDataAsync(stream.StreamId, buffer, buffer.Length, endStream, producer.WriteHeaders); + flushResult = await WriteDataAsync(stream, buffer, buffer.Length, endStream, producer.WriteHeaders); } if (producer.IsTimingWrite) @@ -403,25 +390,30 @@ public void WriteResponseHeaders(int streamId, int statusCode, Http2HeadersFrame return; } - try - { - _headersEnumerator.Initialize(headers); - _outgoingFrame.PrepareHeaders(headerFrameFlags, streamId); - var buffer = _headerEncodingBuffer.AsSpan(); - var done = HPackHeaderWriter.BeginEncodeHeaders(statusCode, _hpackEncoder, _headersEnumerator, buffer, out var payloadLength); - FinishWritingHeaders(streamId, payloadLength, done); - } - // Any exception from the HPack encoder can leave the dynamic table in a corrupt state. - // Since we allow custom header encoders we don't know what type of exceptions to expect. - catch (Exception ex) - { - _log.HPackEncodingError(_connectionId, streamId, ex); - _http2Connection.Abort(new ConnectionAbortedException(ex.Message, ex)); - } + WriteResponseHeadersUnsynchronized(streamId, statusCode, headerFrameFlags, headers); + } + } + + private void WriteResponseHeadersUnsynchronized(int streamId, int statusCode, Http2HeadersFrameFlags headerFrameFlags, HttpResponseHeaders headers) + { + try + { + _headersEnumerator.Initialize(headers); + _outgoingFrame.PrepareHeaders(headerFrameFlags, streamId); + var buffer = _headerEncodingBuffer.AsSpan(); + var done = HPackHeaderWriter.BeginEncodeHeaders(statusCode, _hpackEncoder, _headersEnumerator, buffer, out var payloadLength); + FinishWritingHeaders(streamId, payloadLength, done); + } + // Any exception from the HPack encoder can leave the dynamic table in a corrupt state. + // Since we allow custom header encoders we don't know what type of exceptions to expect. + catch (Exception ex) + { + _log.HPackEncodingError(_connectionId, streamId, ex); + _http2Connection.Abort(new ConnectionAbortedException(ex.Message, ex)); } } - private ValueTask WriteResponseTrailersAsync(int streamId, HttpResponseTrailers headers) + private ValueTask WriteResponseTrailersAsync(Http2Stream stream, bool firstWrite, HttpResponseTrailers headers) { lock (_writeLock) { @@ -430,6 +422,13 @@ private ValueTask WriteResponseTrailersAsync(int streamId, HttpResp return default; } + if (firstWrite) + { + WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + } + + var streamId = stream.StreamId; + try { _headersEnumerator.Initialize(headers); @@ -479,7 +478,7 @@ private void FinishWritingHeaders(int streamId, int payloadLength, bool done) } } - public ValueTask WriteDataAndTrailersAsync(int streamId, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) + public ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) { // This method combines WriteDataAsync and WriteResponseTrailers. // Changes here may need to be mirrored in WriteDataAsync. @@ -494,9 +493,14 @@ public ValueTask WriteDataAndTrailersAsync(int streamId, in ReadOnl return default; } - WriteDataUnsynchronized(streamId, data, dataLength, endStream: false); + if (firstWrite) + { + WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + } + + WriteDataUnsynchronized(stream.StreamId, data, dataLength, endStream: false); - return WriteResponseTrailersAsync(streamId, headers); + return WriteResponseTrailersAsync(stream, firstWrite: false, headers); } } @@ -581,7 +585,7 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen } } - private async ValueTask WriteDataAsync(int streamId, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) + private async ValueTask WriteDataAsync(Http2Stream stream, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) { FlushResult flushResult = default; @@ -589,21 +593,24 @@ private async ValueTask WriteDataAsync(int streamId, ReadOnlySequen lock (_writeLock) { - var shouldFlush = false; - if (_completed) { return flushResult; } - if (dataLength > 0 || endStream) + var shouldFlush = false; + + if (firstWrite) { - WriteDataUnsynchronized(streamId, data, dataLength, endStream); + WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); shouldFlush = true; } - else if (firstWrite) + + if (dataLength > 0 || endStream) { + WriteDataUnsynchronized(stream.StreamId, data, dataLength, endStream); + shouldFlush = true; } From 8593ef29257d859b73a274619851c9fc481399f1 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 07:34:42 -0700 Subject: [PATCH 31/77] More optimizations - Write headers in the lock with the flush - Added debug assert for scheduling output producers --- .../src/Internal/Http2/Http2FrameWriter.cs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index f14b149f71d6..326e0d96d290 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -93,7 +93,9 @@ public Http2FrameWriter( public void Schedule(Http2OutputProducer producer) { - _channel.Writer.TryWrite(producer); + var scheduled = _channel.Writer.TryWrite(producer); + + Debug.Assert(scheduled, $"Unable to schedule Stream {producer.Stream.StreamId}"); } private async Task WriteToOutputPipe() @@ -191,14 +193,8 @@ private async Task WriteToOutputPipe() { stream.DecrementActiveClientStreamCount(); - if (producer.WriteHeaders) - { - // write headers - WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.END_STREAM, (HttpResponseHeaders)stream.ResponseHeaders); - } - // Headers have already been written and there is no other content to write - flushResult = await FlushAsync(outputAborter: null, cancellationToken: default); + flushResult = await FlushAsync(stream, producer.WriteHeaders, outputAborter: null, cancellationToken: default); } } else @@ -334,7 +330,7 @@ public void Abort(ConnectionAbortedException error) } } - public ValueTask FlushAsync(IHttpOutputAborter? outputAborter, CancellationToken cancellationToken) + private ValueTask FlushAsync(Http2Stream stream, bool firstWrite, IHttpOutputAborter? outputAborter, CancellationToken cancellationToken) { lock (_writeLock) { @@ -343,6 +339,12 @@ public ValueTask FlushAsync(IHttpOutputAborter? outputAborter, Canc return default; } + if (firstWrite) + { + // write headers + WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.END_STREAM, (HttpResponseHeaders)stream.ResponseHeaders); + } + var bytesWritten = _unflushedBytes; _unflushedBytes = 0; From 6dbfb142701a91c58eaa9a3f95947ae8035026f0 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 07:41:52 -0700 Subject: [PATCH 32/77] Remove a state machine from the write code path --- .../src/Internal/Http2/Http2FrameWriter.cs | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 326e0d96d290..336961e7345a 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -587,17 +587,15 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen } } - private async ValueTask WriteDataAsync(Http2Stream stream, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) + private ValueTask WriteDataAsync(Http2Stream stream, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) { - FlushResult flushResult = default; - var writeTask = default(ValueTask); lock (_writeLock) { if (_completed) { - return flushResult; + return new(default(FlushResult)); } var shouldFlush = false; @@ -631,19 +629,28 @@ private async ValueTask WriteDataAsync(Http2Stream stream, ReadOnly } } - if (_minResponseDataRate != null) + if (writeTask.IsCompletedSuccessfully) { - _timeoutControl.StartTimingWrite(); + return new(writeTask.Result); } - flushResult = await writeTask; + return FlushAsyncAwaited(writeTask, _timeoutControl, _minResponseDataRate); - if (_minResponseDataRate != null) + static async ValueTask FlushAsyncAwaited(ValueTask writeTask, ITimeoutControl timeoutControl, MinDataRate? minResponseDataRate) { - _timeoutControl.StopTimingWrite(); - } + if (minResponseDataRate != null) + { + timeoutControl.StartTimingWrite(); + } - return flushResult; + var flushResult = await writeTask; + + if (minResponseDataRate != null) + { + timeoutControl.StopTimingWrite(); + } + return flushResult; + } } /* https://tools.ietf.org/html/rfc7540#section-6.9 From 69a03846c9a0efce7d9c536f6ebf141e860c1703 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 07:47:46 -0700 Subject: [PATCH 33/77] Rename firstWrite to writeHeaders --- .../src/Internal/Http2/Http2FrameWriter.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 336961e7345a..46653feb9394 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -330,7 +330,7 @@ public void Abort(ConnectionAbortedException error) } } - private ValueTask FlushAsync(Http2Stream stream, bool firstWrite, IHttpOutputAborter? outputAborter, CancellationToken cancellationToken) + private ValueTask FlushAsync(Http2Stream stream, bool writeHeaders, IHttpOutputAborter? outputAborter, CancellationToken cancellationToken) { lock (_writeLock) { @@ -339,7 +339,7 @@ private ValueTask FlushAsync(Http2Stream stream, bool firstWrite, I return default; } - if (firstWrite) + if (writeHeaders) { // write headers WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.END_STREAM, (HttpResponseHeaders)stream.ResponseHeaders); @@ -415,7 +415,7 @@ private void WriteResponseHeadersUnsynchronized(int streamId, int statusCode, Ht } } - private ValueTask WriteResponseTrailersAsync(Http2Stream stream, bool firstWrite, HttpResponseTrailers headers) + private ValueTask WriteResponseTrailersAsync(Http2Stream stream, bool writeHeaders, HttpResponseTrailers headers) { lock (_writeLock) { @@ -424,7 +424,7 @@ private ValueTask WriteResponseTrailersAsync(Http2Stream stream, bo return default; } - if (firstWrite) + if (writeHeaders) { WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); } @@ -480,7 +480,7 @@ private void FinishWritingHeaders(int streamId, int payloadLength, bool done) } } - public ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in ReadOnlySequence data, bool firstWrite, HttpResponseTrailers headers) + public ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in ReadOnlySequence data, bool writeHeaders, HttpResponseTrailers headers) { // This method combines WriteDataAsync and WriteResponseTrailers. // Changes here may need to be mirrored in WriteDataAsync. @@ -495,14 +495,14 @@ public ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in R return default; } - if (firstWrite) + if (writeHeaders) { WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); } WriteDataUnsynchronized(stream.StreamId, data, dataLength, endStream: false); - return WriteResponseTrailersAsync(stream, firstWrite: false, headers); + return WriteResponseTrailersAsync(stream, writeHeaders: false, headers); } } @@ -587,7 +587,7 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen } } - private ValueTask WriteDataAsync(Http2Stream stream, ReadOnlySequence data, long dataLength, bool endStream, bool firstWrite) + private ValueTask WriteDataAsync(Http2Stream stream, ReadOnlySequence data, long dataLength, bool endStream, bool writeHeaders) { var writeTask = default(ValueTask); @@ -600,7 +600,7 @@ private ValueTask WriteDataAsync(Http2Stream stream, ReadOnlySequen var shouldFlush = false; - if (firstWrite) + if (writeHeaders) { WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); From 4b0119280d9bd45b0e9838226284a68964d166d4 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 21:44:53 -0700 Subject: [PATCH 34/77] Make the channel unboundeded - We allow more than the max streams by a little bit. This won't be much more than limit but we don't need to risk dropping streams as a result. --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 46653feb9394..54b0109e1ac8 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -80,9 +80,11 @@ public Http2FrameWriter( _hpackEncoder = new DynamicHPackEncoder(serviceContext.ServerOptions.AllowResponseHeaderCompression); - _channel = Channel.CreateBounded(new BoundedChannelOptions(serviceContext.ServerOptions.Limits.Http2.MaxStreamsPerConnection) + // In practice, this is bounded by the number of concurrent streams allowed + whatever overflow we allow + _channel = Channel.CreateUnbounded(new UnboundedChannelOptions() { - AllowSynchronousContinuations = _scheduleInline + AllowSynchronousContinuations = _scheduleInline, + SingleReader = true }); // This is null in tests sometimes @@ -93,9 +95,7 @@ public Http2FrameWriter( public void Schedule(Http2OutputProducer producer) { - var scheduled = _channel.Writer.TryWrite(producer); - - Debug.Assert(scheduled, $"Unable to schedule Stream {producer.Stream.StreamId}"); + _channel.Writer.TryWrite(producer); } private async Task WriteToOutputPipe() From 42425d9495dbccb5391b64bd3205cc6c7315be3e Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 22:03:25 -0700 Subject: [PATCH 35/77] Shutdown the write queue after the connection loop ends --- .../Core/src/Internal/Http2/Http2Connection.cs | 1 + .../Core/src/Internal/Http2/Http2FrameWriter.cs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 81e9e99d2a04..c7582924d3d2 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -377,6 +377,7 @@ public async Task ProcessRequestsAsync(IHttpApplication appl Input.Complete(); _context.Transport.Input.CancelPendingRead(); await _inputTask; + await _frameWriter.ShutdownAsync(); } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 54b0109e1ac8..efcbc12c14b1 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -50,6 +50,7 @@ internal class Http2FrameWriter private readonly object _windowUpdateLock = new(); private long _window; private readonly Queue _waitingForMoreWindow = new(); + private readonly Task _writeQueueTask; public Http2FrameWriter( PipeWriter outputPipeWriter, @@ -90,7 +91,7 @@ public Http2FrameWriter( // This is null in tests sometimes _window = connectionOutputFlowControl?.Available ?? 0; - _ = Task.Run(WriteToOutputPipe); + _writeQueueTask = Task.Run(WriteToOutputPipe); } public void Schedule(Http2OutputProducer producer) @@ -314,6 +315,13 @@ public void Complete() } } + public Task ShutdownAsync() + { + _channel.Writer.TryComplete(); + + return _writeQueueTask; + } + public void Abort(ConnectionAbortedException error) { lock (_writeLock) @@ -480,7 +488,7 @@ private void FinishWritingHeaders(int streamId, int payloadLength, bool done) } } - public ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in ReadOnlySequence data, bool writeHeaders, HttpResponseTrailers headers) + private ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in ReadOnlySequence data, bool writeHeaders, HttpResponseTrailers headers) { // This method combines WriteDataAsync and WriteResponseTrailers. // Changes here may need to be mirrored in WriteDataAsync. From eb78e1716223a07f4762cf35455d01290e38f1f5 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 22:06:23 -0700 Subject: [PATCH 36/77] Make James happy --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index efcbc12c14b1..3839ebdbb99a 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -603,7 +603,7 @@ private ValueTask WriteDataAsync(Http2Stream stream, ReadOnlySequen { if (_completed) { - return new(default(FlushResult)); + return ValueTask.FromResult(default); } var shouldFlush = false; From 9d80ad488888b968943e317d783c78593bdaf603 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 22:15:12 -0700 Subject: [PATCH 37/77] Rename schedule again to reschedule --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 3839ebdbb99a..bbd797f25932 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -151,7 +151,7 @@ private async Task WriteToOutputPipe() buffer = buffer.Slice(0, actual); } - var (hasModeData, scheduleAgain) = producer.Dequeue(buffer.Length, observed); + var (hasModeData, reschedule) = producer.Dequeue(buffer.Length, observed); FlushResult flushResult = default; @@ -266,7 +266,7 @@ private async Task WriteToOutputPipe() } } } - else if (scheduleAgain) + else if (reschedule) { Schedule(producer); } From 2b31f993cd4f4561a23f896dc5846bb62a8fe647 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 22:53:58 -0700 Subject: [PATCH 38/77] Rename cancelled to use one l --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 2 +- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index bbd797f25932..9a5284d8ee63 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -131,7 +131,7 @@ private async Task WriteToOutputPipe() if (readResult.IsCanceled) { - observed |= Http2OutputProducer.State.Cancelled; + observed |= Http2OutputProducer.State.Canceled; } if (readResult.IsCompleted) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 5babc35a2ad0..80898b6c33ac 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -546,7 +546,7 @@ public void Stop() _streamCompleted = true; - var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Cancelled); + var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Canceled); _pipeReader.CancelPendingRead(); @@ -683,7 +683,7 @@ public enum State { None = 0, FlushHeaders = 1, - Cancelled = 2, + Canceled = 2, Completed = 4 } } From 7bb3afb911404598b2affbaf98abebd3b260be13 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 23:00:26 -0700 Subject: [PATCH 39/77] Spelling --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 9a5284d8ee63..cf43a0d46f73 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -151,7 +151,7 @@ private async Task WriteToOutputPipe() buffer = buffer.Slice(0, actual); } - var (hasModeData, reschedule) = producer.Dequeue(buffer.Length, observed); + var (hasMoreData, reschedule) = producer.Dequeue(buffer.Length, observed); FlushResult flushResult = default; @@ -164,7 +164,7 @@ private async Task WriteToOutputPipe() WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); } } - else if (readResult.IsCompleted && stream.ResponseTrailers is { Count: > 0 } && !hasModeData) + else if (readResult.IsCompleted && stream.ResponseTrailers is { Count: > 0 } && !hasMoreData) { // Output is ending and there are trailers to write // Write any remaining content then write trailers and there's no @@ -200,7 +200,7 @@ private async Task WriteToOutputPipe() } else { - var endStream = readResult.IsCompleted && !hasModeData; + var endStream = readResult.IsCompleted && !hasMoreData; if (endStream) { @@ -219,7 +219,7 @@ private async Task WriteToOutputPipe() reader.AdvanceTo(buffer.End); - if ((readResult.IsCompleted && !hasModeData) || readResult.IsCanceled) + if ((readResult.IsCompleted && !hasMoreData) || readResult.IsCanceled) { await reader.CompleteAsync(); @@ -227,7 +227,7 @@ private async Task WriteToOutputPipe() } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. - else if (hasModeData) + else if (hasMoreData) { // We have no more connection window, put this producer in a queue waiting for it to // a window update to resume the connection. From 0b94cb238a7982cedd815a79a6ecdf4cfeee0e34 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 23:39:45 -0700 Subject: [PATCH 40/77] Move trailer writing into a single method --- .../src/Internal/Http2/Http2FrameWriter.cs | 52 +++++-------------- 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index cf43a0d46f73..9eb57c5044d1 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -173,15 +173,8 @@ private async Task WriteToOutputPipe() stream.ResponseTrailers.SetReadOnly(); stream.DecrementActiveClientStreamCount(); - if (buffer.Length > 0) - { - // It is faster to write data and trailers together. Locking once reduces lock contention. - flushResult = await WriteDataAndTrailersAsync(stream, buffer, producer.WriteHeaders, stream.ResponseTrailers); - } - else - { - flushResult = await WriteResponseTrailersAsync(stream, producer.WriteHeaders, stream.ResponseTrailers); - } + // It is faster to write data and trailers together. Locking once reduces lock contention. + flushResult = await WriteDataAndTrailersAsync(stream, buffer, producer.WriteHeaders, stream.ResponseTrailers); } else if (readResult.IsCompleted && producer.StreamEnded) { @@ -423,8 +416,11 @@ private void WriteResponseHeadersUnsynchronized(int streamId, int statusCode, Ht } } - private ValueTask WriteResponseTrailersAsync(Http2Stream stream, bool writeHeaders, HttpResponseTrailers headers) + private ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in ReadOnlySequence data, bool writeHeaders, HttpResponseTrailers headers) { + // The Length property of a ReadOnlySequence can be expensive, so we cache the value. + var dataLength = data.Length; + lock (_writeLock) { if (_completed) @@ -432,12 +428,18 @@ private ValueTask WriteResponseTrailersAsync(Http2Stream stream, bo return default; } + var streamId = stream.StreamId; + if (writeHeaders) { - WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + WriteResponseHeadersUnsynchronized(streamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); } - var streamId = stream.StreamId; + + if (dataLength > 0) + { + WriteDataUnsynchronized(streamId, data, dataLength, endStream: false); + } try { @@ -488,32 +490,6 @@ private void FinishWritingHeaders(int streamId, int payloadLength, bool done) } } - private ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in ReadOnlySequence data, bool writeHeaders, HttpResponseTrailers headers) - { - // This method combines WriteDataAsync and WriteResponseTrailers. - // Changes here may need to be mirrored in WriteDataAsync. - - // The Length property of a ReadOnlySequence can be expensive, so we cache the value. - var dataLength = data.Length; - - lock (_writeLock) - { - if (_completed) - { - return default; - } - - if (writeHeaders) - { - WriteResponseHeadersUnsynchronized(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); - } - - WriteDataUnsynchronized(stream.StreamId, data, dataLength, endStream: false); - - return WriteResponseTrailersAsync(stream, writeHeaders: false, headers); - } - } - /* Padding is not implemented +---------------+ |Pad Length? (8)| From 26265679e8e9e38fce7b1eda0fa8d1c74e0e99a4 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 28 Mar 2022 23:58:12 -0700 Subject: [PATCH 41/77] Remove extra lines... --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 9eb57c5044d1..db64a8dede33 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -435,7 +435,6 @@ private ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in WriteResponseHeadersUnsynchronized(streamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); } - if (dataLength > 0) { WriteDataUnsynchronized(streamId, data, dataLength, endStream: false); From 5b93caa8d1fa8d3431471c01096cab029cc8a53b Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 30 Mar 2022 22:19:33 -0700 Subject: [PATCH 42/77] Remove task chain allocation on shutdown - We were waiting on the response completion task in write suffix, this caused us to allocate an set of async state machines while unwinding the request processing. This changes the flow to avoid that await completely and leave the clean up of the stream to either the request processing loop or framewriter queue. --- .../src/Internal/Http2/Http2FrameWriter.cs | 2 +- .../src/Internal/Http2/Http2OutputProducer.cs | 56 ++++++++++++------- .../Core/src/Internal/Http2/Http2Stream.cs | 2 +- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index db64a8dede33..0832f2490105 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -216,7 +216,7 @@ private async Task WriteToOutputPipe() { await reader.CompleteAsync(); - producer.CompleteResponse(flushResult); + producer.CompleteResponse(); } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 80898b6c33ac..0b3cb711b282 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -4,7 +4,6 @@ using System.Buffers; using System.Diagnostics; using System.IO.Pipelines; -using System.Threading.Tasks.Sources; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Internal; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; @@ -14,7 +13,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; -internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IValueTaskSource, IDisposable +internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IDisposable { private int StreamId => _stream.StreamId; private readonly Http2FrameWriter _frameWriter; @@ -47,17 +46,8 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, IV // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state private State _observationState; private bool _waitingForWindowUpdates; - - /// The core logic for the IValueTaskSource implementation. - private ManualResetValueTaskSourceCore _responseCompleteTaskSource = new ManualResetValueTaskSourceCore { RunContinuationsAsynchronously = true }; // mutable struct, do not make this readonly - - // This object is itself usable as a backing source for ValueTask. Since there's only ever one awaiter - // for this object's state transitions at a time, we allow the object to be awaited directly. All functionality - // associated with the implementation is just delegated to the ManualResetValueTaskSourceCore. - private ValueTask GetWaiterTask() => new ValueTask(this, _responseCompleteTaskSource.Version); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _responseCompleteTaskSource.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _responseCompleteTaskSource.OnCompleted(continuation, state, token, flags); - FlushResult IValueTaskSource.GetResult(short token) => _responseCompleteTaskSource.GetResult(token); + private bool _completedResponse; + private bool _requestProcessingComplete; public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, StreamOutputFlowControl flowControl) { @@ -168,7 +158,7 @@ private bool UpdateWindow(long bytes) public void StreamReset(uint initialWindowSize) { // Response should have been completed. - Debug.Assert(_responseCompleteTaskSource.GetStatus(_responseCompleteTaskSource.Version) == ValueTaskSourceStatus.Succeeded); + Debug.Assert(_completedResponse); _streamEnded = false; _suffixSent = false; @@ -177,12 +167,13 @@ public void StreamReset(uint initialWindowSize) _writerComplete = false; _pipe.Reset(); _pipeWriter.Reset(); - _responseCompleteTaskSource.Reset(); _window = initialWindowSize; _unconsumedBytes = 0; _observationState = State.None; _waitingForWindowUpdates = false; + _completedResponse = false; + _requestProcessingComplete = false; WriteHeaders = false; IsTimingWrite = false; } @@ -390,7 +381,7 @@ public ValueTask WriteStreamSuffixAsync() { if (_streamCompleted) { - return GetWaiterTask(); + return ValueTask.FromResult(default); } _streamCompleted = true; @@ -406,7 +397,7 @@ public ValueTask WriteStreamSuffixAsync() Schedule(); } - return GetWaiterTask(); + return ValueTask.FromResult(default); } } @@ -562,9 +553,36 @@ public void Reset() { } - internal void CompleteResponse(in FlushResult flushResult) + internal void OnRequestProcessingEnded() { - _responseCompleteTaskSource.SetResult(flushResult); + lock (_dataWriterLock) + { + if (_requestProcessingComplete) + { + // Noop, we're done + return; + } + + _requestProcessingComplete = true; + + if (_completedResponse) + { + Stream.CompleteStream(errored: false); + } + } + } + + internal void CompleteResponse() + { + lock (_dataWriterLock) + { + _completedResponse = true; + + if (_requestProcessingComplete) + { + Stream.CompleteStream(errored: false); + } + } } internal Memory GetFakeMemory(int minSize) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index ad8fdb1797b0..d66407d0b43e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -129,7 +129,7 @@ protected override void OnReset() protected override void OnRequestProcessingEnded() { - CompleteStream(errored: false); + _http2Output.OnRequestProcessingEnded(); } public void CompleteStream(bool errored) From 66a6d7e73e86f3483628311f7a2601f6ecddaa02 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 30 Mar 2022 23:26:48 -0700 Subject: [PATCH 43/77] Removed the use of the output flow control objects from the source --- .../Kestrel/Core/src/Internal/Http2/Http2Connection.cs | 6 ++---- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 8 ++------ .../Core/src/Internal/Http2/Http2OutputProducer.cs | 7 +------ .../Kestrel/Core/src/Internal/Http2/Http2Stream.cs | 8 +------- .../Kestrel/Core/src/Internal/Http2/Http2StreamContext.cs | 6 +----- .../Kestrel/Core/test/Http2/Http2FrameWriterTests.cs | 2 +- src/Servers/Kestrel/Core/test/PooledStreamStackTests.cs | 3 +-- .../Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs | 2 +- src/Servers/Kestrel/shared/test/TestContextFactory.cs | 4 +--- 9 files changed, 11 insertions(+), 35 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index c7582924d3d2..696a68b9b1ba 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -40,7 +40,6 @@ internal partial class Http2Connection : IHttp2StreamLifetimeHandler, IHttpStrea private readonly int _minAllocBufferSize; private readonly HPackDecoder _hpackDecoder; private readonly InputFlowControl _inputFlowControl; - private readonly OutputFlowControl _outputFlowControl = new OutputFlowControl(new MultipleAwaitableProvider(), Http2PeerSettings.DefaultInitialWindowSize); private readonly Http2PeerSettings _serverSettings = new Http2PeerSettings(); private readonly Http2PeerSettings _clientSettings = new Http2PeerSettings(); @@ -90,7 +89,7 @@ public Http2Connection(HttpConnectionContext context) context.Transport.Output, context.ConnectionContext, this, - _outputFlowControl, + Http2PeerSettings.DefaultInitialWindowSize, context.TimeoutControl, httpLimits.MinResponseDataRate, context.ConnectionId, @@ -758,8 +757,7 @@ private Http2StreamContext CreateHttp2StreamContext() _clientSettings, _serverSettings, _frameWriter, - _inputFlowControl, - _outputFlowControl); + _inputFlowControl); streamContext.TimeoutControl = _context.TimeoutControl; streamContext.InitialExecutionContext = _context.InitialExecutionContext; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 0832f2490105..3ad1dc745a98 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -9,7 +9,6 @@ using System.Threading.Channels; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.FlowControl; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure.PipeWriterHelpers; using Microsoft.Extensions.Logging; @@ -28,7 +27,6 @@ internal class Http2FrameWriter private readonly ConcurrentPipeWriter _outputWriter; private readonly BaseConnectionContext _connectionContext; private readonly Http2Connection _http2Connection; - private readonly OutputFlowControl _connectionOutputFlowControl; private readonly string _connectionId; private readonly KestrelTrace _log; private readonly ITimeoutControl _timeoutControl; @@ -56,7 +54,7 @@ public Http2FrameWriter( PipeWriter outputPipeWriter, BaseConnectionContext connectionContext, Http2Connection http2Connection, - OutputFlowControl connectionOutputFlowControl, + long initialWindowSize, ITimeoutControl timeoutControl, MinDataRate? minResponseDataRate, string connectionId, @@ -67,7 +65,6 @@ public Http2FrameWriter( _outputWriter = new ConcurrentPipeWriter(outputPipeWriter, memoryPool, _writeLock); _connectionContext = connectionContext; _http2Connection = http2Connection; - _connectionOutputFlowControl = connectionOutputFlowControl; _connectionId = connectionId; _log = serviceContext.Log; _timeoutControl = timeoutControl; @@ -88,8 +85,7 @@ public Http2FrameWriter( SingleReader = true }); - // This is null in tests sometimes - _window = connectionOutputFlowControl?.Available ?? 0; + _window = initialWindowSize; _writeQueueTask = Task.Run(WriteToOutputPipe); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 0b3cb711b282..24bb5d8d643e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -7,7 +7,6 @@ using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Internal; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.FlowControl; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure.PipeWriterHelpers; @@ -20,9 +19,6 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private readonly TimingPipeFlusher _flusher; private readonly KestrelTrace _log; - // This should only be accessed via the FrameWriter. The connection-level output flow control is protected by the - // FrameWriter's connection-level write lock. - private readonly StreamOutputFlowControl _flowControl; private readonly MemoryPool _memoryPool; private readonly Http2Stream _stream; private readonly object _dataWriterLock = new object(); @@ -49,11 +45,10 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private bool _completedResponse; private bool _requestProcessingComplete; - public Http2OutputProducer(Http2Stream stream, Http2StreamContext context, StreamOutputFlowControl flowControl) + public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) { _stream = stream; _frameWriter = context.FrameWriter; - _flowControl = flowControl; _memoryPool = context.MemoryPool; _log = context.ServiceContext.Log; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index d66407d0b43e..c69f40bfa2df 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -21,7 +21,6 @@ internal abstract partial class Http2Stream : HttpProtocol, IThreadPoolWorkItem, private Http2StreamContext _context = default!; private Http2OutputProducer _http2Output = default!; private StreamInputFlowControl _inputFlowControl = default!; - private StreamOutputFlowControl _outputFlowControl = default!; private Http2MessageBody? _messageBody; private bool _decrementCalled; @@ -56,11 +55,7 @@ public void Initialize(Http2StreamContext context) context.ServerPeerSettings.InitialWindowSize, context.ServerPeerSettings.InitialWindowSize / 2); - _outputFlowControl = new StreamOutputFlowControl( - context.ConnectionOutputFlowControl, - context.ClientPeerSettings.InitialWindowSize); - - _http2Output = new Http2OutputProducer(this, context, _outputFlowControl); + _http2Output = new Http2OutputProducer(this, context); RequestBodyPipe = CreateRequestBodyPipe(); @@ -69,7 +64,6 @@ public void Initialize(Http2StreamContext context) else { _inputFlowControl.Reset(); - _outputFlowControl.Reset(context.ClientPeerSettings.InitialWindowSize); _http2Output.StreamReset(context.ClientPeerSettings.InitialWindowSize); RequestBodyPipe.Reset(); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2StreamContext.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2StreamContext.cs index d1e6c674615c..506a3d5fa835 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2StreamContext.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2StreamContext.cs @@ -25,8 +25,7 @@ public Http2StreamContext( Http2PeerSettings clientPeerSettings, Http2PeerSettings serverPeerSettings, Http2FrameWriter frameWriter, - InputFlowControl connectionInputFlowControl, - OutputFlowControl connectionOutputFlowControl) : base(connectionId, protocols, altSvcHeader, connectionContext: null!, serviceContext, connectionFeatures, memoryPool, localEndPoint, remoteEndPoint) + InputFlowControl connectionInputFlowControl) : base(connectionId, protocols, altSvcHeader, connectionContext: null!, serviceContext, connectionFeatures, memoryPool, localEndPoint, remoteEndPoint) { StreamId = streamId; StreamLifetimeHandler = streamLifetimeHandler; @@ -34,7 +33,6 @@ public Http2StreamContext( ServerPeerSettings = serverPeerSettings; FrameWriter = frameWriter; ConnectionInputFlowControl = connectionInputFlowControl; - ConnectionOutputFlowControl = connectionOutputFlowControl; } public IHttp2StreamLifetimeHandler StreamLifetimeHandler { get; } @@ -42,7 +40,5 @@ public Http2StreamContext( public Http2PeerSettings ServerPeerSettings { get; } public Http2FrameWriter FrameWriter { get; } public InputFlowControl ConnectionInputFlowControl { get; } - public OutputFlowControl ConnectionOutputFlowControl { get; } - public int StreamId { get; set; } } diff --git a/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs b/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs index ffb169170a6a..a8baef80169f 100644 --- a/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs +++ b/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs @@ -56,7 +56,7 @@ public async Task WriteWindowUpdate_UnsetsReservedBit() private Http2FrameWriter CreateFrameWriter(Pipe pipe) { var serviceContext = TestContextFactory.CreateServiceContext(new KestrelServerOptions()); - return new Http2FrameWriter(pipe.Writer, null, null, null, null, null, null, _dirtyMemoryPool, serviceContext); + return new Http2FrameWriter(pipe.Writer, null, null, 0, null, null, null, _dirtyMemoryPool, serviceContext); } [Fact] diff --git a/src/Servers/Kestrel/Core/test/PooledStreamStackTests.cs b/src/Servers/Kestrel/Core/test/PooledStreamStackTests.cs index efb2581a2ae5..af2d924c23f2 100644 --- a/src/Servers/Kestrel/Core/test/PooledStreamStackTests.cs +++ b/src/Servers/Kestrel/Core/test/PooledStreamStackTests.cs @@ -110,8 +110,7 @@ private static Http2Stream CreateStream(int streamId, long expirati clientPeerSettings: new Http2PeerSettings(), serverPeerSettings: new Http2PeerSettings(), frameWriter: null!, - connectionInputFlowControl: null!, - connectionOutputFlowControl: null! + connectionInputFlowControl: null! ); return new Http2Stream(new DummyApplication(), context) diff --git a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs index bb5d2d7d8b1e..dd96c2d126a8 100644 --- a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs +++ b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs @@ -39,7 +39,7 @@ public void GlobalSetup() new NullPipeWriter(), connectionContext: null, http2Connection: null, - new OutputFlowControl(new SingleAwaitableProvider(), initialWindowSize: int.MaxValue), + int.MaxValue, timeoutControl: null, minResponseDataRate: null, "TestConnectionId", diff --git a/src/Servers/Kestrel/shared/test/TestContextFactory.cs b/src/Servers/Kestrel/shared/test/TestContextFactory.cs index 14df1153cbd8..3e3ed65b8ebb 100644 --- a/src/Servers/Kestrel/shared/test/TestContextFactory.cs +++ b/src/Servers/Kestrel/shared/test/TestContextFactory.cs @@ -141,7 +141,6 @@ public static Http2StreamContext CreateHttp2StreamContext( Http2PeerSettings serverPeerSettings = null, Http2FrameWriter frameWriter = null, InputFlowControl connectionInputFlowControl = null, - OutputFlowControl connectionOutputFlowControl = null, ITimeoutControl timeoutControl = null) { var context = new Http2StreamContext @@ -159,8 +158,7 @@ public static Http2StreamContext CreateHttp2StreamContext( clientPeerSettings: clientPeerSettings ?? new Http2PeerSettings(), serverPeerSettings: serverPeerSettings ?? new Http2PeerSettings(), frameWriter: frameWriter, - connectionInputFlowControl: connectionInputFlowControl, - connectionOutputFlowControl: connectionOutputFlowControl + connectionInputFlowControl: connectionInputFlowControl ); context.TimeoutControl = timeoutControl; From 56481d69042b29351bcd01fd2439bde69e9170d7 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 30 Mar 2022 23:28:48 -0700 Subject: [PATCH 44/77] Delete output flow control types --- .../Http2/FlowControl/AwaitableProvider.cs | 94 ------------------ .../Http2/FlowControl/OutputFlowControl.cs | 74 -------------- .../FlowControl/StreamOutputFlowControl.cs | 97 ------------------- 3 files changed, 265 deletions(-) delete mode 100644 src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/AwaitableProvider.cs delete mode 100644 src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/OutputFlowControl.cs delete mode 100644 src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/StreamOutputFlowControl.cs diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/AwaitableProvider.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/AwaitableProvider.cs deleted file mode 100644 index 4ef7703b174b..000000000000 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/AwaitableProvider.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Threading.Tasks.Sources; - -namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.FlowControl; - -internal abstract class AwaitableProvider -{ - public abstract ManualResetValueTaskSource GetAwaitable(); - public abstract void CompleteCurrent(); - public abstract int ActiveCount { get; } -} - -/// -/// Provider returns multiple awaitables. Awaitables are completed FIFO. -/// -internal class MultipleAwaitableProvider : AwaitableProvider -{ - private Queue>? _awaitableQueue; - private Queue>? _awaitableCache; - - public override void CompleteCurrent() - { - Debug.Assert(_awaitableQueue != null); - Debug.Assert(_awaitableCache != null); - - var awaitable = _awaitableQueue.Dequeue(); - awaitable.TrySetResult(null); - - // Add completed awaitable to the cache for reuse - _awaitableCache.Enqueue(awaitable); - } - - public override ManualResetValueTaskSource GetAwaitable() - { - if (_awaitableQueue == null) - { - _awaitableQueue = new Queue>(); - _awaitableCache = new Queue>(); - } - - // First attempt to reuse an existing awaitable in the queue - // to save allocating a new instance. - if (_awaitableCache!.TryDequeue(out var awaitable)) - { - // Reset previously used awaitable - Debug.Assert(awaitable.GetStatus() == ValueTaskSourceStatus.Succeeded, "Previous awaitable should have been completed."); - awaitable.Reset(); - } - else - { - awaitable = new ManualResetValueTaskSource(); - } - - _awaitableQueue.Enqueue(awaitable); - - return awaitable; - } - - public override int ActiveCount => _awaitableQueue?.Count ?? 0; -} - -/// -/// Provider has a single awaitable. -/// -internal class SingleAwaitableProvider : AwaitableProvider -{ - private ManualResetValueTaskSource? _awaitable; - - public override void CompleteCurrent() - { - Debug.Assert(_awaitable != null); - _awaitable.TrySetResult(null); - } - - public override ManualResetValueTaskSource GetAwaitable() - { - if (_awaitable == null) - { - _awaitable = new ManualResetValueTaskSource(); - } - else - { - Debug.Assert(_awaitable.GetStatus() == ValueTaskSourceStatus.Succeeded, "Previous awaitable should have been completed."); - _awaitable.Reset(); - } - - return _awaitable; - } - - public override int ActiveCount => _awaitable != null && _awaitable.GetStatus() != ValueTaskSourceStatus.Succeeded ? 1 : 0; -} diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/OutputFlowControl.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/OutputFlowControl.cs deleted file mode 100644 index 7570dead8843..000000000000 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/OutputFlowControl.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; - -namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.FlowControl; - -internal class OutputFlowControl -{ - private FlowControl _flow; - private readonly AwaitableProvider _awaitableProvider; - - public OutputFlowControl(AwaitableProvider awaitableProvider, uint initialWindowSize) - { - _flow = new FlowControl(initialWindowSize); - _awaitableProvider = awaitableProvider; - } - - public int Available => _flow.Available; - public bool IsAborted => _flow.IsAborted; - - public ManualResetValueTaskSource AvailabilityAwaitable - { - get - { - Debug.Assert(!_flow.IsAborted, $"({nameof(AvailabilityAwaitable)} accessed after abort."); - Debug.Assert(_flow.Available <= 0, $"({nameof(AvailabilityAwaitable)} accessed with {Available} bytes available."); - - return _awaitableProvider.GetAwaitable(); - } - } - - public void Reset(uint initialWindowSize) - { - // When output flow control is reused the client window size needs to be reset. - // The client might have changed the window size before the stream is reused. - _flow = new FlowControl(initialWindowSize); - Debug.Assert(_awaitableProvider.ActiveCount == 0, "Queue should have been emptied by the previous stream."); - } - - public void Advance(int bytes) - { - _flow.Advance(bytes); - } - - // bytes can be negative when SETTINGS_INITIAL_WINDOW_SIZE decreases mid-connection. - // This can also cause Available to become negative which MUST be allowed. - // https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.2 - public bool TryUpdateWindow(int bytes) - { - if (_flow.TryUpdateWindow(bytes)) - { - while (_flow.Available > 0 && _awaitableProvider.ActiveCount > 0) - { - _awaitableProvider.CompleteCurrent(); - } - - return true; - } - - return false; - } - - public void Abort() - { - // Make sure to set the aborted flag before running any continuations. - _flow.Abort(); - - while (_awaitableProvider.ActiveCount > 0) - { - _awaitableProvider.CompleteCurrent(); - } - } -} diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/StreamOutputFlowControl.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/StreamOutputFlowControl.cs deleted file mode 100644 index 100a0cb34548..000000000000 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/FlowControl/StreamOutputFlowControl.cs +++ /dev/null @@ -1,97 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Threading.Tasks.Sources; - -namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2.FlowControl; - -internal class StreamOutputFlowControl -{ - private readonly OutputFlowControl _connectionLevelFlowControl; - private readonly OutputFlowControl _streamLevelFlowControl; - - private ManualResetValueTaskSource? _currentConnectionLevelAwaitable; - private int _currentConnectionLevelAwaitableVersion; - - public StreamOutputFlowControl(OutputFlowControl connectionLevelFlowControl, uint initialWindowSize) - { - _connectionLevelFlowControl = connectionLevelFlowControl; - _streamLevelFlowControl = new OutputFlowControl(new SingleAwaitableProvider(), initialWindowSize); - } - - public int Available => Math.Min(_connectionLevelFlowControl.Available, _streamLevelFlowControl.Available); - - public bool IsAborted => _connectionLevelFlowControl.IsAborted || _streamLevelFlowControl.IsAborted; - - public void Reset(uint initialWindowSize) - { - _streamLevelFlowControl.Reset(initialWindowSize); - if (_currentConnectionLevelAwaitable != null) - { - Debug.Assert(_currentConnectionLevelAwaitable.GetStatus() == ValueTaskSourceStatus.Succeeded, "Should have been completed by the previous stream."); - _currentConnectionLevelAwaitable = null; - _currentConnectionLevelAwaitableVersion = -1; - } - } - - public void Advance(int bytes) - { - _connectionLevelFlowControl.Advance(bytes); - _streamLevelFlowControl.Advance(bytes); - } - - public int AdvanceUpToAndWait(long bytes, out ValueTask availabilityTask) - { - var leastAvailableFlow = _connectionLevelFlowControl.Available < _streamLevelFlowControl.Available - ? _connectionLevelFlowControl : _streamLevelFlowControl; - - // This cast is safe because leastAvailableFlow.Available is an int. - var actual = (int)Math.Clamp(leastAvailableFlow.Available, 0, bytes); - - // Make sure to advance prior to accessing AvailabilityAwaitable. - _connectionLevelFlowControl.Advance(actual); - _streamLevelFlowControl.Advance(actual); - - availabilityTask = default; - _currentConnectionLevelAwaitable = null; - _currentConnectionLevelAwaitableVersion = -1; - - if (actual < bytes) - { - var awaitable = leastAvailableFlow.AvailabilityAwaitable; - - if (leastAvailableFlow == _connectionLevelFlowControl) - { - _currentConnectionLevelAwaitable = awaitable; - _currentConnectionLevelAwaitableVersion = awaitable.Version; - } - - availabilityTask = new ValueTask(awaitable, awaitable.Version); - } - - return actual; - } - - // The connection-level update window is updated independently. - // https://httpwg.org/specs/rfc7540.html#rfc.section.6.9.1 - public bool TryUpdateWindow(int bytes) - { - return _streamLevelFlowControl.TryUpdateWindow(bytes); - } - - public void Abort() - { - _streamLevelFlowControl.Abort(); - - // If this stream is waiting on a connection-level window update, complete this stream's - // connection-level awaitable so the stream abort is observed immediately. - // This could complete an awaitable still sitting in the connection-level awaitable queue, - // but this is safe because completing it again will just no-op. - if (_currentConnectionLevelAwaitable != null && - _currentConnectionLevelAwaitable.Version == _currentConnectionLevelAwaitableVersion) - { - _currentConnectionLevelAwaitable.TrySetResult(null); - } - } -} From 2388dce3c026487dc4da314042521bed3d4795d1 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 31 Mar 2022 10:08:57 -0700 Subject: [PATCH 45/77] Make ConsumeWindow private --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 3ad1dc745a98..8754a3bc1d50 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -838,7 +838,7 @@ private ValueTask TimeFlushUnsynchronizedAsync() return _flusher.FlushAsync(_minResponseDataRate, bytesWritten); } - internal (long, long) ConsumeWindow(long bytes) + private (long, long) ConsumeWindow(long bytes) { lock (_windowUpdateLock) { From 9b3be394c41bd90025997e449874d7b4f04a7900 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 31 Mar 2022 19:53:39 -0700 Subject: [PATCH 46/77] Remove if from queue processing --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 8754a3bc1d50..94ac406792fc 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -106,17 +106,10 @@ private async Task WriteToOutputPipe() var observed = Http2OutputProducer.State.None; - if (!reader.TryRead(out var readResult)) - { - if (producer.WriteHeaders) - { - // Flush headers, we have nothing to look at on the pipe - } - else - { - readResult = await reader.ReadAsync(); - } - } + // We don't need to check the result because it's either + // - true because we have a result + // - false because we're flushing headers + var hasResult = reader.TryRead(out var readResult); var buffer = readResult.Buffer; From de5787e78906ea7ffcc1edb51334933bbf00eba8 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 31 Mar 2022 21:15:21 -0700 Subject: [PATCH 47/77] Oops --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 94ac406792fc..c9de58f74a11 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -109,7 +109,7 @@ private async Task WriteToOutputPipe() // We don't need to check the result because it's either // - true because we have a result // - false because we're flushing headers - var hasResult = reader.TryRead(out var readResult); + reader.TryRead(out var readResult); var buffer = readResult.Buffer; From aa495ea2b32b9dfd56d92c1e65e7dd0b9f46ca0a Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 31 Mar 2022 21:40:33 -0700 Subject: [PATCH 48/77] Fixed logs --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index c9de58f74a11..e546e5a42bf6 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -169,8 +169,7 @@ private async Task WriteToOutputPipe() { if (buffer.Length != 0) { - // TODO: Use the right logger here. - _log.LogCritical(nameof(Http2OutputProducer) + "." + " observed an unexpected state where the streams output ended with data still remaining in the pipe."); + _log.LogCritical("{StreamId} observed an unexpected state where the streams output ended with data still remaining in the pipe.", stream.StreamId); } else { From 9f54e60a49f828fd6bbacee0a2d6ca3754fe0b91 Mon Sep 17 00:00:00 2001 From: James Newton-King Date: Mon, 4 Apr 2022 08:25:35 +0800 Subject: [PATCH 49/77] Fix merge --- .../Http2/Http2FrameWriterBenchmark.cs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs index dd96c2d126a8..8f03e5013e9c 100644 --- a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs +++ b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs @@ -20,7 +20,6 @@ public class Http2FrameWriterBenchmark private Pipe _pipe; private Http2FrameWriter _frameWriter; private HttpResponseHeaders _responseHeaders; - private Http2Stream _stream; [GlobalSetup] public void GlobalSetup() @@ -46,8 +45,6 @@ public void GlobalSetup() _memoryPool, serviceContext); - _stream = new MockHttp2Stream(TestContextFactory.CreateHttp2StreamContext(streamId: 0)); - _responseHeaders = new HttpResponseHeaders(); var headers = (IHeaderDictionary)_responseHeaders; headers.ContentType = "application/json"; @@ -57,7 +54,7 @@ public void GlobalSetup() [Benchmark] public void WriteResponseHeaders() { - _frameWriter.WriteResponseHeaders(_stream, 200, endStream: true, _responseHeaders); + _frameWriter.WriteResponseHeaders(streamId: 0, 200, Http2HeadersFrameFlags.END_STREAM, _responseHeaders); } [GlobalCleanup] @@ -66,16 +63,4 @@ public void Dispose() _pipe.Writer.Complete(); _memoryPool?.Dispose(); } - - private class MockHttp2Stream : Http2Stream - { - public MockHttp2Stream(Http2StreamContext context) - { - Initialize(context); - } - - public override void Execute() - { - } - } } From cb0e7e169ff0dd4bfc043508ee063f0ba6f212c4 Mon Sep 17 00:00:00 2001 From: Stephen Halter Date: Fri, 8 Apr 2022 17:12:40 -0700 Subject: [PATCH 50/77] Don't reschedule for nothing --- .../src/Internal/Http2/Http2Connection.cs | 23 +++++------ .../src/Internal/Http2/Http2FrameWriter.cs | 39 ++++++++++++------- .../src/Internal/Http2/Http2OutputProducer.cs | 10 ++++- .../Core/test/Http2/Http2FrameWriterTests.cs | 2 +- .../Http2/Http2FrameWriterBenchmark.cs | 3 +- 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 696a68b9b1ba..7b2e334f0821 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -85,17 +85,6 @@ public Http2Connection(HttpConnectionContext context) _input = new Pipe(GetInputPipeOptions()); - _frameWriter = new Http2FrameWriter( - context.Transport.Output, - context.ConnectionContext, - this, - Http2PeerSettings.DefaultInitialWindowSize, - context.TimeoutControl, - httpLimits.MinResponseDataRate, - context.ConnectionId, - context.MemoryPool, - context.ServiceContext); - _minAllocBufferSize = context.MemoryPool.GetMinimumAllocSize(); _hpackDecoder = new HPackDecoder(http2Limits.HeaderTableSize, http2Limits.MaxRequestHeaderFieldSize); @@ -123,6 +112,18 @@ public Http2Connection(HttpConnectionContext context) _scheduleInline = context.ServiceContext.Scheduler == PipeScheduler.Inline; _inputTask = CopyPipeAsync(_context.Transport.Input, _input.Writer); + + _frameWriter = new Http2FrameWriter( + context.Transport.Output, + context.ConnectionContext, + this, + MaxTrackedStreams, + http2Limits.MaxStreamsPerConnection, + context.TimeoutControl, + httpLimits.MinResponseDataRate, + context.ConnectionId, + context.MemoryPool, + context.ServiceContext); } public string ConnectionId => _context.ConnectionId; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index e546e5a42bf6..20c39ace2073 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -46,7 +46,7 @@ internal class Http2FrameWriter private bool _aborted; private readonly object _windowUpdateLock = new(); - private long _window; + private long _connectionWindow; private readonly Queue _waitingForMoreWindow = new(); private readonly Task _writeQueueTask; @@ -54,7 +54,8 @@ public Http2FrameWriter( PipeWriter outputPipeWriter, BaseConnectionContext connectionContext, Http2Connection http2Connection, - long initialWindowSize, + long initialConnectionWindowSize, + int maxStreamsPerConnection, ITimeoutControl timeoutControl, MinDataRate? minResponseDataRate, string connectionId, @@ -78,21 +79,33 @@ public Http2FrameWriter( _hpackEncoder = new DynamicHPackEncoder(serviceContext.ServerOptions.AllowResponseHeaderCompression); - // In practice, this is bounded by the number of concurrent streams allowed + whatever overflow we allow - _channel = Channel.CreateUnbounded(new UnboundedChannelOptions() + // This is bounded by the maximum number of concurrent Http2Streams per Http2Connection. + // This isn't the same as SETTINGS_MAX_CONCURRENT_STREAMS, but typically double (with a floor of 100) + // which is the max number of Http2Streams that can end up in the Http2Connection._streams dictionary. + // + // Setting a lower limit of SETTINGS_MAX_CONCURRENT_STREAMS might be sufficient because a stream shouldn't + // be rescheduling itself after being completed or canceled, but we're going with the more conservative limit + // in case there's some logic scheduling completed or canceled streams unnecessarily. + _channel = Channel.CreateBounded(new BoundedChannelOptions(maxStreamsPerConnection) { AllowSynchronousContinuations = _scheduleInline, SingleReader = true }); - _window = initialWindowSize; + _connectionWindow = initialConnectionWindowSize; _writeQueueTask = Task.Run(WriteToOutputPipe); } public void Schedule(Http2OutputProducer producer) { - _channel.Writer.TryWrite(producer); + if (!_channel.Writer.TryWrite(producer)) + { + // It should not be possible to exceed the bound of the channel. + var ex = new ConnectionAbortedException("Http2FrameWriter._channel exceeded bounds."); + _log.LogCritical(ex, "Http2FrameWriter._channel exceeded bounds.", _connectionId); + _http2Connection.Abort(ex); + } } private async Task WriteToOutputPipe() @@ -232,7 +245,7 @@ private async Task WriteToOutputPipe() else if (remainingStream > 0) { // Move this stream to the back of the queue so we're being fair to the other streams that have data - Schedule(producer); + producer.Schedule(); } else { @@ -249,7 +262,7 @@ private async Task WriteToOutputPipe() } else if (reschedule) { - Schedule(producer); + producer.Schedule(); } } catch (Exception ex) @@ -834,8 +847,8 @@ private ValueTask TimeFlushUnsynchronizedAsync() { lock (_windowUpdateLock) { - var actual = Math.Min(bytes, _window); - var remaining = _window -= actual; + var actual = Math.Min(bytes, _connectionWindow); + var remaining = _connectionWindow -= actual; return (actual, remaining); } @@ -860,14 +873,14 @@ public bool TryUpdateConnectionWindow(int bytes) { lock (_windowUpdateLock) { - var maxUpdate = Http2PeerSettings.MaxWindowSize - _window; + var maxUpdate = Http2PeerSettings.MaxWindowSize - _connectionWindow; if (bytes > maxUpdate) { return false; } - _window += bytes; + _connectionWindow += bytes; while (_waitingForMoreWindow.TryDequeue(out var producer)) { @@ -876,7 +889,7 @@ public bool TryUpdateConnectionWindow(int bytes) // We're no longer waiting for the update producer.MarkWaitingForWindowUpdates(false); - Schedule(producer); + producer.Schedule(); } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 24bb5d8d643e..97dc3c169958 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -32,6 +32,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private bool _suffixSent; private bool _streamEnded; private bool _writerComplete; + private bool _isScheduled; // Internal for testing internal bool _disposed; @@ -120,6 +121,7 @@ private bool EnqueueForObservation(State state) { lock (_dataWriterLock) { + _isScheduled = false; _observationState &= ~state; _unconsumedBytes -= bytes; return (_unconsumedBytes > 0, _observationState != State.None); @@ -282,12 +284,18 @@ public void MarkWaitingForWindowUpdates(bool waitingForUpdates) } } - private void Schedule() + public void Schedule() { lock (_dataWriterLock) { // Lock here + if (_isScheduled) + { + return; + } + _waitingForWindowUpdates = false; + _isScheduled = true; } _frameWriter.Schedule(this); diff --git a/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs b/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs index a8baef80169f..1068f2ad042b 100644 --- a/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs +++ b/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs @@ -56,7 +56,7 @@ public async Task WriteWindowUpdate_UnsetsReservedBit() private Http2FrameWriter CreateFrameWriter(Pipe pipe) { var serviceContext = TestContextFactory.CreateServiceContext(new KestrelServerOptions()); - return new Http2FrameWriter(pipe.Writer, null, null, 0, null, null, null, _dirtyMemoryPool, serviceContext); + return new Http2FrameWriter(pipe.Writer, null, null, 0, 0, null, null, null, _dirtyMemoryPool, serviceContext); } [Fact] diff --git a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs index 8f03e5013e9c..4159949561d3 100644 --- a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs +++ b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs @@ -38,7 +38,8 @@ public void GlobalSetup() new NullPipeWriter(), connectionContext: null, http2Connection: null, - int.MaxValue, + 0, + 0, timeoutControl: null, minResponseDataRate: null, "TestConnectionId", From 9b425616ee0483072052dff6fafd61e12afa708c Mon Sep 17 00:00:00 2001 From: Stephen Halter Date: Mon, 11 Apr 2022 10:08:52 -0700 Subject: [PATCH 51/77] Fix typo --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs | 3 +-- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 3 +-- src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs | 2 +- .../perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index 7b2e334f0821..ff66fabbe52d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -117,8 +117,7 @@ public Http2Connection(HttpConnectionContext context) context.Transport.Output, context.ConnectionContext, this, - MaxTrackedStreams, - http2Limits.MaxStreamsPerConnection, + (int)MaxTrackedStreams, context.TimeoutControl, httpLimits.MinResponseDataRate, context.ConnectionId, diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 20c39ace2073..78f05ffb144e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -54,7 +54,6 @@ public Http2FrameWriter( PipeWriter outputPipeWriter, BaseConnectionContext connectionContext, Http2Connection http2Connection, - long initialConnectionWindowSize, int maxStreamsPerConnection, ITimeoutControl timeoutControl, MinDataRate? minResponseDataRate, @@ -92,7 +91,7 @@ public Http2FrameWriter( SingleReader = true }); - _connectionWindow = initialConnectionWindowSize; + _connectionWindow = Http2PeerSettings.DefaultInitialWindowSize; _writeQueueTask = Task.Run(WriteToOutputPipe); } diff --git a/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs b/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs index 1068f2ad042b..91d2a1c58734 100644 --- a/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs +++ b/src/Servers/Kestrel/Core/test/Http2/Http2FrameWriterTests.cs @@ -56,7 +56,7 @@ public async Task WriteWindowUpdate_UnsetsReservedBit() private Http2FrameWriter CreateFrameWriter(Pipe pipe) { var serviceContext = TestContextFactory.CreateServiceContext(new KestrelServerOptions()); - return new Http2FrameWriter(pipe.Writer, null, null, 0, 0, null, null, null, _dirtyMemoryPool, serviceContext); + return new Http2FrameWriter(pipe.Writer, null, null, 1, null, null, null, _dirtyMemoryPool, serviceContext); } [Fact] diff --git a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs index 4159949561d3..a80186a0af60 100644 --- a/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs +++ b/src/Servers/Kestrel/perf/Microbenchmarks/Http2/Http2FrameWriterBenchmark.cs @@ -38,8 +38,7 @@ public void GlobalSetup() new NullPipeWriter(), connectionContext: null, http2Connection: null, - 0, - 0, + maxStreamsPerConnection: 1, timeoutControl: null, minResponseDataRate: null, "TestConnectionId", From a1d56b889c964c7fe52718952aa813e8f960247c Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 18:58:24 -0700 Subject: [PATCH 52/77] Apply PR feedback --- .../src/Internal/Http2/Http2Connection.cs | 2 +- .../src/Internal/Http2/Http2FrameWriter.cs | 240 +++++++++--------- .../src/Internal/Http2/Http2OutputProducer.cs | 86 +++---- .../PipeWriterHelpers/ConcurrentPipeWriter.cs | 4 - 4 files changed, 153 insertions(+), 179 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs index ff66fabbe52d..68eaf98be9b8 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs @@ -117,7 +117,7 @@ public Http2Connection(HttpConnectionContext context) context.Transport.Output, context.ConnectionContext, this, - (int)MaxTrackedStreams, + (int)Math.Min(MaxTrackedStreams, int.MaxValue), context.TimeoutControl, httpLimits.MinResponseDataRate, context.ConnectionId, diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 78f05ffb144e..ca14c938e28e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -109,165 +109,168 @@ public void Schedule(Http2OutputProducer producer) private async Task WriteToOutputPipe() { - await foreach (var producer in _channel.Reader.ReadAllAsync()) + while (await _channel.Reader.WaitToReadAsync()) { - try + while (_channel.Reader.TryRead(out var producer)) { - var reader = producer.PipeReader; - var stream = producer.Stream; + try + { + var reader = producer.PipeReader; + var stream = producer.Stream; - var observed = Http2OutputProducer.State.None; + var observed = Http2OutputProducer.State.None; - // We don't need to check the result because it's either - // - true because we have a result - // - false because we're flushing headers - reader.TryRead(out var readResult); + // We don't need to check the result because it's either + // - true because we have a result + // - false because we're flushing headers + reader.TryRead(out var readResult); - var buffer = readResult.Buffer; + var buffer = readResult.Buffer; - if (producer.WriteHeaders) - { - observed |= Http2OutputProducer.State.FlushHeaders; - } + if (producer.WriteHeaders) + { + observed |= Http2OutputProducer.State.FlushHeaders; + } - if (readResult.IsCanceled) - { - observed |= Http2OutputProducer.State.Canceled; - } + if (readResult.IsCanceled) + { + observed |= Http2OutputProducer.State.Canceled; + } - if (readResult.IsCompleted) - { - observed |= Http2OutputProducer.State.Completed; - } + if (readResult.IsCompleted) + { + observed |= Http2OutputProducer.State.Completed; + } - // Check the stream window - var (actual, remainingStream) = producer.ConsumeWindow(buffer.Length); + // Check the stream window + var (actual, remainingStream) = producer.ConsumeStreamWindow(buffer.Length); - // Now check the connection window - (actual, var remainingConnection) = ConsumeWindow(actual); + // Now check the connection window + (actual, var remainingConnection) = ConsumeConnectionWindow(actual); - // Write what we can - if (actual < buffer.Length) - { - buffer = buffer.Slice(0, actual); - } + // Write what we can + if (actual < buffer.Length) + { + buffer = buffer.Slice(0, actual); + } - var (hasMoreData, reschedule) = producer.Dequeue(buffer.Length, observed); + var (hasMoreData, reschedule) = producer.ObserveDataAndState(buffer.Length, observed); - FlushResult flushResult = default; + FlushResult flushResult = default; - if (readResult.IsCanceled) - { - // Response body is aborted, complete reader for this output producer. - if (producer.WriteHeaders) + if (readResult.IsCanceled) { - // write headers - WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + // Response body is aborted, complete reader for this output producer. + if (producer.WriteHeaders) + { + // write headers + WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); + } } - } - else if (readResult.IsCompleted && stream.ResponseTrailers is { Count: > 0 } && !hasMoreData) - { - // Output is ending and there are trailers to write - // Write any remaining content then write trailers and there's no - // flow control back pressure being applied (hasMoreData) + else if (readResult.IsCompleted && stream.ResponseTrailers is { Count: > 0 } && !hasMoreData) + { + // Output is ending and there are trailers to write + // Write any remaining content then write trailers and there's no + // flow control back pressure being applied (hasMoreData) - stream.ResponseTrailers.SetReadOnly(); - stream.DecrementActiveClientStreamCount(); + stream.ResponseTrailers.SetReadOnly(); + stream.DecrementActiveClientStreamCount(); - // It is faster to write data and trailers together. Locking once reduces lock contention. - flushResult = await WriteDataAndTrailersAsync(stream, buffer, producer.WriteHeaders, stream.ResponseTrailers); - } - else if (readResult.IsCompleted && producer.StreamEnded) - { - if (buffer.Length != 0) + // It is faster to write data and trailers together. Locking once reduces lock contention. + flushResult = await WriteDataAndTrailersAsync(stream, buffer, producer.WriteHeaders, stream.ResponseTrailers); + } + else if (readResult.IsCompleted && producer.StreamEnded) { - _log.LogCritical("{StreamId} observed an unexpected state where the streams output ended with data still remaining in the pipe.", stream.StreamId); + if (buffer.Length != 0) + { + _log.LogCritical("{StreamId} observed an unexpected state where the streams output ended with data still remaining in the pipe.", stream.StreamId); + } + else + { + stream.DecrementActiveClientStreamCount(); + + // Headers have already been written and there is no other content to write + flushResult = await FlushAsync(stream, producer.WriteHeaders, outputAborter: null, cancellationToken: default); + } } else { - stream.DecrementActiveClientStreamCount(); + var endStream = readResult.IsCompleted && !hasMoreData; + + if (endStream) + { + stream.DecrementActiveClientStreamCount(); + } - // Headers have already been written and there is no other content to write - flushResult = await FlushAsync(stream, producer.WriteHeaders, outputAborter: null, cancellationToken: default); + flushResult = await WriteDataAsync(stream, buffer, buffer.Length, endStream, producer.WriteHeaders); } - } - else - { - var endStream = readResult.IsCompleted && !hasMoreData; - if (endStream) + if (producer.IsTimingWrite) { - stream.DecrementActiveClientStreamCount(); + _timeoutControl.StopTimingWrite(); } - flushResult = await WriteDataAsync(stream, buffer, buffer.Length, endStream, producer.WriteHeaders); - } + producer.WriteHeaders = false; - if (producer.IsTimingWrite) - { - _timeoutControl.StopTimingWrite(); - } - - producer.WriteHeaders = false; - - reader.AdvanceTo(buffer.End); + reader.AdvanceTo(buffer.End); - if ((readResult.IsCompleted && !hasMoreData) || readResult.IsCanceled) - { - await reader.CompleteAsync(); - - producer.CompleteResponse(); - } - // We're not going to schedule this again if there's no remaining window. - // When the window update is sent, the producer will be re-queued if needed. - else if (hasMoreData) - { - // We have no more connection window, put this producer in a queue waiting for it to - // a window update to resume the connection. - if (remainingConnection == 0) + if ((readResult.IsCompleted && !hasMoreData) || readResult.IsCanceled) { - // Mark the output as waiting for a window upate to resume writing (there's still data) - producer.MarkWaitingForWindowUpdates(true); + await reader.CompleteAsync(); - lock (_windowUpdateLock) + producer.CompleteResponse(); + } + // We're not going to schedule this again if there's no remaining window. + // When the window update is sent, the producer will be re-queued if needed. + else if (hasMoreData) + { + // We have no more connection window, put this producer in a queue waiting for it to + // a window update to resume the connection. + if (remainingConnection == 0) { - _waitingForMoreWindow.Enqueue(producer); + // Mark the output as waiting for a window upate to resume writing (there's still data) + producer.MarkWaitingForWindowUpdates(true); + + lock (_windowUpdateLock) + { + _waitingForMoreWindow.Enqueue(producer); + } + + // Include waiting for window updates in timing writes + if (_minResponseDataRate != null) + { + producer.IsTimingWrite = true; + _timeoutControl.StartTimingWrite(); + } } - - // Include waiting for window updates in timing writes - if (_minResponseDataRate != null) + else if (remainingStream > 0) + { + // Move this stream to the back of the queue so we're being fair to the other streams that have data + producer.Schedule(); + } + else { - producer.IsTimingWrite = true; - _timeoutControl.StartTimingWrite(); + // Mark the output as waiting for a window upate to resume writing (there's still data) + producer.MarkWaitingForWindowUpdates(true); + + // Include waiting for window updates in timing writes + if (_minResponseDataRate != null) + { + producer.IsTimingWrite = true; + _timeoutControl.StartTimingWrite(); + } } } - else if (remainingStream > 0) + else if (reschedule) { - // Move this stream to the back of the queue so we're being fair to the other streams that have data producer.Schedule(); } - else - { - // Mark the output as waiting for a window upate to resume writing (there's still data) - producer.MarkWaitingForWindowUpdates(true); - - // Include waiting for window updates in timing writes - if (_minResponseDataRate != null) - { - producer.IsTimingWrite = true; - _timeoutControl.StartTimingWrite(); - } - } } - else if (reschedule) + catch (Exception ex) { - producer.Schedule(); + _log.LogCritical(ex, "The event loop in connection {ConnectionId} failed unexpectedly", _connectionId); } } - catch (Exception ex) - { - _log.LogCritical(ex, "The event loop in connection {ConnectionId} failed unexpectedly", _connectionId); - } } _log.LogDebug("The connection processing loop for {ConnectionId} ended gracefully", _connectionId); @@ -842,14 +845,13 @@ private ValueTask TimeFlushUnsynchronizedAsync() return _flusher.FlushAsync(_minResponseDataRate, bytesWritten); } - private (long, long) ConsumeWindow(long bytes) + private (long, long) ConsumeConnectionWindow(long bytes) { lock (_windowUpdateLock) { var actual = Math.Min(bytes, _connectionWindow); - var remaining = _connectionWindow -= actual; - - return (actual, remaining); + _connectionWindow -= actual; + return (actual, _connectionWindow); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 97dc3c169958..a8a3626a9f93 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -38,7 +38,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID internal bool _disposed; private long _unconsumedBytes; - private long _window; + private long _streamWindow; // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state private State _observationState; @@ -56,14 +56,13 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) _pipe = CreateDataPipe(_memoryPool); _pipeWriter = new ConcurrentPipeWriter(_pipe.Writer, _memoryPool, _dataWriterLock); - Debug.Assert(_pipeWriter.CanGetUnflushedBytes); _pipeReader = _pipe.Reader; // No need to pass in timeoutControl here, since no minDataRates are passed to the TimingPipeFlusher. // The minimum output data rate is enforced at the connection level by Http2FrameWriter. _flusher = new TimingPipeFlusher(timeoutControl: null, _log); _flusher.Initialize(_pipeWriter); - _window = context.ClientPeerSettings.InitialWindowSize; + _streamWindow = context.ClientPeerSettings.InitialWindowSize; } public Http2Stream Stream => _stream; @@ -87,37 +86,35 @@ public bool StreamCompleted } // Useful for debugging the scheduling state in the debugger - internal (int, long, State, long, bool) SchedulingState => (Stream.StreamId, _unconsumedBytes, _observationState, _window, _waitingForWindowUpdates); + internal (int, long, State, long, bool) SchedulingState => (Stream.StreamId, _unconsumedBytes, _observationState, _streamWindow, _waitingForWindowUpdates); // Added bytes to the queue. // Returns a bool that represents whether we should schedule this producer to write // the enqueued bytes - private bool Enqueue(long bytes) + private void EnqueueDataWrite(long bytes) { lock (_dataWriterLock) { var wasEmpty = _unconsumedBytes == 0; _unconsumedBytes += bytes; - return wasEmpty && _unconsumedBytes > 0 && _observationState == State.None; } } // Determines if we should schedule this producer to observe // any state changes made. - private bool EnqueueForObservation(State state) + private void EnqueueStateUpdate(State state) { lock (_dataWriterLock) { var wasEnqueuedForObservation = _observationState != State.None; _observationState |= state; - return (_unconsumedBytes == 0 || _waitingForWindowUpdates) && !wasEnqueuedForObservation; } } // Removes consumed bytes from the queue. // Returns a bool that represents whether we should schedule this producer to write // the remaining bytes. - internal (bool, bool) Dequeue(long bytes, State state) + internal (bool hasMoreData, bool reschedule) ObserveDataAndState(long bytes, State state) { lock (_dataWriterLock) { @@ -129,13 +126,13 @@ private bool EnqueueForObservation(State state) } // Consumes bytes from the stream's window and returns the remaining bytes and actual bytes consumed - internal (long, long) ConsumeWindow(long bytes) + internal (long actual, long remaining) ConsumeStreamWindow(long bytes) { lock (_dataWriterLock) { - var actual = Math.Min(bytes, _window); - var remaining = _window -= actual; - return (actual, remaining); + var actual = Math.Min(bytes, _streamWindow); + _streamWindow -= actual; + return (actual, _streamWindow); } } @@ -146,9 +143,9 @@ private bool UpdateWindow(long bytes) { lock (_dataWriterLock) { - var wasDepleted = _window <= 0; - _window += bytes; - return wasDepleted && _window > 0 && _unconsumedBytes > 0; + var wasDepleted = _streamWindow <= 0; + _streamWindow += bytes; + return wasDepleted && _streamWindow > 0 && _unconsumedBytes > 0; } } @@ -165,7 +162,7 @@ public void StreamReset(uint initialWindowSize) _pipe.Reset(); _pipeWriter.Reset(); - _window = initialWindowSize; + _streamWindow = initialWindowSize; _unconsumedBytes = 0; _observationState = State.None; _waitingForWindowUpdates = false; @@ -190,15 +187,12 @@ public void Complete() if (!_streamCompleted) { - var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Completed); + EnqueueStateUpdate(State.Completed); // Make sure the writing side is completed. _pipeWriter.Complete(); - if (enqueue) - { - Schedule(); - } + Schedule(); } else { @@ -249,27 +243,19 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) if (_startedWritingDataFrames) { - var enqueue = Enqueue(_pipeWriter.UnflushedBytes); - // If there's already been response data written to the stream, just wait for that. Any header // should be in front of the data frames in the connection pipe. Trailers could change things. var task = _flusher.FlushAsync(this, cancellationToken); - if (enqueue) - { - Schedule(); - } + Schedule(); return task; } else { - var enqueue = EnqueueForObservation(State.FlushHeaders); + EnqueueStateUpdate(State.FlushHeaders); - if (enqueue) - { - Schedule(); - } + Schedule(); return default; } @@ -365,14 +351,11 @@ public Task WriteDataAsync(ReadOnlySpan data, CancellationToken cancellati _pipeWriter.Write(data); - var enqueue = Enqueue(data.Length); + EnqueueDataWrite(data.Length); var task = _flusher.FlushAsync(this, cancellationToken).GetAsTask(); - if (enqueue) - { - Schedule(); - } + Schedule(); return task; } @@ -391,14 +374,11 @@ public ValueTask WriteStreamSuffixAsync() _suffixSent = true; // Try to enqueue any unflushed bytes - var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Completed); + EnqueueStateUpdate(State.Completed); _pipeWriter.Complete(); - if (enqueue) - { - Schedule(); - } + Schedule(); return ValueTask.FromResult(default); } @@ -429,6 +409,8 @@ public void Advance(int bytes) _startedWritingDataFrames = true; _pipeWriter.Advance(bytes); + + EnqueueDataWrite(bytes); } } @@ -497,13 +479,10 @@ public ValueTask WriteDataToPipeAsync(ReadOnlySpan data, Canc _pipeWriter.Write(data); - var enqueue = Enqueue(data.Length); + EnqueueDataWrite(data.Length); var task = _flusher.FlushAsync(this, cancellationToken); - if (enqueue) - { - Schedule(); - } + Schedule(); return task; } @@ -540,15 +519,12 @@ public void Stop() _streamCompleted = true; - var enqueue = Enqueue(_pipeWriter.UnflushedBytes) || EnqueueForObservation(State.Canceled); + EnqueueStateUpdate(State.Canceled); _pipeReader.CancelPendingRead(); - if (enqueue) - { - // We need to make sure the cancellation is observed by the code - Schedule(); - } + // We need to make sure the cancellation is observed by the code + Schedule(); } } @@ -636,7 +612,7 @@ public bool TryUpdateStreamWindow(int bytes) { lock (_dataWriterLock) { - var maxUpdate = Http2PeerSettings.MaxWindowSize - _window; + var maxUpdate = Http2PeerSettings.MaxWindowSize - _streamWindow; if (bytes > maxUpdate) { diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs index d1683adb54dc..0256e7830a23 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/PipeWriterHelpers/ConcurrentPipeWriter.cs @@ -57,10 +57,6 @@ public ConcurrentPipeWriter(PipeWriter innerPipeWriter, MemoryPool pool, o _sync = sync; } - public override bool CanGetUnflushedBytes => true; - - public override long UnflushedBytes => _currentFlushTcs is null && _head is null ? _innerPipeWriter.UnflushedBytes : _bytesBuffered; - public void Reset() { Debug.Assert(_currentFlushTcs == null, "There should not be a pending flush."); From 610ec1dac726d810c1dc54c5eb1f21447a582162 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 19:18:30 -0700 Subject: [PATCH 53/77] Removed unneeded state --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 9 --------- .../Core/src/Internal/Http2/Http2OutputProducer.cs | 13 +------------ 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index ca14c938e28e..cac42ad20e65 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -228,9 +228,6 @@ private async Task WriteToOutputPipe() // a window update to resume the connection. if (remainingConnection == 0) { - // Mark the output as waiting for a window upate to resume writing (there's still data) - producer.MarkWaitingForWindowUpdates(true); - lock (_windowUpdateLock) { _waitingForMoreWindow.Enqueue(producer); @@ -250,9 +247,6 @@ private async Task WriteToOutputPipe() } else { - // Mark the output as waiting for a window upate to resume writing (there's still data) - producer.MarkWaitingForWindowUpdates(true); - // Include waiting for window updates in timing writes if (_minResponseDataRate != null) { @@ -887,9 +881,6 @@ public bool TryUpdateConnectionWindow(int bytes) { if (!producer.StreamCompleted) { - // We're no longer waiting for the update - producer.MarkWaitingForWindowUpdates(false); - producer.Schedule(); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index a8a3626a9f93..8caa316fff28 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -42,7 +42,6 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state private State _observationState; - private bool _waitingForWindowUpdates; private bool _completedResponse; private bool _requestProcessingComplete; @@ -86,7 +85,7 @@ public bool StreamCompleted } // Useful for debugging the scheduling state in the debugger - internal (int, long, State, long, bool) SchedulingState => (Stream.StreamId, _unconsumedBytes, _observationState, _streamWindow, _waitingForWindowUpdates); + internal (int, long, State, long) SchedulingState => (Stream.StreamId, _unconsumedBytes, _observationState, _streamWindow); // Added bytes to the queue. // Returns a bool that represents whether we should schedule this producer to write @@ -165,7 +164,6 @@ public void StreamReset(uint initialWindowSize) _streamWindow = initialWindowSize; _unconsumedBytes = 0; _observationState = State.None; - _waitingForWindowUpdates = false; _completedResponse = false; _requestProcessingComplete = false; WriteHeaders = false; @@ -262,14 +260,6 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) } } - public void MarkWaitingForWindowUpdates(bool waitingForUpdates) - { - lock (_dataWriterLock) - { - _waitingForWindowUpdates = waitingForUpdates; - } - } - public void Schedule() { lock (_dataWriterLock) @@ -280,7 +270,6 @@ public void Schedule() return; } - _waitingForWindowUpdates = false; _isScheduled = true; } From 7a92bee51877d246f96c628d0247f0bb84539fff Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 19:26:42 -0700 Subject: [PATCH 54/77] More PR suggestions --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 10 ++-------- .../src/Internal/Http2/Http2OutputProducer.cs | 17 ++++++++--------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index cac42ad20e65..2dc42ed0fa11 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -538,10 +538,7 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen WriteHeaderUnsynchronized(); - foreach (var buffer in currentData) - { - _outputWriter.Write(buffer.Span); - } + currentData.CopyTo(_outputWriter); // Plus padding dataLength -= dataPayloadLength; @@ -558,10 +555,7 @@ void TrimAndWriteDataUnsynchronized(in ReadOnlySequence data, long dataLen WriteHeaderUnsynchronized(); - foreach (var buffer in remainingData) - { - _outputWriter.Write(buffer.Span); - } + remainingData.CopyTo(_outputWriter); // Plus padding } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 8caa316fff28..7e01f152829d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -41,7 +41,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private long _streamWindow; // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state - private State _observationState; + private State _unobservedState; private bool _completedResponse; private bool _requestProcessingComplete; @@ -85,7 +85,7 @@ public bool StreamCompleted } // Useful for debugging the scheduling state in the debugger - internal (int, long, State, long) SchedulingState => (Stream.StreamId, _unconsumedBytes, _observationState, _streamWindow); + internal (int, long, State, long) SchedulingState => (Stream.StreamId, _unconsumedBytes, _unobservedState, _streamWindow); // Added bytes to the queue. // Returns a bool that represents whether we should schedule this producer to write @@ -105,8 +105,7 @@ private void EnqueueStateUpdate(State state) { lock (_dataWriterLock) { - var wasEnqueuedForObservation = _observationState != State.None; - _observationState |= state; + _unobservedState |= state; } } @@ -118,9 +117,9 @@ private void EnqueueStateUpdate(State state) lock (_dataWriterLock) { _isScheduled = false; - _observationState &= ~state; + _unobservedState &= ~state; _unconsumedBytes -= bytes; - return (_unconsumedBytes > 0, _observationState != State.None); + return (_unconsumedBytes > 0, _unobservedState != State.None); } } @@ -138,7 +137,7 @@ private void EnqueueStateUpdate(State state) // Adds more bytes to the stream's window // Returns a bool that represents whether we should schedule this producer to write // the remaining bytes. - private bool UpdateWindow(long bytes) + private bool UpdateStreamWindow(long bytes) { lock (_dataWriterLock) { @@ -163,7 +162,7 @@ public void StreamReset(uint initialWindowSize) _streamWindow = initialWindowSize; _unconsumedBytes = 0; - _observationState = State.None; + _unobservedState = State.None; _completedResponse = false; _requestProcessingComplete = false; WriteHeaders = false; @@ -609,7 +608,7 @@ public bool TryUpdateStreamWindow(int bytes) } } - if (UpdateWindow(bytes)) + if (UpdateStreamWindow(bytes)) { Schedule(); } From de855de0a94cc009720ad08d7cf1d5083d6a8816 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 19:36:44 -0700 Subject: [PATCH 55/77] Remove the FlushHeaders bool and use the unobserved state --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 14 +++++++------- .../src/Internal/Http2/Http2OutputProducer.cs | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 2dc42ed0fa11..31d56fa3e865 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -126,8 +126,10 @@ private async Task WriteToOutputPipe() reader.TryRead(out var readResult); var buffer = readResult.Buffer; + var unobservedState = producer.UnobservedState; + var flushHeaders = unobservedState.HasFlag(Http2OutputProducer.State.FlushHeaders); - if (producer.WriteHeaders) + if (flushHeaders) { observed |= Http2OutputProducer.State.FlushHeaders; } @@ -161,7 +163,7 @@ private async Task WriteToOutputPipe() if (readResult.IsCanceled) { // Response body is aborted, complete reader for this output producer. - if (producer.WriteHeaders) + if (flushHeaders) { // write headers WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); @@ -177,7 +179,7 @@ private async Task WriteToOutputPipe() stream.DecrementActiveClientStreamCount(); // It is faster to write data and trailers together. Locking once reduces lock contention. - flushResult = await WriteDataAndTrailersAsync(stream, buffer, producer.WriteHeaders, stream.ResponseTrailers); + flushResult = await WriteDataAndTrailersAsync(stream, buffer, flushHeaders, stream.ResponseTrailers); } else if (readResult.IsCompleted && producer.StreamEnded) { @@ -190,7 +192,7 @@ private async Task WriteToOutputPipe() stream.DecrementActiveClientStreamCount(); // Headers have already been written and there is no other content to write - flushResult = await FlushAsync(stream, producer.WriteHeaders, outputAborter: null, cancellationToken: default); + flushResult = await FlushAsync(stream, flushHeaders, outputAborter: null, cancellationToken: default); } } else @@ -202,7 +204,7 @@ private async Task WriteToOutputPipe() stream.DecrementActiveClientStreamCount(); } - flushResult = await WriteDataAsync(stream, buffer, buffer.Length, endStream, producer.WriteHeaders); + flushResult = await WriteDataAsync(stream, buffer, buffer.Length, endStream, flushHeaders); } if (producer.IsTimingWrite) @@ -210,8 +212,6 @@ private async Task WriteToOutputPipe() _timeoutControl.StopTimingWrite(); } - producer.WriteHeaders = false; - reader.AdvanceTo(buffer.End); if ((readResult.IsCompleted && !hasMoreData) || readResult.IsCanceled) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 7e01f152829d..64116ee85255 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -69,8 +69,6 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) public bool IsTimingWrite { get; set; } - public bool WriteHeaders { get; set; } - public bool StreamEnded => _streamEnded; public bool StreamCompleted @@ -87,6 +85,17 @@ public bool StreamCompleted // Useful for debugging the scheduling state in the debugger internal (int, long, State, long) SchedulingState => (Stream.StreamId, _unconsumedBytes, _unobservedState, _streamWindow); + public State UnobservedState + { + get + { + lock (_dataWriterLock) + { + return _unobservedState; + } + } + } + // Added bytes to the queue. // Returns a bool that represents whether we should schedule this producer to write // the enqueued bytes @@ -165,7 +174,6 @@ public void StreamReset(uint initialWindowSize) _unobservedState = State.None; _completedResponse = false; _requestProcessingComplete = false; - WriteHeaders = false; IsTimingWrite = false; } @@ -313,7 +321,7 @@ public void WriteResponseHeaders(int statusCode, string? reasonPhrase, HttpRespo _streamEnded = true; } - WriteHeaders = true; + EnqueueStateUpdate(State.FlushHeaders); } } From c3a1289b25fee656da04dccdd8d00c49d16b8bef Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 20:07:04 -0700 Subject: [PATCH 56/77] Small nits --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 31d56fa3e865..5f42a3324a1c 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -47,7 +47,7 @@ internal class Http2FrameWriter private readonly object _windowUpdateLock = new(); private long _connectionWindow; - private readonly Queue _waitingForMoreWindow = new(); + private readonly Queue _waitingForMoreConnectionWindow = new(); private readonly Task _writeQueueTask; public Http2FrameWriter( @@ -101,8 +101,8 @@ public void Schedule(Http2OutputProducer producer) if (!_channel.Writer.TryWrite(producer)) { // It should not be possible to exceed the bound of the channel. - var ex = new ConnectionAbortedException("Http2FrameWriter._channel exceeded bounds."); - _log.LogCritical(ex, "Http2FrameWriter._channel exceeded bounds.", _connectionId); + var ex = new ConnectionAbortedException("HTTP/2 connection exceeded the output operations maximum queue size."); + _log.LogCritical(ex, "HTTP/2 connection exceeded the output operations maximum queue size on connection {ConnectionId}.", _connectionId); _http2Connection.Abort(ex); } } @@ -230,7 +230,7 @@ private async Task WriteToOutputPipe() { lock (_windowUpdateLock) { - _waitingForMoreWindow.Enqueue(producer); + _waitingForMoreConnectionWindow.Enqueue(producer); } // Include waiting for window updates in timing writes @@ -847,7 +847,7 @@ private void AbortFlowControl() { lock (_windowUpdateLock) { - while (_waitingForMoreWindow.TryDequeue(out var producer)) + while (_waitingForMoreConnectionWindow.TryDequeue(out var producer)) { if (!producer.StreamCompleted) { @@ -871,7 +871,7 @@ public bool TryUpdateConnectionWindow(int bytes) _connectionWindow += bytes; - while (_waitingForMoreWindow.TryDequeue(out var producer)) + while (_waitingForMoreConnectionWindow.TryDequeue(out var producer)) { if (!producer.StreamCompleted) { From 97e728dd0d327eb92df0686710c29a5dccc833f1 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 20:14:24 -0700 Subject: [PATCH 57/77] Small tweaks --- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 64116ee85255..9cd13af19c67 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -369,7 +369,6 @@ public ValueTask WriteStreamSuffixAsync() _streamCompleted = true; _suffixSent = true; - // Try to enqueue any unflushed bytes EnqueueStateUpdate(State.Completed); _pipeWriter.Complete(); @@ -519,7 +518,6 @@ public void Stop() _pipeReader.CancelPendingRead(); - // We need to make sure the cancellation is observed by the code Schedule(); } } From 69f204d90e5d26567f2aea419e5b9df1401a1bcd Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 20:15:46 -0700 Subject: [PATCH 58/77] More naming changs --- .../Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 5f42a3324a1c..1784f08f7fca 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -300,7 +300,7 @@ public void Complete() } _completed = true; - AbortFlowControl(); + AbortConnectionFlowControl(); _outputWriter.Abort(); } } @@ -843,7 +843,7 @@ private ValueTask TimeFlushUnsynchronizedAsync() } } - private void AbortFlowControl() + private void AbortConnectionFlowControl() { lock (_windowUpdateLock) { From 66a8d4edace145336ed0f4718f8181275cac618c Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 21:15:43 -0700 Subject: [PATCH 59/77] More PR feedback - Added logging entries to source gen - Moved UpdateStreamWindow to local function --- .../src/Internal/Http2/Http2FrameWriter.cs | 8 ++--- .../src/Internal/Http2/Http2OutputProducer.cs | 29 +++++++++-------- .../Internal/Infrastructure/KestrelTrace.cs | 32 +++++++++++++++++++ 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 1784f08f7fca..d27fa4e17071 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -102,7 +102,7 @@ public void Schedule(Http2OutputProducer producer) { // It should not be possible to exceed the bound of the channel. var ex = new ConnectionAbortedException("HTTP/2 connection exceeded the output operations maximum queue size."); - _log.LogCritical(ex, "HTTP/2 connection exceeded the output operations maximum queue size on connection {ConnectionId}.", _connectionId); + _log.Http2QueueOperationsExceeded(_connectionId, ex); _http2Connection.Abort(ex); } } @@ -185,7 +185,7 @@ private async Task WriteToOutputPipe() { if (buffer.Length != 0) { - _log.LogCritical("{StreamId} observed an unexpected state where the streams output ended with data still remaining in the pipe.", stream.StreamId); + _log.Http2UnexpectedDataRemaining(stream.StreamId); } else { @@ -262,12 +262,12 @@ private async Task WriteToOutputPipe() } catch (Exception ex) { - _log.LogCritical(ex, "The event loop in connection {ConnectionId} failed unexpectedly", _connectionId); + _log.Http2UnexpectedConnectionQueueError(_connectionId, ex); } } } - _log.LogDebug("The connection processing loop for {ConnectionId} ended gracefully", _connectionId); + _log.Http2ConnectionQueueProcessingCompleted(_connectionId); } public void UpdateMaxHeaderTableSize(uint maxHeaderTableSize) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 9cd13af19c67..f454a250f1e2 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -143,19 +143,6 @@ private void EnqueueStateUpdate(State state) } } - // Adds more bytes to the stream's window - // Returns a bool that represents whether we should schedule this producer to write - // the remaining bytes. - private bool UpdateStreamWindow(long bytes) - { - lock (_dataWriterLock) - { - var wasDepleted = _streamWindow <= 0; - _streamWindow += bytes; - return wasDepleted && _streamWindow > 0 && _unconsumedBytes > 0; - } - } - public void StreamReset(uint initialWindowSize) { // Response should have been completed. @@ -604,6 +591,8 @@ internal Memory GetFakeMemory(int minSize) public bool TryUpdateStreamWindow(int bytes) { + var schedule = false; + lock (_dataWriterLock) { var maxUpdate = Http2PeerSettings.MaxWindowSize - _streamWindow; @@ -612,14 +601,26 @@ public bool TryUpdateStreamWindow(int bytes) { return false; } + + schedule = UpdateStreamWindow(bytes); } - if (UpdateStreamWindow(bytes)) + if (schedule) { Schedule(); } return true; + + // Adds more bytes to the stream's window + // Returns a bool that represents whether we should schedule this producer to write + // the remaining bytes. + bool UpdateStreamWindow(long bytes) + { + var wasDepleted = _streamWindow <= 0; + _streamWindow += bytes; + return wasDepleted && _streamWindow > 0 && _unconsumedBytes > 0; + } } [StackTraceHidden] diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs index c8df0403ac4e..62a0ebd37ba3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs @@ -298,6 +298,38 @@ public void Http2MaxConcurrentStreamsReached(string connectionId) Http2MaxConcurrentStreamsReached(_http2Logger, connectionId); } + [LoggerMessage(60, LogLevel.Critical, @"Connection id ""{ConnectionId}"" exceeded the output operations maximum queue size.", EventName = "Http2QueueOperationsExceeded")] + private static partial void Http2QueueOperationsExceeded(ILogger logger, string connectionId, ConnectionAbortedException ex); + + public void Http2QueueOperationsExceeded(string connectionId, ConnectionAbortedException ex) + { + Http2QueueOperationsExceeded(_http2Logger, connectionId, ex); + } + + [LoggerMessage(61, LogLevel.Critical, @"Stream {StreamId} observed an unexpected state where the streams output ended with data still remaining in the pipe.", EventName = "Http2UnexpectedDataRemaining")] + private static partial void Http2UnexpectedDataRemaining(ILogger logger, int streamId); + + public void Http2UnexpectedDataRemaining(int streamId) + { + Http2UnexpectedDataRemaining(_http2Logger, streamId); + } + + [LoggerMessage(62, LogLevel.Debug, @"The connection queue processing loop for {ConnectionId} completed.", EventName = "Http2ConnectionQueueProcessingCompleted")] + private static partial void Http2ConnectionQueueProcessingCompleted(ILogger logger, string connectionId); + + public void Http2ConnectionQueueProcessingCompleted(string connectionId) + { + Http2ConnectionQueueProcessingCompleted(_http2Logger, connectionId); + } + + [LoggerMessage(63, LogLevel.Critical, @"The event loop in connection {ConnectionId} failed unexpectedly.", EventName = "Http2UnexpectedConnectionQueueError")] + private static partial void Http2UnexpectedConnectionQueueError(ILogger logger, string connectionId, Exception ex); + + public void Http2UnexpectedConnectionQueueError(string connectionId, Exception ex) + { + Http2UnexpectedConnectionQueueError(_http2Logger, connectionId, ex); + } + [LoggerMessage(41, LogLevel.Warning, "One or more of the following response headers have been removed because they are invalid for HTTP/2 and HTTP/3 responses: 'Connection', 'Transfer-Encoding', 'Keep-Alive', 'Upgrade' and 'Proxy-Connection'.", EventName = "InvalidResponseHeaderRemoved")] private static partial void InvalidResponseHeaderRemoved(ILogger logger); From 41d8e2e4d2806c4bf0f404f91cb6dccff9ba74dd Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 21:26:32 -0700 Subject: [PATCH 60/77] Removed usings... --- src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index d27fa4e17071..eb6d3a9e0582 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -11,7 +11,6 @@ using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure.PipeWriterHelpers; -using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; From 7acfdd4becb29e9c20a94ed2b5355e7f320c8193 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 23:08:16 -0700 Subject: [PATCH 61/77] Aborts and writes are no longer synchronized - Writes have not been written to the connection on complete async so aborting them can cancel pending data can be cancelled even if called after complete. --- .../HttpClientHttp2InteropTests.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs index 84b7dfed76a8..19b158a28f63 100644 --- a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs +++ b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs @@ -637,6 +637,8 @@ public async Task ServerReset_AfterHeaders_ClientBodyThrows(string scheme) [MemberData(nameof(SupportedSchemes))] public async Task ServerReset_AfterEndStream_NoError(string scheme) { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hostBuilder = new HostBuilder() .ConfigureWebHost(webHostBuilder => { @@ -646,6 +648,7 @@ public async Task ServerReset_AfterEndStream_NoError(string scheme) { await context.Response.WriteAsync("Hello World"); await context.Response.CompleteAsync(); + await tcs.Task; context.Features.Get().Reset(8); // Cancel })); }); @@ -656,6 +659,7 @@ public async Task ServerReset_AfterEndStream_NoError(string scheme) var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead).DefaultTimeout(); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync().DefaultTimeout(); + tcs.TrySetResult(); Assert.Equal("Hello World", body); await host.StopAsync().DefaultTimeout(); } @@ -664,6 +668,8 @@ public async Task ServerReset_AfterEndStream_NoError(string scheme) [MemberData(nameof(SupportedSchemes))] public async Task ServerReset_AfterTrailers_NoError(string scheme) { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hostBuilder = new HostBuilder() .ConfigureWebHost(webHostBuilder => { @@ -675,6 +681,7 @@ public async Task ServerReset_AfterTrailers_NoError(string scheme) await context.Response.WriteAsync("Hello World"); context.Response.AppendTrailer("TestTrailer", "TestValue"); await context.Response.CompleteAsync(); + await tcs.Task; context.Features.Get().Reset(8); // Cancel })); }); @@ -686,6 +693,7 @@ public async Task ServerReset_AfterTrailers_NoError(string scheme) Assert.Equal(HttpVersion.Version20, response.Version); Assert.Equal("TestTrailer", response.Headers.Trailer.Single()); var responseBody = await response.Content.ReadAsStringAsync().DefaultTimeout(); + tcs.TrySetResult(); Assert.Equal("Hello World", responseBody); // The response is buffered, we must already have the trailers. Assert.Equal("TestValue", response.TrailingHeaders.GetValues("TestTrailer").Single()); From 600cffbdac92ced51607d0254372adb3adb4e8a0 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Mon, 11 Apr 2022 23:59:09 -0700 Subject: [PATCH 62/77] Preserve existing behavior - If we complete the stream before calling WriteRst then wait until the response completes before writing the RST --- .../src/Internal/Http2/Http2OutputProducer.cs | 24 ++++++++++++++++++- .../HttpClientHttp2InteropTests.cs | 8 ------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index f454a250f1e2..c478af7bc665 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -44,6 +44,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private State _unobservedState; private bool _completedResponse; private bool _requestProcessingComplete; + private Http2ErrorCode? _resetErrorCode; public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) { @@ -161,6 +162,7 @@ public void StreamReset(uint initialWindowSize) _unobservedState = State.None; _completedResponse = false; _requestProcessingComplete = false; + _resetErrorCode = null; IsTimingWrite = false; } @@ -370,7 +372,14 @@ public ValueTask WriteRstStreamAsync(Http2ErrorCode error) { lock (_dataWriterLock) { - // Always send the reset even if the response body is _completed. The request body may not have completed yet. + // We queued the stream to complete but didn't complete the response yet + if (_streamCompleted && !_completedResponse) + { + // Set the error so that we can write the RST when the response completes. + _resetErrorCode = error; + return default; + } + Stop(); return _frameWriter.WriteRstStreamAsync(StreamId, error); @@ -536,8 +545,21 @@ internal void CompleteResponse() { lock (_dataWriterLock) { + if (_completedResponse) + { + // This should never be called twice + return; + } + _completedResponse = true; + if (_resetErrorCode is { } error) + { + // If we have an error code to write, write it now that we're done with the response. + // Always send the reset even if the response body is completed. The request body may not have completed yet. + _ = _frameWriter.WriteRstStreamAsync(StreamId, error).Preserve(); + } + if (_requestProcessingComplete) { Stream.CompleteStream(errored: false); diff --git a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs index 19b158a28f63..84b7dfed76a8 100644 --- a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs +++ b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs @@ -637,8 +637,6 @@ public async Task ServerReset_AfterHeaders_ClientBodyThrows(string scheme) [MemberData(nameof(SupportedSchemes))] public async Task ServerReset_AfterEndStream_NoError(string scheme) { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var hostBuilder = new HostBuilder() .ConfigureWebHost(webHostBuilder => { @@ -648,7 +646,6 @@ public async Task ServerReset_AfterEndStream_NoError(string scheme) { await context.Response.WriteAsync("Hello World"); await context.Response.CompleteAsync(); - await tcs.Task; context.Features.Get().Reset(8); // Cancel })); }); @@ -659,7 +656,6 @@ public async Task ServerReset_AfterEndStream_NoError(string scheme) var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead).DefaultTimeout(); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync().DefaultTimeout(); - tcs.TrySetResult(); Assert.Equal("Hello World", body); await host.StopAsync().DefaultTimeout(); } @@ -668,8 +664,6 @@ public async Task ServerReset_AfterEndStream_NoError(string scheme) [MemberData(nameof(SupportedSchemes))] public async Task ServerReset_AfterTrailers_NoError(string scheme) { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var hostBuilder = new HostBuilder() .ConfigureWebHost(webHostBuilder => { @@ -681,7 +675,6 @@ public async Task ServerReset_AfterTrailers_NoError(string scheme) await context.Response.WriteAsync("Hello World"); context.Response.AppendTrailer("TestTrailer", "TestValue"); await context.Response.CompleteAsync(); - await tcs.Task; context.Features.Get().Reset(8); // Cancel })); }); @@ -693,7 +686,6 @@ public async Task ServerReset_AfterTrailers_NoError(string scheme) Assert.Equal(HttpVersion.Version20, response.Version); Assert.Equal("TestTrailer", response.Headers.Trailer.Single()); var responseBody = await response.Content.ReadAsStringAsync().DefaultTimeout(); - tcs.TrySetResult(); Assert.Equal("Hello World", responseBody); // The response is buffered, we must already have the trailers. Assert.Equal("TestValue", response.TrailingHeaders.GetValues("TestTrailer").Single()); From 243643d02f56e99dd752c104a53b839dab0727b4 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Tue, 12 Apr 2022 00:02:47 -0700 Subject: [PATCH 63/77] Removed unused state --- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index c478af7bc665..1034d4a64d80 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -104,7 +104,6 @@ private void EnqueueDataWrite(long bytes) { lock (_dataWriterLock) { - var wasEmpty = _unconsumedBytes == 0; _unconsumedBytes += bytes; } } From 6af2c0dd800de28970b8aee18327c440a8af46b5 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Tue, 12 Apr 2022 08:12:25 -0700 Subject: [PATCH 64/77] Make the test await the process request loop --- .../InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index 3a9ac6fc49f2..f2354d1fe847 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -4796,14 +4796,14 @@ public async Task IgnoreNewStreamsDuringClosedConnection() } [Fact] - public void IOExceptionDuringFrameProcessingIsNotLoggedHigherThanDebug() + public async Task IOExceptionDuringFrameProcessingIsNotLoggedHigherThanDebug() { CreateConnection(); var ioException = new IOException(); _pair.Application.Output.Complete(ioException); - Assert.Equal(TaskStatus.RanToCompletion, _connection.ProcessRequestsAsync(new DummyApplication(_noopApplication)).Status); + await _connection.ProcessRequestsAsync(new DummyApplication(_noopApplication)).DefaultTimeout(); Assert.All(LogMessages, w => Assert.InRange(w.LogLevel, LogLevel.Trace, LogLevel.Debug)); From 98963b7092dc6d69c259e9522c158ae4b3a6a008 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Tue, 12 Apr 2022 09:36:29 -0700 Subject: [PATCH 65/77] Removed extra flush headers state. --- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 1034d4a64d80..bfcfcd408214 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -246,8 +246,6 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) } else { - EnqueueStateUpdate(State.FlushHeaders); - Schedule(); return default; From 812964f8f3a0a81e3d9787631185a46d642444d4 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Tue, 12 Apr 2022 13:39:20 -0700 Subject: [PATCH 66/77] PR feedback --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 4 ++-- .../Core/src/Internal/Http2/Http2OutputProducer.cs | 10 +++++++--- .../Core/src/Internal/Infrastructure/KestrelTrace.cs | 8 ++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index eb6d3a9e0582..764a8990a2c2 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -184,7 +184,7 @@ private async Task WriteToOutputPipe() { if (buffer.Length != 0) { - _log.Http2UnexpectedDataRemaining(stream.StreamId); + _log.Http2UnexpectedDataRemaining(stream.StreamId, _connectionId); } else { @@ -217,7 +217,7 @@ private async Task WriteToOutputPipe() { await reader.CompleteAsync(); - producer.CompleteResponse(); + await producer.CompleteResponseAsync(); } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index bfcfcd408214..f523ece53814 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -538,29 +538,33 @@ internal void OnRequestProcessingEnded() } } - internal void CompleteResponse() + internal ValueTask CompleteResponseAsync() { lock (_dataWriterLock) { if (_completedResponse) { // This should never be called twice - return; + return default; } _completedResponse = true; + ValueTask task = default; + if (_resetErrorCode is { } error) { // If we have an error code to write, write it now that we're done with the response. // Always send the reset even if the response body is completed. The request body may not have completed yet. - _ = _frameWriter.WriteRstStreamAsync(StreamId, error).Preserve(); + task = _frameWriter.WriteRstStreamAsync(StreamId, error); } if (_requestProcessingComplete) { Stream.CompleteStream(errored: false); } + + return task; } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs index 62a0ebd37ba3..f0335df43005 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/KestrelTrace.cs @@ -306,12 +306,12 @@ public void Http2QueueOperationsExceeded(string connectionId, ConnectionAbortedE Http2QueueOperationsExceeded(_http2Logger, connectionId, ex); } - [LoggerMessage(61, LogLevel.Critical, @"Stream {StreamId} observed an unexpected state where the streams output ended with data still remaining in the pipe.", EventName = "Http2UnexpectedDataRemaining")] - private static partial void Http2UnexpectedDataRemaining(ILogger logger, int streamId); + [LoggerMessage(61, LogLevel.Critical, @"Stream {StreamId} on connection id ""{ConnectionId}"" observed an unexpected state where the streams output ended with data still remaining in the pipe.", EventName = "Http2UnexpectedDataRemaining")] + private static partial void Http2UnexpectedDataRemaining(ILogger logger, int streamId, string connectionId); - public void Http2UnexpectedDataRemaining(int streamId) + public void Http2UnexpectedDataRemaining(int streamId, string connectionId) { - Http2UnexpectedDataRemaining(_http2Logger, streamId); + Http2UnexpectedDataRemaining(_http2Logger, streamId, connectionId); } [LoggerMessage(62, LogLevel.Debug, @"The connection queue processing loop for {ConnectionId} completed.", EventName = "Http2ConnectionQueueProcessingCompleted")] From 7cc588d02065cd5b39f0e52e0c39238380ccd8c2 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Tue, 12 Apr 2022 23:42:55 -0700 Subject: [PATCH 67/77] Improve the fairness of connection window updates - Today its possible for a single stream to keep getting rescheduled and draining the connection window. This change makes it so that streams that have data to be written and haven't written anything are preferred over ones that have written something. It does this by changing the head of the queue before dequeuing everything to start at the first index of the stream that hasn't written anything. --- .../src/Internal/Http2/Http2FrameWriter.cs | 30 +++- .../Internal/Http2/QueueWithMovableHead.cs | 137 ++++++++++++++++++ .../test/Http2/QueueWithMovableHeadTests.cs | 93 ++++++++++++ 3 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs create mode 100644 src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 764a8990a2c2..1c0d1276ac69 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -46,7 +46,8 @@ internal class Http2FrameWriter private readonly object _windowUpdateLock = new(); private long _connectionWindow; - private readonly Queue _waitingForMoreConnectionWindow = new(); + private readonly QueueWithMovableHead _waitingForMoreConnectionWindow = new(); + private int? _waitingForMoreConnectionWindowStart; private readonly Task _writeQueueTask; public Http2FrameWriter( @@ -229,7 +230,19 @@ private async Task WriteToOutputPipe() { lock (_windowUpdateLock) { - _waitingForMoreConnectionWindow.Enqueue(producer); + // In order to make scheduling more fair we want to make sure that streams that have data get a chance to run in a round robin manner. + // To do this we will store everything in the queue waiting for the connection update + // but will store the index of the first element that had something to write but didn't write anything. This is the index that we will start dequeuing + // items from instead of the start of the queue. + if (actual == 0 && _waitingForMoreConnectionWindowStart is null) + { + // This index will be stable until we start dequeueing everything + _waitingForMoreConnectionWindowStart = _waitingForMoreConnectionWindow.Enqueue(producer); + } + else + { + _waitingForMoreConnectionWindow.Enqueue(producer); + } } // Include waiting for window updates in timing writes @@ -870,6 +883,19 @@ public bool TryUpdateConnectionWindow(int bytes) _connectionWindow += bytes; + if (_waitingForMoreConnectionWindowStart is { } start) + { + // Reset the index + _waitingForMoreConnectionWindowStart = null; + + // Set the head of the queue to the start so that we can start dequeueing from that position. + + // WARNING: Setting the head must be called once and then the queue must be drained. Interleaving + // calls to Enqueue/Dequeue will mutate the underlying data structure and will invalidate indexes + // returned from Enqueue. + _waitingForMoreConnectionWindow.SetHead(start); + } + while (_waitingForMoreConnectionWindow.TryDequeue(out var producer)) { if (!producer.StreamCompleted) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs new file mode 100644 index 000000000000..a8f591e8f80f --- /dev/null +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs @@ -0,0 +1,137 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; + +// This is a normal queue with a twist, we have the ability to set the head to a different start index +internal class QueueWithMovableHead +{ + private T[] _array; + private int _head; // The index from which to dequeue if the queue isn't empty. + private int _tail; // The index at which to enqueue if the queue isn't full. + private int _size; // Number of elements. + + public int Count => _size; + + // Creates a queue with room for capacity objects. The default initial + // capacity and grow factor are used. + public QueueWithMovableHead() + { + _array = Array.Empty(); + } + + public int Enqueue(T item) + { + if (_size == _array.Length) + { + Grow(_size + 1); + } + + var pos = _tail; + _array[_tail] = item; + MoveNext(ref _tail, _array.Length); + _size++; + return pos; + } + + // Sets the index of the head of the queue + public void SetHead(int head) + { + _head = head; + } + + public bool TryDequeue([MaybeNullWhen(false)] out T result) + { + var head = _head; + var array = _array; + + if (_size == 0) + { + result = default!; + return false; + } + + result = array[head]; + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + array[head] = default!; + } + MoveNext(ref _head, _head < _tail ? _tail : _array.Length); + _size--; + + if (_size == 0) + { + // Restore the invarint that head = tail on empty queue + _head = _tail; + } + + return true; + } + + private static void MoveNext(ref int index, int end) + { + // It is tempting to use the remainder operator here but it is actually much slower + // than a simple comparison and a rarely taken branch. + // JIT produces better code than with ternary operator ?: + var tmp = index + 1; + if (tmp == end) + { + tmp = 0; + } + index = tmp; + } + + private void Grow(int capacity) + { + Debug.Assert(_array.Length < capacity); + + const int GrowFactor = 2; + const int MinimumGrow = 4; + + var newcapacity = GrowFactor * _array.Length; + + // Allow the list to grow to maximum possible capacity (~2G elements) before encountering overflow. + // Note that this check works even when _items.Length overflowed thanks to the (uint) cast + if ((uint)newcapacity > Array.MaxLength) + { + newcapacity = Array.MaxLength; + } + + // Ensure minimum growth is respected. + newcapacity = Math.Max(newcapacity, _array.Length + MinimumGrow); + + // If the computed capacity is still less than specified, set to the original argument. + // Capacities exceeding Array.MaxLength will be surfaced as OutOfMemoryException by Array.Resize. + if (newcapacity < capacity) + { + newcapacity = capacity; + } + + SetCapacity(newcapacity); + } + + private void SetCapacity(int capacity) + { + var newarray = new T[capacity]; + if (_size > 0) + { + if (_head < _tail) + { + Array.Copy(_array, _head, newarray, 0, _size); + } + else + { + Array.Copy(_array, _head, newarray, 0, _array.Length - _head); + Array.Copy(_array, 0, newarray, _array.Length - _head, _tail); + } + } + + _array = newarray; + _head = 0; + _tail = (_size == capacity) ? 0 : _size; + } +} diff --git a/src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs b/src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs new file mode 100644 index 000000000000..bdb521db029c --- /dev/null +++ b/src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests.Http2; + +public class QueueWithMovableHeadTests +{ + [Fact] + public void FIFOWorks() + { + var queue = new QueueWithMovableHead(); + queue.Enqueue(1); + queue.Enqueue(2); + + Assert.True(queue.TryDequeue(out var val1)); + Assert.True(queue.TryDequeue(out var val2)); + Assert.Equal(1, val1); + Assert.Equal(2, val2); + } + + [Fact] + public void SettingTheHeadOfTheQueueWorks() + { + var queue = new QueueWithMovableHead(); + queue.Enqueue(1); + queue.Enqueue(2); + queue.Enqueue(3); + queue.Enqueue(4); + + queue.SetHead(2); + + Assert.True(queue.TryDequeue(out var val1)); + Assert.True(queue.TryDequeue(out var val2)); + Assert.Equal(3, val1); + Assert.Equal(4, val2); + } + + [Fact] + public void SettingTheHeadOfTheQueueAndDrainingItWorks() + { + var queue = new QueueWithMovableHead(); + queue.Enqueue(1); + queue.Enqueue(2); + queue.Enqueue(3); + queue.Enqueue(4); + + queue.SetHead(2); + + Assert.True(queue.TryDequeue(out var val1)); + Assert.True(queue.TryDequeue(out var val2)); + Assert.True(queue.TryDequeue(out var val3)); + Assert.True(queue.TryDequeue(out var val4)); + Assert.False(queue.TryDequeue(out var val5)); + + Assert.Equal(3, val1); + Assert.Equal(4, val2); + Assert.Equal(1, val3); + Assert.Equal(2, val4); + } + + [Fact] + public void GrowingTheCapacitySettingTheHeadOfTheQueueAndDrainingItWorks() + { + var queue = new QueueWithMovableHead(); + + for (int i = 0; i < 2; i++) + { + for (var j = 0; j < 10; j++) + { + queue.Enqueue(j); + } + + queue.SetHead(5); + + var expected = 5; + for (var j = 0; j < 10; j++) + { + Assert.True(queue.TryDequeue(out var val)); + Assert.Equal(expected, val); + expected = (expected + 1) % 10; + } + + Assert.Equal(5, expected); + } + } +} From c243966a96235fac8de993a4d9f65398ebd1a7fd Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 13 Apr 2022 01:51:07 -0700 Subject: [PATCH 68/77] Fixed test --- .../InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs index f2354d1fe847..cf1b6fdc7f40 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2ConnectionTests.cs @@ -4814,14 +4814,14 @@ public async Task IOExceptionDuringFrameProcessingIsNotLoggedHigherThanDebug() } [Fact] - public void UnexpectedExceptionDuringFrameProcessingLoggedAWarning() + public async Task UnexpectedExceptionDuringFrameProcessingLoggedAWarning() { CreateConnection(); var exception = new Exception(); _pair.Application.Output.Complete(exception); - Assert.Equal(TaskStatus.RanToCompletion, _connection.ProcessRequestsAsync(new DummyApplication(_noopApplication)).Status); + await _connection.ProcessRequestsAsync(new DummyApplication(_noopApplication)).DefaultTimeout(); var logMessage = LogMessages.Single(m => m.LogLevel >= LogLevel.Information); From 6650be8fae3196ff58004ed14aac0f56e5bc594c Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 13 Apr 2022 23:00:17 -0700 Subject: [PATCH 69/77] Increased pause threshold to 4K - Reworked logic in frameworker to use the state enum as well as the read result. - Fixed races around how work can be scheduled now that we're not running inline with a small pause threshold. --- .../src/Internal/Http2/Http2FrameWriter.cs | 43 ++++++++----------- .../src/Internal/Http2/Http2OutputProducer.cs | 20 ++++++--- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 1c0d1276ac69..4ef821efa536 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -118,31 +118,19 @@ private async Task WriteToOutputPipe() var reader = producer.PipeReader; var stream = producer.Stream; - var observed = Http2OutputProducer.State.None; - // We don't need to check the result because it's either // - true because we have a result // - false because we're flushing headers reader.TryRead(out var readResult); - var buffer = readResult.Buffer; - var unobservedState = producer.UnobservedState; - var flushHeaders = unobservedState.HasFlag(Http2OutputProducer.State.FlushHeaders); - if (flushHeaders) - { - observed |= Http2OutputProducer.State.FlushHeaders; - } + // Stash the unobserved state, we're going to mark this snapshot as observed + var observed = producer.UnobservedState; + var flushHeaders = observed.HasFlag(Http2OutputProducer.State.FlushHeaders); + var aborted = observed.HasFlag(Http2OutputProducer.State.Canceled); - if (readResult.IsCanceled) - { - observed |= Http2OutputProducer.State.Canceled; - } - - if (readResult.IsCompleted) - { - observed |= Http2OutputProducer.State.Completed; - } + // Completed is special because it's a terminal state that should ideally never be removed. + var completed = observed.HasFlag(Http2OutputProducer.State.Completed) || readResult.IsCompleted; // Check the stream window var (actual, remainingStream) = producer.ConsumeStreamWindow(buffer.Length); @@ -156,11 +144,14 @@ private async Task WriteToOutputPipe() buffer = buffer.Slice(0, actual); } - var (hasMoreData, reschedule) = producer.ObserveDataAndState(buffer.Length, observed); + var (hasMoreData, nextState) = producer.ObserveDataAndState(buffer.Length, observed); FlushResult flushResult = default; - if (readResult.IsCanceled) + // There are 2 cases where we abort if: + // 1. We're done writing data and there's no more data to be written + // 2. We're not done writing data but we got the abort message + if ((aborted && completed && actual == 0) || (aborted && !completed)) { // Response body is aborted, complete reader for this output producer. if (flushHeaders) @@ -169,7 +160,7 @@ private async Task WriteToOutputPipe() WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); } } - else if (readResult.IsCompleted && stream.ResponseTrailers is { Count: > 0 } && !hasMoreData) + else if (completed && stream.ResponseTrailers is { Count: > 0 } && !hasMoreData) { // Output is ending and there are trailers to write // Write any remaining content then write trailers and there's no @@ -181,7 +172,7 @@ private async Task WriteToOutputPipe() // It is faster to write data and trailers together. Locking once reduces lock contention. flushResult = await WriteDataAndTrailersAsync(stream, buffer, flushHeaders, stream.ResponseTrailers); } - else if (readResult.IsCompleted && producer.StreamEnded) + else if (completed && producer.StreamEnded) { if (buffer.Length != 0) { @@ -197,7 +188,7 @@ private async Task WriteToOutputPipe() } else { - var endStream = readResult.IsCompleted && !hasMoreData; + var endStream = completed && !hasMoreData; if (endStream) { @@ -214,7 +205,7 @@ private async Task WriteToOutputPipe() reader.AdvanceTo(buffer.End); - if ((readResult.IsCompleted && !hasMoreData) || readResult.IsCanceled) + if ((completed && !hasMoreData) || aborted) { await reader.CompleteAsync(); @@ -222,7 +213,7 @@ private async Task WriteToOutputPipe() } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. - else if (hasMoreData) + else if (hasMoreData && !nextState.HasFlag(Http2OutputProducer.State.Canceled)) { // We have no more connection window, put this producer in a queue waiting for it to // a window update to resume the connection. @@ -267,7 +258,7 @@ private async Task WriteToOutputPipe() } } } - else if (reschedule) + else if (nextState != Http2OutputProducer.State.None) { producer.Schedule(); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index f523ece53814..41e4a186397e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -52,8 +52,9 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) _frameWriter = context.FrameWriter; _memoryPool = context.MemoryPool; _log = context.ServiceContext.Log; + var scheduleInline = context.ServiceContext.Scheduler == PipeScheduler.Inline; - _pipe = CreateDataPipe(_memoryPool); + _pipe = CreateDataPipe(_memoryPool, scheduleInline); _pipeWriter = new ConcurrentPipeWriter(_pipe.Writer, _memoryPool, _dataWriterLock); _pipeReader = _pipe.Reader; @@ -121,14 +122,14 @@ private void EnqueueStateUpdate(State state) // Removes consumed bytes from the queue. // Returns a bool that represents whether we should schedule this producer to write // the remaining bytes. - internal (bool hasMoreData, bool reschedule) ObserveDataAndState(long bytes, State state) + internal (bool hasMoreData, State unobservedState) ObserveDataAndState(long bytes, State state) { lock (_dataWriterLock) { _isScheduled = false; _unobservedState &= ~state; _unconsumedBytes -= bytes; - return (_unconsumedBytes > 0, _unobservedState != State.None); + return (_unconsumedBytes > 0, _unobservedState); } } @@ -500,8 +501,10 @@ public void Stop() { lock (_dataWriterLock) { - if (_streamCompleted) + if (_streamCompleted && _completedResponse) { + // We can overschedule as long as we haven't yet completed the response. This is important because + // we may need to abort the stream if it's waiting for a window update. return; } @@ -672,14 +675,17 @@ private static void ThrowWriterComplete() throw new InvalidOperationException("Cannot write to response after the request has completed."); } - private static Pipe CreateDataPipe(MemoryPool pool) + private static Pipe CreateDataPipe(MemoryPool pool, bool scheduleInline) => new Pipe(new PipeOptions ( pool: pool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.ThreadPool, - pauseWriterThreshold: 1, - resumeWriterThreshold: 1, + // The unit tests rely on inline scheduling and the ability to control individual writes + // and assert individual frames. Setting the thresholds to 1 avoids frames being coaleased together + // and allows the test to assert them individually. + pauseWriterThreshold: scheduleInline ? 1 : 4096, + resumeWriterThreshold: scheduleInline ? 1 : 2048, useSynchronizationContext: false, minimumSegmentSize: pool.GetMinimumSegmentSize() )); From 893ebd761fc8c69fa9d040694305d911107ecdce Mon Sep 17 00:00:00 2001 From: David Fowler Date: Wed, 13 Apr 2022 23:25:56 -0700 Subject: [PATCH 70/77] Simplify the queue fairness - Store the last window consumer in as a field and make sure it gets queued last on the next window update. --- .../src/Internal/Http2/Http2FrameWriter.cs | 29 ++-- .../Internal/Http2/QueueWithMovableHead.cs | 137 ------------------ .../test/Http2/QueueWithMovableHeadTests.cs | 93 ------------ 3 files changed, 11 insertions(+), 248 deletions(-) delete mode 100644 src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs delete mode 100644 src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 4ef821efa536..bacded05a16f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -46,8 +46,9 @@ internal class Http2FrameWriter private readonly object _windowUpdateLock = new(); private long _connectionWindow; - private readonly QueueWithMovableHead _waitingForMoreConnectionWindow = new(); - private int? _waitingForMoreConnectionWindowStart; + private readonly Queue _waitingForMoreConnectionWindow = new(); + // This is the stream that consumed the last set of connection window + private Http2OutputProducer? _lastWindowConsumer; private readonly Task _writeQueueTask; public Http2FrameWriter( @@ -222,13 +223,10 @@ private async Task WriteToOutputPipe() lock (_windowUpdateLock) { // In order to make scheduling more fair we want to make sure that streams that have data get a chance to run in a round robin manner. - // To do this we will store everything in the queue waiting for the connection update - // but will store the index of the first element that had something to write but didn't write anything. This is the index that we will start dequeuing - // items from instead of the start of the queue. - if (actual == 0 && _waitingForMoreConnectionWindowStart is null) + // To do this we will store the producer that consumed the window in a field and put it to the back of the queue. + if (actual != 0 && _lastWindowConsumer is null) { - // This index will be stable until we start dequeueing everything - _waitingForMoreConnectionWindowStart = _waitingForMoreConnectionWindow.Enqueue(producer); + _lastWindowConsumer = producer; } else { @@ -874,20 +872,15 @@ public bool TryUpdateConnectionWindow(int bytes) _connectionWindow += bytes; - if (_waitingForMoreConnectionWindowStart is { } start) + if (_lastWindowConsumer is { } producer) { - // Reset the index - _waitingForMoreConnectionWindowStart = null; + _lastWindowConsumer = null; - // Set the head of the queue to the start so that we can start dequeueing from that position. - - // WARNING: Setting the head must be called once and then the queue must be drained. Interleaving - // calls to Enqueue/Dequeue will mutate the underlying data structure and will invalidate indexes - // returned from Enqueue. - _waitingForMoreConnectionWindow.SetHead(start); + // Put the consumer of the connection window last + _waitingForMoreConnectionWindow.Enqueue(producer); } - while (_waitingForMoreConnectionWindow.TryDequeue(out var producer)) + while (_waitingForMoreConnectionWindow.TryDequeue(out producer)) { if (!producer.StreamCompleted) { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs deleted file mode 100644 index a8f591e8f80f..000000000000 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/QueueWithMovableHead.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; - -namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; - -// This is a normal queue with a twist, we have the ability to set the head to a different start index -internal class QueueWithMovableHead -{ - private T[] _array; - private int _head; // The index from which to dequeue if the queue isn't empty. - private int _tail; // The index at which to enqueue if the queue isn't full. - private int _size; // Number of elements. - - public int Count => _size; - - // Creates a queue with room for capacity objects. The default initial - // capacity and grow factor are used. - public QueueWithMovableHead() - { - _array = Array.Empty(); - } - - public int Enqueue(T item) - { - if (_size == _array.Length) - { - Grow(_size + 1); - } - - var pos = _tail; - _array[_tail] = item; - MoveNext(ref _tail, _array.Length); - _size++; - return pos; - } - - // Sets the index of the head of the queue - public void SetHead(int head) - { - _head = head; - } - - public bool TryDequeue([MaybeNullWhen(false)] out T result) - { - var head = _head; - var array = _array; - - if (_size == 0) - { - result = default!; - return false; - } - - result = array[head]; - if (RuntimeHelpers.IsReferenceOrContainsReferences()) - { - array[head] = default!; - } - MoveNext(ref _head, _head < _tail ? _tail : _array.Length); - _size--; - - if (_size == 0) - { - // Restore the invarint that head = tail on empty queue - _head = _tail; - } - - return true; - } - - private static void MoveNext(ref int index, int end) - { - // It is tempting to use the remainder operator here but it is actually much slower - // than a simple comparison and a rarely taken branch. - // JIT produces better code than with ternary operator ?: - var tmp = index + 1; - if (tmp == end) - { - tmp = 0; - } - index = tmp; - } - - private void Grow(int capacity) - { - Debug.Assert(_array.Length < capacity); - - const int GrowFactor = 2; - const int MinimumGrow = 4; - - var newcapacity = GrowFactor * _array.Length; - - // Allow the list to grow to maximum possible capacity (~2G elements) before encountering overflow. - // Note that this check works even when _items.Length overflowed thanks to the (uint) cast - if ((uint)newcapacity > Array.MaxLength) - { - newcapacity = Array.MaxLength; - } - - // Ensure minimum growth is respected. - newcapacity = Math.Max(newcapacity, _array.Length + MinimumGrow); - - // If the computed capacity is still less than specified, set to the original argument. - // Capacities exceeding Array.MaxLength will be surfaced as OutOfMemoryException by Array.Resize. - if (newcapacity < capacity) - { - newcapacity = capacity; - } - - SetCapacity(newcapacity); - } - - private void SetCapacity(int capacity) - { - var newarray = new T[capacity]; - if (_size > 0) - { - if (_head < _tail) - { - Array.Copy(_array, _head, newarray, 0, _size); - } - else - { - Array.Copy(_array, _head, newarray, 0, _array.Length - _head); - Array.Copy(_array, 0, newarray, _array.Length - _head, _tail); - } - } - - _array = newarray; - _head = 0; - _tail = (_size == capacity) ? 0 : _size; - } -} diff --git a/src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs b/src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs deleted file mode 100644 index bdb521db029c..000000000000 --- a/src/Servers/Kestrel/Core/test/Http2/QueueWithMovableHeadTests.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2; - -namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests.Http2; - -public class QueueWithMovableHeadTests -{ - [Fact] - public void FIFOWorks() - { - var queue = new QueueWithMovableHead(); - queue.Enqueue(1); - queue.Enqueue(2); - - Assert.True(queue.TryDequeue(out var val1)); - Assert.True(queue.TryDequeue(out var val2)); - Assert.Equal(1, val1); - Assert.Equal(2, val2); - } - - [Fact] - public void SettingTheHeadOfTheQueueWorks() - { - var queue = new QueueWithMovableHead(); - queue.Enqueue(1); - queue.Enqueue(2); - queue.Enqueue(3); - queue.Enqueue(4); - - queue.SetHead(2); - - Assert.True(queue.TryDequeue(out var val1)); - Assert.True(queue.TryDequeue(out var val2)); - Assert.Equal(3, val1); - Assert.Equal(4, val2); - } - - [Fact] - public void SettingTheHeadOfTheQueueAndDrainingItWorks() - { - var queue = new QueueWithMovableHead(); - queue.Enqueue(1); - queue.Enqueue(2); - queue.Enqueue(3); - queue.Enqueue(4); - - queue.SetHead(2); - - Assert.True(queue.TryDequeue(out var val1)); - Assert.True(queue.TryDequeue(out var val2)); - Assert.True(queue.TryDequeue(out var val3)); - Assert.True(queue.TryDequeue(out var val4)); - Assert.False(queue.TryDequeue(out var val5)); - - Assert.Equal(3, val1); - Assert.Equal(4, val2); - Assert.Equal(1, val3); - Assert.Equal(2, val4); - } - - [Fact] - public void GrowingTheCapacitySettingTheHeadOfTheQueueAndDrainingItWorks() - { - var queue = new QueueWithMovableHead(); - - for (int i = 0; i < 2; i++) - { - for (var j = 0; j < 10; j++) - { - queue.Enqueue(j); - } - - queue.SetHead(5); - - var expected = 5; - for (var j = 0; j < 10; j++) - { - Assert.True(queue.TryDequeue(out var val)); - Assert.Equal(expected, val); - expected = (expected + 1) % 10; - } - - Assert.Equal(5, expected); - } - } -} From e41c7b32aa73bb17605aa7cffe9892515cddbf18 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 14 Apr 2022 00:33:24 -0700 Subject: [PATCH 71/77] Fixed 2 remaining issues - Over scheduling races (cancel during a dequeue that will complete the pipe) - Handle cancellation when there are trailers --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 16 +++++++++------- .../src/Internal/Http2/Http2OutputProducer.cs | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index bacded05a16f..6a10d8124694 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -112,7 +112,9 @@ private async Task WriteToOutputPipe() { while (await _channel.Reader.WaitToReadAsync()) { - while (_channel.Reader.TryRead(out var producer)) + // We need to handle the case where aborts can be scheduled while this loop is running and might be on the way to complete + // the reader. + while (_channel.Reader.TryRead(out var producer) && !producer.CompletedResponse) { try { @@ -149,10 +151,10 @@ private async Task WriteToOutputPipe() FlushResult flushResult = default; - // There are 2 cases where we abort if: - // 1. We're done writing data and there's no more data to be written - // 2. We're not done writing data but we got the abort message - if ((aborted && completed && actual == 0) || (aborted && !completed)) + // There are 2 cases where we abort: + // 1. We're not complete but we got the abort. + // 2. We're complete and there's no more response data to be written. + if ((aborted && !completed) || (aborted && completed && actual == 0 && stream.ResponseTrailers is null or { Count: 0 })) { // Response body is aborted, complete reader for this output producer. if (flushHeaders) @@ -850,7 +852,7 @@ private void AbortConnectionFlowControl() { while (_waitingForMoreConnectionWindow.TryDequeue(out var producer)) { - if (!producer.StreamCompleted) + if (!producer.CompletedResponse) { // Stop the output producer.Stop(); @@ -882,7 +884,7 @@ public bool TryUpdateConnectionWindow(int bytes) while (_waitingForMoreConnectionWindow.TryDequeue(out producer)) { - if (!producer.StreamCompleted) + if (!producer.CompletedResponse) { producer.Schedule(); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 41e4a186397e..0220bebeb092 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -73,13 +73,13 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) public bool StreamEnded => _streamEnded; - public bool StreamCompleted + public bool CompletedResponse { get { lock (_dataWriterLock) { - return _streamCompleted; + return _completedResponse; } } } From 1c542e1c8210e1e43de71dea79ae0c5fa82c092c Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 14 Apr 2022 10:07:08 -0700 Subject: [PATCH 72/77] Added last window consumer to the queue when aborting --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 6a10d8124694..f32d174eb04f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -850,7 +850,15 @@ private void AbortConnectionFlowControl() { lock (_windowUpdateLock) { - while (_waitingForMoreConnectionWindow.TryDequeue(out var producer)) + if (_lastWindowConsumer is { } producer) + { + _lastWindowConsumer = null; + + // Put the consumer of the connection window last + _waitingForMoreConnectionWindow.Enqueue(producer); + } + + while (_waitingForMoreConnectionWindow.TryDequeue(out producer)) { if (!producer.CompletedResponse) { From 618f7f27ee0505fe8cd91681a66793c2bef34d90 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 14 Apr 2022 18:14:06 -0700 Subject: [PATCH 73/77] Stop for RST frames as well. --- .../Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 0220bebeb092..031562a1876f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -370,6 +370,8 @@ public ValueTask WriteRstStreamAsync(Http2ErrorCode error) { lock (_dataWriterLock) { + Stop(); + // We queued the stream to complete but didn't complete the response yet if (_streamCompleted && !_completedResponse) { @@ -378,8 +380,6 @@ public ValueTask WriteRstStreamAsync(Http2ErrorCode error) return default; } - Stop(); - return _frameWriter.WriteRstStreamAsync(StreamId, error); } } From 6ab3f4f40845ca8d0ce437ba1aeed2e18b54b8ea Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 14 Apr 2022 20:07:41 -0700 Subject: [PATCH 74/77] Small tweaks to state management - Added current state to make sure state is terminal. We base decisions on the current state and use the unobserved state to determine when to reschedule. - Remove CancelPendingRead since there's never a pending read and we're scheduling the pipe reads manually. - Rename the Canceled state to Aborted --- .../src/Internal/Http2/Http2FrameWriter.cs | 26 +++++++------ .../src/Internal/Http2/Http2OutputProducer.cs | 39 ++++++++++++------- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index f32d174eb04f..ba9c554f0965 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -127,14 +127,6 @@ private async Task WriteToOutputPipe() reader.TryRead(out var readResult); var buffer = readResult.Buffer; - // Stash the unobserved state, we're going to mark this snapshot as observed - var observed = producer.UnobservedState; - var flushHeaders = observed.HasFlag(Http2OutputProducer.State.FlushHeaders); - var aborted = observed.HasFlag(Http2OutputProducer.State.Canceled); - - // Completed is special because it's a terminal state that should ideally never be removed. - var completed = observed.HasFlag(Http2OutputProducer.State.Completed) || readResult.IsCompleted; - // Check the stream window var (actual, remainingStream) = producer.ConsumeStreamWindow(buffer.Length); @@ -147,7 +139,17 @@ private async Task WriteToOutputPipe() buffer = buffer.Slice(0, actual); } - var (hasMoreData, nextState) = producer.ObserveDataAndState(buffer.Length, observed); + // Stash the unobserved state, we're going to mark this snapshot as observed + var observed = producer.UnobservedState; + var currentState = producer.CurrentState; + + // Check if we need to write headers + var flushHeaders = observed.HasFlag(Http2OutputProducer.State.FlushHeaders) && !currentState.HasFlag(Http2OutputProducer.State.FlushHeaders); + + (var hasMoreData, var reschedule, currentState) = producer.ObserveDataAndState(buffer.Length, observed); + + var aborted = currentState.HasFlag(Http2OutputProducer.State.Aborted); + var completed = currentState.HasFlag(Http2OutputProducer.State.Completed); FlushResult flushResult = default; @@ -175,7 +177,7 @@ private async Task WriteToOutputPipe() // It is faster to write data and trailers together. Locking once reduces lock contention. flushResult = await WriteDataAndTrailersAsync(stream, buffer, flushHeaders, stream.ResponseTrailers); } - else if (completed && producer.StreamEnded) + else if (completed && producer.AppCompletedWithNoResponseBodyOrTrailers) { if (buffer.Length != 0) { @@ -216,7 +218,7 @@ private async Task WriteToOutputPipe() } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. - else if (hasMoreData && !nextState.HasFlag(Http2OutputProducer.State.Canceled)) + else if (hasMoreData && !aborted) { // We have no more connection window, put this producer in a queue waiting for it to // a window update to resume the connection. @@ -258,7 +260,7 @@ private async Task WriteToOutputPipe() } } } - else if (nextState != Http2OutputProducer.State.None) + else if (reschedule) { producer.Schedule(); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 031562a1876f..9e0b63187bd4 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -30,7 +30,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private bool _startedWritingDataFrames; private bool _streamCompleted; private bool _suffixSent; - private bool _streamEnded; + private bool _appCompletedWithNoResponseBodyOrTrailers; private bool _writerComplete; private bool _isScheduled; @@ -40,8 +40,11 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private long _unconsumedBytes; private long _streamWindow; - // For changes scheduling changes that don't affect the number of bytes written to the pipe, we need another state + // For scheduling changes that don't affect the number of bytes written to the pipe, we need another state. private State _unobservedState; + + // This reflects the current state of the output, the current state becomes the unobserved state after it has been observed. + private State _currentState; private bool _completedResponse; private bool _requestProcessingComplete; private Http2ErrorCode? _resetErrorCode; @@ -71,7 +74,7 @@ public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) public bool IsTimingWrite { get; set; } - public bool StreamEnded => _streamEnded; + public bool AppCompletedWithNoResponseBodyOrTrailers => _appCompletedWithNoResponseBodyOrTrailers; public bool CompletedResponse { @@ -85,7 +88,7 @@ public bool CompletedResponse } // Useful for debugging the scheduling state in the debugger - internal (int, long, State, long) SchedulingState => (Stream.StreamId, _unconsumedBytes, _unobservedState, _streamWindow); + internal (int, long, State, State, long) SchedulingState => (Stream.StreamId, _unconsumedBytes, _unobservedState, _currentState, _streamWindow); public State UnobservedState { @@ -98,6 +101,17 @@ public State UnobservedState } } + public State CurrentState + { + get + { + lock (_dataWriterLock) + { + return _currentState; + } + } + } + // Added bytes to the queue. // Returns a bool that represents whether we should schedule this producer to write // the enqueued bytes @@ -122,14 +136,15 @@ private void EnqueueStateUpdate(State state) // Removes consumed bytes from the queue. // Returns a bool that represents whether we should schedule this producer to write // the remaining bytes. - internal (bool hasMoreData, State unobservedState) ObserveDataAndState(long bytes, State state) + internal (bool hasMoreData, bool reschedule, State currentState) ObserveDataAndState(long bytes, State state) { lock (_dataWriterLock) { _isScheduled = false; _unobservedState &= ~state; + _currentState |= state; _unconsumedBytes -= bytes; - return (_unconsumedBytes > 0, _unobservedState); + return (_unconsumedBytes > 0, _unobservedState != State.None, _currentState); } } @@ -149,7 +164,7 @@ public void StreamReset(uint initialWindowSize) // Response should have been completed. Debug.Assert(_completedResponse); - _streamEnded = false; + _appCompletedWithNoResponseBodyOrTrailers = false; _suffixSent = false; _startedWritingDataFrames = false; _streamCompleted = false; @@ -160,6 +175,7 @@ public void StreamReset(uint initialWindowSize) _streamWindow = initialWindowSize; _unconsumedBytes = 0; _unobservedState = State.None; + _currentState = State.None; _completedResponse = false; _requestProcessingComplete = false; _resetErrorCode = null; @@ -305,7 +321,7 @@ public void WriteResponseHeaders(int statusCode, string? reasonPhrase, HttpRespo // 2. There is no trailing HEADERS frame. if (appCompleted && !_startedWritingDataFrames && (_stream.ResponseTrailers == null || _stream.ResponseTrailers.Count == 0)) { - _streamEnded = true; + _appCompletedWithNoResponseBodyOrTrailers = true; } EnqueueStateUpdate(State.FlushHeaders); @@ -371,7 +387,6 @@ public ValueTask WriteRstStreamAsync(Http2ErrorCode error) lock (_dataWriterLock) { Stop(); - // We queued the stream to complete but didn't complete the response yet if (_streamCompleted && !_completedResponse) { @@ -510,9 +525,7 @@ public void Stop() _streamCompleted = true; - EnqueueStateUpdate(State.Canceled); - - _pipeReader.CancelPendingRead(); + EnqueueStateUpdate(State.Aborted); Schedule(); } @@ -704,7 +717,7 @@ public enum State { None = 0, FlushHeaders = 1, - Canceled = 2, + Aborted = 2, Completed = 4 } } From d6e0a099b9f6ff2437ecbd520e61c368bcc16a3c Mon Sep 17 00:00:00 2001 From: David Fowler Date: Fri, 15 Apr 2022 17:52:07 -0700 Subject: [PATCH 75/77] This is the last commit that fixes all of the bugs (I hope...) --- .../src/Internal/Http2/Http2FrameWriter.cs | 29 +++++++++------- .../src/Internal/Http2/Http2OutputProducer.cs | 33 +++++++++++++++++-- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index ba9c554f0965..55aebb139a53 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -146,7 +146,7 @@ private async Task WriteToOutputPipe() // Check if we need to write headers var flushHeaders = observed.HasFlag(Http2OutputProducer.State.FlushHeaders) && !currentState.HasFlag(Http2OutputProducer.State.FlushHeaders); - (var hasMoreData, var reschedule, currentState) = producer.ObserveDataAndState(buffer.Length, observed); + (var hasMoreData, var reschedule, currentState, var waitingForWindowUpdates) = producer.ObserveDataAndState(buffer.Length, observed); var aborted = currentState.HasFlag(Http2OutputProducer.State.Aborted); var completed = currentState.HasFlag(Http2OutputProducer.State.Completed); @@ -164,6 +164,15 @@ private async Task WriteToOutputPipe() // write headers WriteResponseHeaders(stream.StreamId, stream.StatusCode, Http2HeadersFrameFlags.NONE, (HttpResponseHeaders)stream.ResponseHeaders); } + + if (actual > 0) + { + // If we got here it means we're going to cancel the write. Restore any consumed bytes to the connection window. + lock (_windowUpdateLock) + { + _connectionWindow += actual; + } + } } else if (completed && stream.ResponseTrailers is { Count: > 0 } && !hasMoreData) { @@ -218,7 +227,7 @@ private async Task WriteToOutputPipe() } // We're not going to schedule this again if there's no remaining window. // When the window update is sent, the producer will be re-queued if needed. - else if (hasMoreData && !aborted) + else if (hasMoreData && !aborted && !waitingForWindowUpdates) { // We have no more connection window, put this producer in a queue waiting for it to // a window update to resume the connection. @@ -238,6 +247,8 @@ private async Task WriteToOutputPipe() } } + producer.SetWaitingForWindowUpdates(); + // Include waiting for window updates in timing writes if (_minResponseDataRate != null) { @@ -252,6 +263,8 @@ private async Task WriteToOutputPipe() } else { + producer.SetWaitingForWindowUpdates(); + // Include waiting for window updates in timing writes if (_minResponseDataRate != null) { @@ -862,11 +875,8 @@ private void AbortConnectionFlowControl() while (_waitingForMoreConnectionWindow.TryDequeue(out producer)) { - if (!producer.CompletedResponse) - { - // Stop the output - producer.Stop(); - } + // Abort the stream + producer.Stop(); } } } @@ -894,10 +904,7 @@ public bool TryUpdateConnectionWindow(int bytes) while (_waitingForMoreConnectionWindow.TryDequeue(out producer)) { - if (!producer.CompletedResponse) - { - producer.Schedule(); - } + producer.ScheduleResumeFromWindowUpdate(); } } return true; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 9e0b63187bd4..d58ea5827720 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -47,6 +47,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private State _currentState; private bool _completedResponse; private bool _requestProcessingComplete; + private bool _waitingForWindowUpdates; private Http2ErrorCode? _resetErrorCode; public Http2OutputProducer(Http2Stream stream, Http2StreamContext context) @@ -133,10 +134,18 @@ private void EnqueueStateUpdate(State state) } } + public void SetWaitingForWindowUpdates() + { + lock (_dataWriterLock) + { + _waitingForWindowUpdates = true; + } + } + // Removes consumed bytes from the queue. // Returns a bool that represents whether we should schedule this producer to write // the remaining bytes. - internal (bool hasMoreData, bool reschedule, State currentState) ObserveDataAndState(long bytes, State state) + internal (bool hasMoreData, bool reschedule, State currentState, bool waitingForWindowUpdates) ObserveDataAndState(long bytes, State state) { lock (_dataWriterLock) { @@ -144,7 +153,7 @@ private void EnqueueStateUpdate(State state) _unobservedState &= ~state; _currentState |= state; _unconsumedBytes -= bytes; - return (_unconsumedBytes > 0, _unobservedState != State.None, _currentState); + return (_unconsumedBytes > 0, _unobservedState != State.None, _currentState, _waitingForWindowUpdates); } } @@ -178,6 +187,7 @@ public void StreamReset(uint initialWindowSize) _currentState = State.None; _completedResponse = false; _requestProcessingComplete = false; + _waitingForWindowUpdates = false; _resetErrorCode = null; IsTimingWrite = false; } @@ -286,6 +296,21 @@ public void Schedule() _frameWriter.Schedule(this); } + public void ScheduleResumeFromWindowUpdate() + { + if (_completedResponse) + { + return; + } + + lock (_dataWriterLock) + { + _waitingForWindowUpdates = false; + } + + Schedule(); + } + public ValueTask Write100ContinueAsync() { lock (_dataWriterLock) @@ -516,6 +541,8 @@ public void Stop() { lock (_dataWriterLock) { + _waitingForWindowUpdates = false; + if (_streamCompleted && _completedResponse) { // We can overschedule as long as we haven't yet completed the response. This is important because @@ -646,7 +673,7 @@ public bool TryUpdateStreamWindow(int bytes) if (schedule) { - Schedule(); + ScheduleResumeFromWindowUpdate(); } return true; From 77ff682a3f901a4868ca55f3776645f8dbc46b7b Mon Sep 17 00:00:00 2001 From: David Fowler Date: Fri, 15 Apr 2022 18:05:43 -0700 Subject: [PATCH 76/77] Make things "simpler" --- .../src/Internal/Http2/Http2FrameWriter.cs | 8 ++--- .../src/Internal/Http2/Http2OutputProducer.cs | 35 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 55aebb139a53..6ca49fae28d9 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -149,7 +149,7 @@ private async Task WriteToOutputPipe() (var hasMoreData, var reschedule, currentState, var waitingForWindowUpdates) = producer.ObserveDataAndState(buffer.Length, observed); var aborted = currentState.HasFlag(Http2OutputProducer.State.Aborted); - var completed = currentState.HasFlag(Http2OutputProducer.State.Completed); + var completed = currentState.HasFlag(Http2OutputProducer.State.Completed) && !hasMoreData; FlushResult flushResult = default; @@ -174,7 +174,7 @@ private async Task WriteToOutputPipe() } } } - else if (completed && stream.ResponseTrailers is { Count: > 0 } && !hasMoreData) + else if (completed && stream.ResponseTrailers is { Count: > 0 }) { // Output is ending and there are trailers to write // Write any remaining content then write trailers and there's no @@ -202,7 +202,7 @@ private async Task WriteToOutputPipe() } else { - var endStream = completed && !hasMoreData; + var endStream = completed; if (endStream) { @@ -219,7 +219,7 @@ private async Task WriteToOutputPipe() reader.AdvanceTo(buffer.End); - if ((completed && !hasMoreData) || aborted) + if (completed || aborted) { await reader.CompleteAsync(); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index d58ea5827720..6440b2f16d19 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -28,7 +28,7 @@ internal class Http2OutputProducer : IHttpOutputProducer, IHttpOutputAborter, ID private IMemoryOwner? _fakeMemoryOwner; private byte[]? _fakeMemory; private bool _startedWritingDataFrames; - private bool _streamCompleted; + private bool _completeScheduled; private bool _suffixSent; private bool _appCompletedWithNoResponseBodyOrTrailers; private bool _writerComplete; @@ -176,7 +176,7 @@ public void StreamReset(uint initialWindowSize) _appCompletedWithNoResponseBodyOrTrailers = false; _suffixSent = false; _startedWritingDataFrames = false; - _streamCompleted = false; + _completeScheduled = false; _writerComplete = false; _pipe.Reset(); _pipeWriter.Reset(); @@ -205,7 +205,7 @@ public void Complete() Stop(); - if (!_streamCompleted) + if (!_completeScheduled) { EnqueueStateUpdate(State.Completed); @@ -256,7 +256,8 @@ public ValueTask FlushAsync(CancellationToken cancellationToken) lock (_dataWriterLock) { ThrowIfSuffixSentOrCompleted(); - if (_streamCompleted) + + if (_completeScheduled) { return new ValueTask(new FlushResult(false, true)); } @@ -317,7 +318,7 @@ public ValueTask Write100ContinueAsync() { ThrowIfSuffixSentOrCompleted(); - if (_streamCompleted) + if (_completeScheduled) { return default; } @@ -332,7 +333,7 @@ public void WriteResponseHeaders(int statusCode, string? reasonPhrase, HttpRespo { // The HPACK header compressor is stateful, if we compress headers for an aborted stream we must send them. // Optimize for not compressing or sending them. - if (_streamCompleted) + if (_completeScheduled) { return; } @@ -366,7 +367,7 @@ public Task WriteDataAsync(ReadOnlySpan data, CancellationToken cancellati // This length check is important because we don't want to set _startedWritingDataFrames unless a data // frame will actually be written causing the headers to be flushed. - if (_streamCompleted || data.Length == 0) + if (_completeScheduled || data.Length == 0) { return Task.CompletedTask; } @@ -389,12 +390,12 @@ public ValueTask WriteStreamSuffixAsync() { lock (_dataWriterLock) { - if (_streamCompleted) + if (_completeScheduled) { return ValueTask.FromResult(default); } - _streamCompleted = true; + _completeScheduled = true; _suffixSent = true; EnqueueStateUpdate(State.Completed); @@ -413,7 +414,7 @@ public ValueTask WriteRstStreamAsync(Http2ErrorCode error) { Stop(); // We queued the stream to complete but didn't complete the response yet - if (_streamCompleted && !_completedResponse) + if (_completeScheduled && !_completedResponse) { // Set the error so that we can write the RST when the response completes. _resetErrorCode = error; @@ -430,7 +431,7 @@ public void Advance(int bytes) { ThrowIfSuffixSentOrCompleted(); - if (_streamCompleted) + if (_completeScheduled) { return; } @@ -449,7 +450,7 @@ public Span GetSpan(int sizeHint = 0) { ThrowIfSuffixSentOrCompleted(); - if (_streamCompleted) + if (_completeScheduled) { return GetFakeMemory(sizeHint).Span; } @@ -464,7 +465,7 @@ public Memory GetMemory(int sizeHint = 0) { ThrowIfSuffixSentOrCompleted(); - if (_streamCompleted) + if (_completeScheduled) { return GetFakeMemory(sizeHint); } @@ -477,7 +478,7 @@ public void CancelPendingFlush() { lock (_dataWriterLock) { - if (_streamCompleted) + if (_completeScheduled) { return; } @@ -499,7 +500,7 @@ public ValueTask WriteDataToPipeAsync(ReadOnlySpan data, Canc // This length check is important because we don't want to set _startedWritingDataFrames unless a data // frame will actually be written causing the headers to be flushed. - if (_streamCompleted || data.Length == 0) + if (_completeScheduled || data.Length == 0) { return new ValueTask(new FlushResult(false, true)); } @@ -543,14 +544,14 @@ public void Stop() { _waitingForWindowUpdates = false; - if (_streamCompleted && _completedResponse) + if (_completeScheduled && _completedResponse) { // We can overschedule as long as we haven't yet completed the response. This is important because // we may need to abort the stream if it's waiting for a window update. return; } - _streamCompleted = true; + _completeScheduled = true; EnqueueStateUpdate(State.Aborted); From 02319f0d840a7dd60f0316380114708531845d9a Mon Sep 17 00:00:00 2001 From: David Fowler Date: Fri, 15 Apr 2022 19:03:41 -0700 Subject: [PATCH 77/77] We're not boxing today --- .../Core/src/Internal/Http2/Http2FrameWriter.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index 6ca49fae28d9..d7a86bd31423 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -143,13 +143,17 @@ private async Task WriteToOutputPipe() var observed = producer.UnobservedState; var currentState = producer.CurrentState; + // Avoid boxing the enum (though the JIT optimizes this eventually) + static bool HasStateFlag(Http2OutputProducer.State state, Http2OutputProducer.State flags) + => (state & flags) == flags; + // Check if we need to write headers - var flushHeaders = observed.HasFlag(Http2OutputProducer.State.FlushHeaders) && !currentState.HasFlag(Http2OutputProducer.State.FlushHeaders); + var flushHeaders = HasStateFlag(observed, Http2OutputProducer.State.FlushHeaders) && !HasStateFlag(currentState, Http2OutputProducer.State.FlushHeaders); (var hasMoreData, var reschedule, currentState, var waitingForWindowUpdates) = producer.ObserveDataAndState(buffer.Length, observed); - var aborted = currentState.HasFlag(Http2OutputProducer.State.Aborted); - var completed = currentState.HasFlag(Http2OutputProducer.State.Completed) && !hasMoreData; + var aborted = HasStateFlag(currentState, Http2OutputProducer.State.Aborted); + var completed = HasStateFlag(currentState, Http2OutputProducer.State.Completed) && !hasMoreData; FlushResult flushResult = default;