Add custom request header decoder API to Kestrel

Author: halter73

src/Servers/Kestrel/Core/src/Internal/Http/HttpHeaders.Generated.cs

@@ -369,7 +369,7 @@ public void Reset()
             ConnectionIdFeature = ConnectionId;
 
             HttpRequestHeaders.Reset();
-            HttpRequestHeaders.UseLatin1 = ServerOptions.Latin1RequestHeaders;
+            HttpRequestHeaders.EncodingSelector = ServerOptions.GetRequestHeaderEncodingSelector();
             HttpRequestHeaders.ReuseHeaderValues = !ServerOptions.DisableStringReuse;
             HttpResponseHeaders.Reset();
             RequestHeaders = HttpRequestHeaders;
@@ -532,7 +532,7 @@ public void OnTrailer(ReadOnlySpan<byte> name, ReadOnlySpan<byte> value)
             }
 
             string key = name.GetHeaderName();
-            var valueStr = value.GetRequestHeaderStringNonNullCharacters(ServerOptions.Latin1RequestHeaders);
+            var valueStr = value.GetRequestHeaderString(key, HttpRequestHeaders.EncodingSelector);
             RequestTrailers.Append(key, valueStr);
         }
 

src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs

@@ -5,7 +5,9 @@
 using System.Buffers.Text;
 using System.Collections;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Runtime.CompilerServices;
+using System.Text;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.Extensions.Primitives;
 using Microsoft.Net.Http.Headers;
@@ -17,12 +19,12 @@ internal sealed partial class HttpRequestHeaders : HttpHeaders
         private long _previousBits = 0;
 
         public bool ReuseHeaderValues { get; set; }
-        public bool UseLatin1 { get; set; }
+        public Func<string, Encoding> EncodingSelector { get; set; }
 
-        public HttpRequestHeaders(bool reuseHeaderValues = true, bool useLatin1 = false)
+        public HttpRequestHeaders(bool reuseHeaderValues = true, Func<string, Encoding> encodingSelector = null)
         {
             ReuseHeaderValues = reuseHeaderValues;
-            UseLatin1 = useLatin1;
+            EncodingSelector = encodingSelector ?? KestrelServerOptions.DefaultRequestHeaderEncodingSelector;
         }
 
         public void OnHeadersComplete()
@@ -87,12 +89,36 @@ private void AppendContentLength(ReadOnlySpan<byte> value)
                 parsed < 0 ||
                 consumed != value.Length)
             {
-                KestrelBadHttpRequestException.Throw(RequestRejectionReason.InvalidContentLength, value.GetRequestHeaderStringNonNullCharacters(UseLatin1));
+                KestrelBadHttpRequestException.Throw(RequestRejectionReason.InvalidContentLength, value.GetRequestHeaderString(HeaderNames.ContentLength, EncodingSelector));
             }
 
             _contentLength = parsed;
         }
 
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void AppendContentLengthCustomEncoding(ReadOnlySpan<byte> value, Encoding customEncoding)
+        {
+            if (_contentLength.HasValue)
+            {
+                KestrelBadHttpRequestException.Throw(RequestRejectionReason.MultipleContentLengths);
+            }
+
+            // long.MaxValue = 9223372036854775807 (19 chars)
+            Span<char> decodedChars = stackalloc char[20];
+            var numChars = customEncoding.GetChars(value, decodedChars);
+            long parsed = -1;
+
+            if (numChars > 19 ||
+                !long.TryParse(decodedChars.Slice(0, numChars), NumberStyles.Integer, CultureInfo.InvariantCulture, out parsed) ||
+                parsed < 0)
+            {
+                KestrelBadHttpRequestException.Throw(RequestRejectionReason.InvalidContentLength, value.GetRequestHeaderString(HeaderNames.ContentLength, EncodingSelector));
+            }
+
+            _contentLength = parsed;
+        }
+
+
         [MethodImpl(MethodImplOptions.NoInlining)]
         private void SetValueUnknown(string key, StringValues value)
         {
@@ -108,11 +134,10 @@ private bool AddValueUnknown(string key, StringValues value)
         }
 
         [MethodImpl(MethodImplOptions.NoInlining)]
-        private unsafe void AppendUnknownHeaders(ReadOnlySpan<byte> name, string valueString)
+        private unsafe void AppendUnknownHeaders(string name, string valueString)
         {
-            string key = name.GetHeaderName();
-            Unknown.TryGetValue(key, out var existing);
-            Unknown[key] = AppendValue(existing, valueString);
+            Unknown.TryGetValue(name, out var existing);
+            Unknown[name] = AppendValue(existing, valueString);
         }
 
         public Enumerator GetEnumerator()

src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestHeaders.cs

@@ -9,6 +9,7 @@
 using System.Runtime.InteropServices;
 using System.Text;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
@@ -27,7 +28,6 @@ internal static partial class HttpUtilities
         private const ulong _http10VersionLong = 3471766442030158920; // GetAsciiStringAsLong("HTTP/1.0"); const results in better codegen
         private const ulong _http11VersionLong = 3543824036068086856; // GetAsciiStringAsLong("HTTP/1.1"); const results in better codegen
 
-        private static readonly UTF8EncodingSealed HeaderValueEncoding = new UTF8EncodingSealed();
         private static readonly SpanAction<char, IntPtr> _getHeaderName = GetHeaderName;
         private static readonly SpanAction<char, IntPtr> _getAsciiStringNonNullCharacters = GetAsciiStringNonNullCharacters;
 
@@ -120,11 +120,8 @@ public static unsafe string GetAsciiStringNonNullCharacters(this ReadOnlySpan<by
             }
         }
 
-        public static string GetAsciiOrUTF8StringNonNullCharacters(this Span<byte> span)
-            => GetAsciiOrUTF8StringNonNullCharacters((ReadOnlySpan<byte>)span);
-
         public static string GetAsciiOrUTF8StringNonNullCharacters(this ReadOnlySpan<byte> span)
-            => StringUtilities.GetAsciiOrUTF8StringNonNullCharacters(span, HeaderValueEncoding);
+            => StringUtilities.GetAsciiOrUTF8StringNonNullCharacters(span, KestrelServerOptions.DefaultRequestHeaderEncoding);
 
         private static unsafe void GetAsciiStringNonNullCharacters(Span<char> buffer, IntPtr state)
         {
@@ -139,8 +136,34 @@ private static unsafe void GetAsciiStringNonNullCharacters(Span<char> buffer, In
             }
         }
 
-        public static string GetRequestHeaderStringNonNullCharacters(this ReadOnlySpan<byte> span, bool useLatin1) =>
-            useLatin1 ? span.GetLatin1StringNonNullCharacters() : span.GetAsciiOrUTF8StringNonNullCharacters(HeaderValueEncoding);
+        public static string GetRequestHeaderString(this ReadOnlySpan<byte> span, string name, Func<string, Encoding> encodingSelector)
+        {
+            if (ReferenceEquals(KestrelServerOptions.DefaultRequestHeaderEncodingSelector, encodingSelector))
+            {
+                return span.GetAsciiOrUTF8StringNonNullCharacters(KestrelServerOptions.DefaultRequestHeaderEncoding);
+            }
+
+            var encoding = encodingSelector(name);
+
+            if (encoding is null)
+            {
+                return span.GetAsciiOrUTF8StringNonNullCharacters(KestrelServerOptions.DefaultRequestHeaderEncoding);
+            }
+
+            if (ReferenceEquals(encoding, Encoding.Latin1))
+            {
+                return span.GetLatin1StringNonNullCharacters();
+            }
+
+            try
+            {
+                return encoding.GetString(span);
+            }
+            catch (DecoderFallbackException)
+            {
+                throw new InvalidOperationException();
+            }
+        }
 
         public static string GetAsciiStringEscaped(this ReadOnlySpan<byte> span, int maxChars)
         {
@@ -529,13 +552,5 @@ private static bool IsHex(char ch)
                 // Check if less than 6 representing chars 'a' - 'f'
                 || (uint)((ch | 32) - 'a') < 6u;
         }
-
-        // Allow for de-virtualization (see https://github.com/dotnet/coreclr/pull/9230)
-        private sealed class UTF8EncodingSealed : UTF8Encoding
-        {
-            public UTF8EncodingSealed() : base(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true) { }
-
-            public override byte[] GetPreamble() => Array.Empty<byte>();
-        }
     }
 }

src/Servers/Kestrel/Core/src/Internal/Infrastructure/HttpUtilities.cs

@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Text;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure
+{
+    // Allow for de-virtualization (see https://github.com/dotnet/coreclr/pull/9230)
+    internal sealed class UTF8EncodingSealed : UTF8Encoding
+    {
+        public UTF8EncodingSealed() : base(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true) { }
+
+        public override byte[] GetPreamble() => Array.Empty<byte>();
+    }
+}

src/Servers/Kestrel/Core/src/Internal/Infrastructure/UTF8EncodingSealed.cs

@@ -5,11 +5,13 @@
 using System.Collections.Generic;
 using System.IO.Pipelines;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Connections;
 using Microsoft.AspNetCore.Hosting.Server;
 using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
@@ -35,12 +37,19 @@ public class KestrelServer : IServer
 
         private IDisposable _configChangedRegistration;
 
-        public KestrelServer(IOptions<KestrelServerOptions> options, IEnumerable<IConnectionListenerFactory> transportFactories, ILoggerFactory loggerFactory)
+        public KestrelServer(
+            IOptions<KestrelServerOptions> options,
+            IEnumerable<IConnectionListenerFactory> transportFactories,
+            ILoggerFactory loggerFactory)
             : this(transportFactories, null, CreateServiceContext(options, loggerFactory))
         {
         }
 
-        public KestrelServer(IOptions<KestrelServerOptions> options, IEnumerable<IConnectionListenerFactory> transportFactories, IEnumerable<IMultiplexedConnectionListenerFactory> multiplexedFactories, ILoggerFactory loggerFactory)
+        public KestrelServer(
+            IOptions<KestrelServerOptions> options,
+            IEnumerable<IConnectionListenerFactory> transportFactories,
+            IEnumerable<IMultiplexedConnectionListenerFactory> multiplexedFactories,
+            ILoggerFactory loggerFactory)
             : this(transportFactories, multiplexedFactories, CreateServiceContext(options, loggerFactory))
         {
         }
@@ -52,7 +61,10 @@ internal KestrelServer(IEnumerable<IConnectionListenerFactory> transportFactorie
         }
 
         // For testing
-        internal KestrelServer(IEnumerable<IConnectionListenerFactory> transportFactories, IEnumerable<IMultiplexedConnectionListenerFactory> multiplexedFactories, ServiceContext serviceContext)
+        internal KestrelServer(
+            IEnumerable<IConnectionListenerFactory> transportFactories,
+            IEnumerable<IMultiplexedConnectionListenerFactory> multiplexedFactories,
+            ServiceContext serviceContext)
         {
             if (transportFactories == null)
             {
@@ -362,6 +374,11 @@ private void ValidateOptions()
                 throw new InvalidOperationException(
                     CoreStrings.FormatMaxRequestBufferSmallerThanRequestHeaderBuffer(Options.Limits.MaxRequestBufferSize.Value, Options.Limits.MaxRequestHeadersTotalSize));
             }
+
+            if (Options.RequestHeaderEncodingSelector is null)
+            {
+                throw new InvalidOperationException($"{nameof(KestrelServerOptions)}.{nameof(KestrelServerOptions.RequestHeaderEncodingSelector)} must not be null.");
+            }
         }
 
         private static ConnectionDelegate EnforceConnectionLimit(ConnectionDelegate innerDelegate, long? connectionLimit, IKestrelTrace trace)

src/Servers/Kestrel/Core/src/KestrelServer.cs

@@ -1,15 +1,19 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+#nullable enable
+
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Security.Cryptography.X509Certificates;
+using System.Text;
 using Microsoft.AspNetCore.Certificates.Generation;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
+using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.AspNetCore.Server.Kestrel.Https;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -22,6 +26,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core
     /// </summary>
     public class KestrelServerOptions
     {
+        // Internal to fast-path header decoding when RequestHeaderEncodingSelector is unchanged.
+        internal static readonly UTF8EncodingSealed DefaultRequestHeaderEncoding = new UTF8EncodingSealed();
+        internal static readonly Func<string, Encoding> DefaultRequestHeaderEncodingSelector = _ => DefaultRequestHeaderEncoding;
+        internal static readonly Func<string, Encoding> DefaultLatin1RequestHeaderEncodingSelector = _ => Encoding.Latin1;
+
         // The following two lists configure the endpoints that Kestrel should listen to. If both lists are empty, the "urls" config setting (e.g. UseUrls) is used.
         internal List<ListenOptions> CodeBackedListenOptions { get; } = new List<ListenOptions>();
         internal List<ListenOptions> ConfigurationBackedListenOptions { get; } = new List<ListenOptions>();
@@ -65,11 +74,27 @@ public class KestrelServerOptions
         /// </remarks>
         public bool DisableStringReuse { get; set; } = false;
 
+        /// <summary>
+        /// Controls whether to return the AltSvcHeader from on an HTTP/2 or lower response for HTTP/3
+        /// </summary>
+        /// <remarks>
+        /// Defaults to false.
+        /// </remarks>
+        public bool EnableAltSvc { get; set; } = false;
+
+        /// <summary>
+        /// Gets or sets a callback that returns the <see cref="Encoding"/> to decode the value for the specified request header name.
+        /// </summary>
+        /// <remarks>
+        /// Defaults to returning a <see cref="UTF8Encoding"/> for all headers.
+        /// </remarks>
+        public Func<string, Encoding?> RequestHeaderEncodingSelector { get; set; } = DefaultRequestHeaderEncodingSelector;
+
         /// <summary>
         /// Enables the Listen options callback to resolve and use services registered by the application during startup.
         /// Typically initialized by UseKestrel()"/>.
         /// </summary>
-        public IServiceProvider ApplicationServices { get; set; }
+        public IServiceProvider? ApplicationServices { get; set; }
 
         /// <summary>
         /// Provides access to request limit options.
@@ -80,12 +105,7 @@ public class KestrelServerOptions
         /// Provides a configuration source where endpoints will be loaded from on server start.
         /// The default is null.
         /// </summary>
-        public KestrelConfigurationLoader ConfigurationLoader { get; set; }
-
-        /// <summary>
-        /// Controls whether to return the AltSvcHeader from on an HTTP/2 or lower response for HTTP/3
-        /// </summary>
-        public bool EnableAltSvc { get; set; } = false;
+        public KestrelConfigurationLoader? ConfigurationLoader { get; set; }
 
         /// <summary>
         /// A default configuration action for all endpoints. Use for Listen, configuration, the default url, and URLs.
@@ -100,7 +120,7 @@ public class KestrelServerOptions
         /// <summary>
         /// The default server certificate for https endpoints. This is applied lazily after HttpsDefaults and user options.
         /// </summary>
-        internal X509Certificate2 DefaultCertificate { get; set; }
+        internal X509Certificate2? DefaultCertificate { get; set; }
 
         /// <summary>
         /// Has the default dev certificate load been attempted?
@@ -121,9 +141,19 @@ public void ConfigureEndpointDefaults(Action<ListenOptions> configureOptions)
             EndpointDefaults = configureOptions ?? throw new ArgumentNullException(nameof(configureOptions));
         }
 
+        internal Func<string, Encoding?> GetRequestHeaderEncodingSelector()
+        {
+            if (ReferenceEquals(RequestHeaderEncodingSelector, DefaultRequestHeaderEncodingSelector) && Latin1RequestHeaders)
+            {
+                return DefaultLatin1RequestHeaderEncodingSelector;
+            }
+
+            return RequestHeaderEncodingSelector;
+        }
+
         internal void ApplyEndpointDefaults(ListenOptions listenOptions)
         {
-            listenOptions.KestrelServerOptions = this;
+            listenOptions.KestrelServerOptions = this; 
             ConfigurationLoader?.ApplyConfigurationDefaults(listenOptions);
             EndpointDefaults(listenOptions);
         }
@@ -159,7 +189,7 @@ private void EnsureDefaultCert()
             if (DefaultCertificate == null && !IsDevCertLoaded)
             {
                 IsDevCertLoaded = true; // Only try once
-                var logger = ApplicationServices.GetRequiredService<ILogger<KestrelServer>>();
+                var logger = ApplicationServices!.GetRequiredService<ILogger<KestrelServer>>();
                 try
                 {
                     DefaultCertificate = CertificateManager.Instance.ListCertificates(StoreName.My, StoreLocation.CurrentUser, isValid: true)
@@ -220,10 +250,11 @@ private void EnsureDefaultCert()
         /// </summary>
         /// <param name="config">The configuration section for Kestrel.</param>
         /// <param name="reloadOnChange">
-        /// If <see langword="true" />, Kestrel will dynamically update endpoint bindings when configuration changes.
+        /// If <see langword="true"/>, Kestrel will dynamically update endpoint bindings when configuration changes.
         /// This will only reload endpoints defined in the "Endpoints" section of your <paramref name="config"/>. Endpoints defined in code will not be reloaded.
         /// </param>
         /// <returns>A <see cref="KestrelConfigurationLoader"/> for further endpoint configuration.</returns>
+
         public KestrelConfigurationLoader Configure(IConfiguration config, bool reloadOnChange)
         {
             var loader = new KestrelConfigurationLoader(this, config, reloadOnChange);