Add support delegating requests in HttpSysServer (#24857)

* Add new ctors for RequestQueue and UrlGroup

* Add DelegateRequest pinvokes

* Add Request Transfer Feature

* Fix accessibility of feature

* Test cleanup

* Update ref assembly

* hack: Make HttpSysServer packable

* Cleanup based on PR feedback

* Avoid sending headers after transfer

* Fix ref assembly

* Fix rebase conflict

* Switch to DelegateRequestEx

* Add feature detection

* Delete ref folder

* Add server feature

* s/RequestQueueWrapper/DelegationRule

* Fix UrlGroup was null issue

* Add light-up for ServerDelegationPropertyFeature

* Revert changes to sample

* Revert changes to sample take 2

* PR feedback

* s/Transfered/Transferred

* DelegateAfterRequestBodyReadShouldThrow

* Make DelegationRule disposable

* More license headers

* Incomplete XML doc

* PR feedback

* Fix broken test

* PR feedback

* Fixup test

* s/Transfer/Delegate

* s/transfer/delegate

* PR feedback

Author: shirhatti

src/Servers/HttpSys/src/AsyncAcceptContext.cs

@@ -127,7 +127,9 @@ internal uint QueueBeginGetContext()
                 statusCode = HttpApi.HttpReceiveHttpRequest(
                     Server.RequestQueue.Handle,
                     _nativeRequestContext.RequestId,
-                    (uint)HttpApiTypes.HTTP_FLAGS.HTTP_RECEIVE_REQUEST_FLAG_COPY_BODY,
+                    // Small perf impact by not using HTTP_RECEIVE_REQUEST_FLAG_COPY_BODY
+                    // if the request sends header+body in a single TCP packet 
+                    (uint)HttpApiTypes.HTTP_FLAGS.NONE,
                     _nativeRequestContext.NativeRequest,
                     _nativeRequestContext.Size,
                     &bytesTransferred,

src/Servers/HttpSys/src/DelegationRule.cs

@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.AspNetCore.Server.HttpSys
+{
+    /// <summary>
+    /// Rule that maintains a handle to the Request Queue and UrlPrefix to
+    /// delegate to.
+    /// </summary>
+    public class DelegationRule : IDisposable
+    {
+        private readonly ILogger _logger;
+        /// <summary>
+        /// The name of the Http.Sys request queue
+        /// </summary>
+        public string QueueName { get; }
+        /// <summary>
+        /// The URL of the Http.Sys Url Prefix
+        /// </summary>
+        public string UrlPrefix { get; }
+        internal RequestQueue Queue { get; }
+
+        internal DelegationRule(string queueName, string urlPrefix, ILogger logger)
+        {
+            _logger = logger;
+            QueueName = queueName;
+            UrlPrefix = urlPrefix;
+            Queue = new RequestQueue(queueName, UrlPrefix, _logger, receiver: true);
+        }
+
+        public void Dispose()
+        {
+            Queue.UrlGroup?.Dispose();
+            Queue?.Dispose();
+        }
+    }
+}

src/Servers/HttpSys/src/FeatureContext.cs

@@ -37,7 +37,8 @@ internal class FeatureContext :
         IHttpBodyControlFeature,
         IHttpSysRequestInfoFeature,
         IHttpResponseTrailersFeature,
-        IHttpResetFeature
+        IHttpResetFeature,
+        IHttpSysRequestDelegationFeature
     {
         private RequestContext _requestContext;
         private IFeatureCollection _features;
@@ -591,6 +592,8 @@ IHeaderDictionary IHttpResponseTrailersFeature.Trailers
             set => _responseTrailers = value;
         }
 
+        public bool CanDelegate => Request.CanDelegate;
+
         internal async Task OnResponseStart()
         {
             if (_responseStarted)
@@ -711,5 +714,11 @@ private async Task NotifyOnCompletedAsync()
                 await actionPair.Item1(actionPair.Item2);
             }
         }
+
+        public void DelegateRequest(DelegationRule destination)
+        {
+            _requestContext.Delegate(destination);
+            _responseStarted = true;
+        }
     }
 }

src/Servers/HttpSys/src/IHttpSysRequestDelegationFeature.cs

@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.AspNetCore.Server.HttpSys
+{
+    public interface IHttpSysRequestDelegationFeature
+    {
+        /// <summary>
+        /// Indicates if the server can delegate this request to another HttpSys request queue.
+        /// </summary>
+        bool CanDelegate { get; }
+
+        /// <summary>
+        /// Attempt to delegate the request to another Http.Sys request queue. The request body
+        /// must not be read nor the response started before this is invoked. Check <see cref="CanDelegate"/>
+        /// before invoking.
+        /// </summary>
+        void DelegateRequest(DelegationRule destination);
+    }
+}

src/Servers/HttpSys/src/IServerDelegationFeature.cs

@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.AspNetCore.Server.HttpSys
+{
+    public interface IServerDelegationFeature
+    {
+        /// <summary>
+        /// Create a delegation rule on request queue owned by the server.
+        /// </summary>
+        /// <returns>
+        /// Creates a <see cref="DelegationRule"/> that can used to delegate individual requests.
+        /// </returns>
+        DelegationRule CreateDelegationRule(string queueName, string urlPrefix);
+    }
+}

src/Servers/HttpSys/src/MessagePump.cs

@@ -55,6 +55,12 @@ public MessagePump(IOptions<HttpSysOptions> options, ILoggerFactory loggerFactor
             _serverAddresses = new ServerAddressesFeature();
             Features.Set<IServerAddressesFeature>(_serverAddresses);
 
+            if (HttpApi.IsFeatureSupported(HttpApiTypes.HTTP_FEATURE_ID.HttpFeatureDelegateEx))
+            {
+                var delegationProperty = new ServerDelegationPropertyFeature(Listener.RequestQueue, _logger);
+                Features.Set<IServerDelegationFeature>(delegationProperty);
+            }
+
             _maxAccepts = _options.MaxAccepts;
         }
 

src/Servers/HttpSys/src/NativeInterop/HttpApi.cs

@@ -45,6 +45,9 @@ internal static unsafe class HttpApi
         [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, SetLastError = true)]
         internal static extern uint HttpCreateUrlGroup(ulong serverSessionId, ulong* urlGroupId, uint reserved);
 
+        [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, SetLastError = true)]
+        internal static extern uint HttpFindUrlGroupId(string pFullyQualifiedUrl, SafeHandle requestQueueHandle, ulong* urlGroupId);
+
         [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, SetLastError = true)]
         internal static extern uint HttpAddUrlToUrlGroup(ulong urlGroupId, string pFullyQualifiedUrl, ulong context, uint pReserved);
 
@@ -70,6 +73,13 @@ internal static extern unsafe uint HttpCreateRequestQueue(HTTPAPI_VERSION versio
         [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, SetLastError = true)]
         internal static extern unsafe uint HttpCloseRequestQueue(IntPtr pReqQueueHandle);
 
+        [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, SetLastError = true)]
+        internal static extern bool HttpIsFeatureSupported(HTTP_FEATURE_ID feature);
+
+        [DllImport(HTTPAPI, ExactSpelling = true, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, SetLastError = true)]
+        internal static extern unsafe uint HttpDelegateRequestEx(SafeHandle pReqQueueHandle, SafeHandle pDelegateQueueHandle, ulong requestId,
+            ulong delegateUrlGroupId, ulong propertyInfoSetSize, HTTP_DELEGATE_REQUEST_PROPERTY_INFO* pRequestPropertyBuffer);
+
         internal delegate uint HttpSetRequestPropertyInvoker(SafeHandle requestQueueHandle, ulong requestId, HTTP_REQUEST_PROPERTY propertyId, void* input, uint inputSize, IntPtr overlapped);
 
         private static HTTPAPI_VERSION version;
@@ -145,5 +155,16 @@ internal static bool Supported
                 return supported;
             }
         }
+
+        internal static bool IsFeatureSupported(HTTP_FEATURE_ID feature)
+        {
+            try
+            {
+                return HttpIsFeatureSupported(feature);
+            }
+            catch (EntryPointNotFoundException) { }
+
+            return false;
+        }
     }
 }

src/Servers/HttpSys/src/NativeInterop/RequestQueue.cs

@@ -16,22 +16,44 @@ internal class RequestQueue
             Marshal.SizeOf<HttpApiTypes.HTTP_BINDING_INFO>();
 
         private readonly RequestQueueMode _mode;
-        private readonly UrlGroup _urlGroup;
         private readonly ILogger _logger;
         private bool _disposed;
 
+        internal RequestQueue(string requestQueueName, string urlPrefix, ILogger logger, bool receiver)
+            : this(urlGroup: null, requestQueueName, RequestQueueMode.Attach, logger, receiver)
+        {
+            try
+            {
+                UrlGroup = new UrlGroup(this, UrlPrefix.Create(urlPrefix));
+            }
+            catch
+            {
+                Dispose();
+                throw;
+            }
+        }
+
         internal RequestQueue(UrlGroup urlGroup, string requestQueueName, RequestQueueMode mode, ILogger logger)
+            : this(urlGroup, requestQueueName, mode, logger, false)
+        { }
+
+        private RequestQueue(UrlGroup urlGroup, string requestQueueName, RequestQueueMode mode, ILogger logger, bool receiver)
         {
             _mode = mode;
-            _urlGroup = urlGroup;
+            UrlGroup = urlGroup;
             _logger = logger;
 
             var flags = HttpApiTypes.HTTP_CREATE_REQUEST_QUEUE_FLAG.None;
             Created = true;
+
             if (_mode == RequestQueueMode.Attach)
             {
                 flags = HttpApiTypes.HTTP_CREATE_REQUEST_QUEUE_FLAG.OpenExisting;
                 Created = false;
+                if (receiver)
+                {
+                    flags |= HttpApiTypes.HTTP_CREATE_REQUEST_QUEUE_FLAG.Delegation;
+                }
             }
 
             var statusCode = HttpApi.HttpCreateRequestQueue(
@@ -54,7 +76,7 @@ internal RequestQueue(UrlGroup urlGroup, string requestQueueName, RequestQueueMo
                         out requestQueueHandle);
             }
 
-            if (flags == HttpApiTypes.HTTP_CREATE_REQUEST_QUEUE_FLAG.OpenExisting && statusCode == UnsafeNclNativeMethods.ErrorCodes.ERROR_FILE_NOT_FOUND)
+            if (flags.HasFlag(HttpApiTypes.HTTP_CREATE_REQUEST_QUEUE_FLAG.OpenExisting) && statusCode == UnsafeNclNativeMethods.ErrorCodes.ERROR_FILE_NOT_FOUND)
             {
                 throw new HttpSysException((int)statusCode, $"Failed to attach to the given request queue '{requestQueueName}', the queue could not be found.");
             }
@@ -95,6 +117,8 @@ internal RequestQueue(UrlGroup urlGroup, string requestQueueName, RequestQueueMo
         internal SafeHandle Handle { get; }
         internal ThreadPoolBoundHandle BoundHandle { get; }
 
+        internal UrlGroup UrlGroup { get; }
+
         internal unsafe void AttachToUrlGroup()
         {
             Debug.Assert(Created);
@@ -108,7 +132,7 @@ internal unsafe void AttachToUrlGroup()
 
             var infoptr = new IntPtr(&info);
 
-            _urlGroup.SetProperty(HttpApiTypes.HTTP_SERVER_PROPERTY.HttpServerBindingProperty,
+            UrlGroup.SetProperty(HttpApiTypes.HTTP_SERVER_PROPERTY.HttpServerBindingProperty,
                 infoptr, (uint)BindingInfoSize);
         }
 
@@ -128,7 +152,7 @@ internal unsafe void DetachFromUrlGroup()
 
             var infoptr = new IntPtr(&info);
 
-            _urlGroup.SetProperty(HttpApiTypes.HTTP_SERVER_PROPERTY.HttpServerBindingProperty,
+            UrlGroup.SetProperty(HttpApiTypes.HTTP_SERVER_PROPERTY.HttpServerBindingProperty,
                 infoptr, (uint)BindingInfoSize, throwOnError: false);
         }
 

src/Servers/HttpSys/src/NativeInterop/UrlGroup.cs

@@ -13,6 +13,8 @@ internal class UrlGroup : IDisposable
     {
         private static readonly int QosInfoSize =
             Marshal.SizeOf<HttpApiTypes.HTTP_QOS_SETTING_INFO>();
+        private static readonly int RequestPropertyInfoSize =
+            Marshal.SizeOf<HttpApiTypes.HTTP_BINDING_INFO>();
 
         private ServerSession _serverSession;
         private ILogger _logger;
@@ -36,6 +38,21 @@ internal unsafe UrlGroup(ServerSession serverSession, ILogger logger)
             Id = urlGroupId;
         }
 
+        internal unsafe UrlGroup(RequestQueue requestQueue, UrlPrefix url)
+        {
+            ulong urlGroupId = 0;
+            var statusCode = HttpApi.HttpFindUrlGroupId(
+                url.FullPrefix, requestQueue.Handle, &urlGroupId);
+
+            if (statusCode != UnsafeNclNativeMethods.ErrorCodes.ERROR_SUCCESS)
+            {
+                throw new HttpSysException((int)statusCode);
+            }
+
+            Debug.Assert(urlGroupId != 0, "Invalid id returned by HttpCreateUrlGroup");
+            Id = urlGroupId;
+        }
+
         internal ulong Id { get; private set; }
 
         internal unsafe void SetMaxConnections(long maxConnections)
@@ -51,6 +68,15 @@ internal unsafe void SetMaxConnections(long maxConnections)
             SetProperty(HttpApiTypes.HTTP_SERVER_PROPERTY.HttpServerQosProperty, new IntPtr(&qosSettings), (uint)QosInfoSize);
         }
 
+        internal unsafe void SetDelegationProperty(RequestQueue destination)
+        {
+            var propertyInfo = new HttpApiTypes.HTTP_BINDING_INFO();
+            propertyInfo.Flags = HttpApiTypes.HTTP_FLAGS.HTTP_PROPERTY_FLAG_PRESENT;
+            propertyInfo.RequestQueueHandle = destination.Handle.DangerousGetHandle();
+
+            SetProperty(HttpApiTypes.HTTP_SERVER_PROPERTY.HttpServerDelegationProperty, new IntPtr(&propertyInfo), (uint)RequestPropertyInfoSize);
+        }
+
         internal void SetProperty(HttpApiTypes.HTTP_SERVER_PROPERTY property, IntPtr info, uint infosize, bool throwOnError = true)
         {
             Debug.Assert(info != IntPtr.Zero, "SetUrlGroupProperty called with invalid pointer");

src/Servers/HttpSys/src/RequestProcessing/Request.cs

@@ -61,43 +61,40 @@ internal Request(RequestContext requestContext, NativeRequestContext nativeReque
             var rawUrlInBytes = _nativeRequestContext.GetRawUrlInBytes();
             var originalPath = RequestUriBuilder.DecodeAndUnescapePath(rawUrlInBytes);
 
+            PathBase = string.Empty;
+            Path = originalPath;
+
             // 'OPTIONS * HTTP/1.1'
             if (KnownMethod == HttpApiTypes.HTTP_VERB.HttpVerbOPTIONS && string.Equals(RawUrl, "*", StringComparison.Ordinal))
             {
                 PathBase = string.Empty;
                 Path = string.Empty;
             }
-            else if (requestContext.Server.RequestQueue.Created)
+            else
             {
                 var prefix = requestContext.Server.Options.UrlPrefixes.GetPrefix((int)nativeRequestContext.UrlContext);
-
-                if (originalPath.Length == prefix.PathWithoutTrailingSlash.Length)
-                {
-                    // They matched exactly except for the trailing slash.
-                    PathBase = originalPath;
-                    Path = string.Empty;
-                }
-                else
+                // Prefix may be null if the requested has been transfered to our queue
+                if (!(prefix is null))
                 {
-                    // url: /base/path, prefix: /base/, base: /base, path: /path
-                    // url: /, prefix: /, base: , path: /
-                    PathBase = originalPath.Substring(0, prefix.PathWithoutTrailingSlash.Length); // Preserve the user input casing
-                    Path = originalPath.Substring(prefix.PathWithoutTrailingSlash.Length);
+                    if (originalPath.Length == prefix.PathWithoutTrailingSlash.Length)
+                    {
+                        // They matched exactly except for the trailing slash.
+                        PathBase = originalPath;
+                        Path = string.Empty;
+                    }
+                    else
+                    {
+                        // url: /base/path, prefix: /base/, base: /base, path: /path
+                        // url: /, prefix: /, base: , path: /
+                        PathBase = originalPath.Substring(0, prefix.PathWithoutTrailingSlash.Length); // Preserve the user input casing
+                        Path = originalPath.Substring(prefix.PathWithoutTrailingSlash.Length);
+                    }
                 }
-            }
-            else
-            {
-                // When attaching to an existing queue, the UrlContext hint may not match our configuration. Search manualy.
-                if (requestContext.Server.Options.UrlPrefixes.TryMatchLongestPrefix(IsHttps, cookedUrl.GetHost(), originalPath, out var pathBase, out var path))
+                 else if (requestContext.Server.Options.UrlPrefixes.TryMatchLongestPrefix(IsHttps, cookedUrl.GetHost(), originalPath, out var pathBase, out var path))
                 {
                     PathBase = pathBase;
                     Path = path;
                 }
-                else
-                {
-                    PathBase = string.Empty;
-                    Path = originalPath;
-                }
             }
 
             ProtocolVersion = _nativeRequestContext.GetVersion();
@@ -350,6 +347,8 @@ public X509Certificate2 ClientCertificate
             }
         }
 
+        public bool CanDelegate => !(HasRequestBodyStarted || RequestContext.Response.HasStarted);
+
         // Populates the client certificate.  The result may be null if there is no client cert.
         // TODO: Does it make sense for this to be invoked multiple times (e.g. renegotiate)? Client and server code appear to
         // enable this, but it's unclear what Http.Sys would do.