diff --git a/src/DefaultBuilder/src/WebApplicationBuilder.cs b/src/DefaultBuilder/src/WebApplicationBuilder.cs
index 19e8c0661c84..870b2f51f679 100644
--- a/src/DefaultBuilder/src/WebApplicationBuilder.cs
+++ b/src/DefaultBuilder/src/WebApplicationBuilder.cs
@@ -22,6 +22,7 @@ public sealed class WebApplicationBuilder
         private readonly BootstrapHostBuilder _bootstrapHostBuilder;
         private readonly WebApplicationServiceCollection _services = new();
         private readonly List<KeyValuePair<string, string>> _hostConfigurationValues;
+        private readonly ConfigurationManager _hostConfigurationManager = new();
 
         private WebApplication? _builtApplication;
 
@@ -76,6 +77,8 @@ internal WebApplicationBuilder(WebApplicationOptions options, Action<IHostBuilde
             });
 
             Configuration = new();
+            // This is chained as the first configuration source in Configuration so host config can be added later without overriding app config.
+            Configuration.AddConfiguration(_hostConfigurationManager, shouldDisposeConfiguration: true);
 
             // Collect the hosted services separately since we want those to run after the user's hosted services
             _services.TrackHostedServices = true;
@@ -194,6 +197,9 @@ public WebApplication Build()
                 // to the new one. This allows code that has references to the service collection to still function.
                 _services.InnerCollection = services;
 
+                // Keep any configuration sources added before the TrackingChainedConfigurationSource (namely host configuration from _hostConfigurationValues)
+                // from overriding config values set via Configuration by inserting them at beginning using _hostConfigurationValues.
+                var beforeChainedConfig = true;
                 var hostBuilderProviders = ((IConfigurationRoot)context.Configuration).Providers;
 
                 if (!hostBuilderProviders.Contains(chainedConfigSource.BuiltProvider))
@@ -201,15 +207,22 @@ public WebApplication Build()
                     // Something removed the _hostBuilder's TrackingChainedConfigurationSource pointing back to the ConfigurationManager.
                     // This is likely a test using WebApplicationFactory. Replicate the effect by clearing the ConfingurationManager sources.
                     ((IConfigurationBuilder)Configuration).Sources.Clear();
+                    beforeChainedConfig = false;
                 }
 
-                // Make builder.Configuration match the final configuration. To do that, we add the additional
-                // providers in the inner _hostBuilders's Configuration to the ConfigurationManager.
+                // Make the ConfigurationManager match the final _hostBuilder's configuration. To do that, we add the additional providers
+                // to the inner _hostBuilders's configuration to the ConfigurationManager. We wrap the existing provider in a
+                // configuration source to avoid rebuilding or reloading the already added configuration sources.
                 foreach (var provider in hostBuilderProviders)
                 {
-                    if (!ReferenceEquals(provider, chainedConfigSource.BuiltProvider))
+                    if (ReferenceEquals(provider, chainedConfigSource.BuiltProvider))
                     {
-                        ((IConfigurationBuilder)Configuration).Add(new ConfigurationProviderSource(provider));
+                        beforeChainedConfig = false;
+                    }
+                    else
+                    {
+                        IConfigurationBuilder configBuilder = beforeChainedConfig ? _hostConfigurationManager : Configuration;
+                        configBuilder.Add(new ConfigurationProviderSource(provider));
                     }
                 }
             });
diff --git a/src/DefaultBuilder/test/Microsoft.AspNetCore.Tests/WebApplicationTests.cs b/src/DefaultBuilder/test/Microsoft.AspNetCore.Tests/WebApplicationTests.cs
index 8a29e17b21ba..8f724498dde7 100644
--- a/src/DefaultBuilder/test/Microsoft.AspNetCore.Tests/WebApplicationTests.cs
+++ b/src/DefaultBuilder/test/Microsoft.AspNetCore.Tests/WebApplicationTests.cs
@@ -129,6 +129,33 @@ public async Task WebApplicationRunUrls_OverridesIServerAddressesFeature()
             await runTask;
         }
 
+        [Fact]
+        public async Task WebApplicationWebHostUseUrls_OverridesDefaultHostingConfiguration()
+        {
+            var builder = new WebApplicationBuilder(new(), bootstrapBuilder =>
+            {
+                bootstrapBuilder.ConfigureHostConfiguration(configBuilder =>
+                {
+                    configBuilder.AddInMemoryCollection(new Dictionary<string, string>
+                    {
+                        [WebHostDefaults.ServerUrlsKey] = "http://localhost:5000",
+                    });
+                });
+            });
+
+            builder.WebHost.UseUrls("http://localhost:5001");
+
+            var urls = new List<string>();
+            var server = new MockAddressesServer(urls);
+            builder.Services.AddSingleton<IServer>(server);
+            await using var app = builder.Build();
+
+            await app.StartAsync();
+
+            var url = Assert.Single(urls);
+            Assert.Equal("http://localhost:5001", url);
+        }
+
         [Fact]
         public async Task WebApplicationUrls_ThrowsInvalidOperationExceptionIfThereIsNoIServerAddressesFeature()
         {