From fb3f4047d9f8dd9c45cfe7b3168d1b0c849a8413 Mon Sep 17 00:00:00 2001 From: John Luo Date: Fri, 11 Sep 2020 16:24:56 -0700 Subject: [PATCH 1/4] Fix chrome/selenium tests (#25330) * Revert "Disable failing/hanging tests due to Chrome/Selenium issue (#25323)" This reverts commit 332f1504128a63431cb67830703922cdeefee525. * Update Selenium to latest * Update API * Try specifying a version * Update Selenium to 4.0.0-beta5 * Disable browser log tests * Fix components e2e tests and disable blazor standalone template test --- eng/Versions.props | 6 +-- ...soft.AspNetCore.Components.E2ETests.csproj | 3 -- .../MultipleRootComponentsTest.cs | 2 +- .../ServerReconnectionTest.cs | 2 +- .../Tests/WebAssemblyAuthenticationTests.cs | 2 +- .../E2ETest/Tests/WebAssemblyLazyLoadTest.cs | 2 +- .../E2ETest/Tests/WebAssemblyLoggingTest.cs | 8 +-- .../testassets/TestServer/ServerStartup.cs | 2 +- .../BlazorTemplates.Tests.csproj | 4 -- .../BlazorWasmTemplateTest.cs | 2 +- src/Shared/E2ETesting/BrowserFixture.cs | 51 ++++++++++++++----- src/Shared/E2ETesting/selenium-config.json | 6 ++- 12 files changed, 54 insertions(+), 36 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index a873bd89668f..ecc9f0e6c4c0 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -270,10 +270,10 @@ 1.0.2 12.0.2 13.0.4 - 3.12.1 + 4.0.0-alpha05 17.17134.0 - 2.43.0 - 3.12.1 + 85.0.4183.8300 + 4.0.0-alpha05 1.4.0 4.0.0 2.0.593 diff --git a/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj b/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj index 10e34bd39256..266bd5d4fc7d 100644 --- a/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj +++ b/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj @@ -14,9 +14,6 @@ true false - - true - diff --git a/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs b/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs index 6fc25c2c4624..d1bc0ca7ec52 100644 --- a/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs +++ b/src/Components/test/E2ETest/ServerExecutionTests/MultipleRootComponentsTest.cs @@ -43,7 +43,7 @@ public override async Task InitializeAsync() } } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void DoesNotStartMultipleConnections() { Navigate("/multiple-components"); diff --git a/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs b/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs index 9b92d33aab63..aa8ed996e7ba 100644 --- a/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs +++ b/src/Components/test/E2ETest/ServerExecutionTests/ServerReconnectionTest.cs @@ -78,7 +78,7 @@ public void RendersContinueAfterReconnect() Browser.False(() => Browser.FindElement(selector).Text == currentValue); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void ErrorsStopTheRenderingProcess() { Browser.FindElement(By.Id("cause-error")).Click(); diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs b/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs index 52b25edefcb6..10a16061f892 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyAuthenticationTests.cs @@ -306,7 +306,7 @@ public void LoggedInUser_OnTheIdP_CanLogInSilently() ValidateLoggedIn(userName); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void CanNotRedirect_To_External_ReturnUrl() { Browser.Navigate().GoToUrl(new Uri(new Uri(Browser.Url), "/authentication/login?returnUrl=https%3A%2F%2Fwww.bing.com").AbsoluteUri); diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs index 8076ae61dc6b..317bae624ba0 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs @@ -111,7 +111,7 @@ public void CanLazyLoadAssemblyWithRoutes() Assert.True(renderedElement.Displayed); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void ThrowsErrorForUnavailableAssemblies() { // Navigate to a page with lazy loaded assemblies for the first time diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs index c6440eecd8c5..f659d2494249 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyLoggingTest.cs @@ -32,7 +32,7 @@ protected override void InitializeAsyncCore() Assert.Equal("none", errorUi.GetCssValue("display")); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsSimpleExceptionsUsingLogger() { Browser.FindElement(By.Id("throw-simple-exception")).Click(); @@ -44,7 +44,7 @@ public void LogsSimpleExceptionsUsingLogger() "at BasicTestApp.ErrorComponent.ThrowSimple"); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsInnerExceptionsUsingLogger() { Browser.FindElement(By.Id("throw-inner-exception")).Click(); @@ -57,7 +57,7 @@ public void LogsInnerExceptionsUsingLogger() "at BasicTestApp.ErrorComponent.ThrowInner"); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsAggregateExceptionsUsingLogger() { Browser.FindElement(By.Id("throw-aggregate-exception")).Click(); @@ -72,7 +72,7 @@ public void LogsAggregateExceptionsUsingLogger() "System.InvalidTimeZoneException: Aggregate exception 3"); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LogsUsingCustomLogger() { Browser.MountTestComponent(); diff --git a/src/Components/test/testassets/TestServer/ServerStartup.cs b/src/Components/test/testassets/TestServer/ServerStartup.cs index 83578c5dcb3a..bcd6e56e8848 100644 --- a/src/Components/test/testassets/TestServer/ServerStartup.cs +++ b/src/Components/test/testassets/TestServer/ServerStartup.cs @@ -19,7 +19,7 @@ public ServerStartup(IConfiguration configuration) public void ConfigureServices(IServiceCollection services) { services.AddMvc(); - services.AddServerSideBlazor(options => options.DetailedErrors = true); + services.AddServerSideBlazor(); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. diff --git a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj index 6a024814c24f..2da0e11666ef 100644 --- a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj +++ b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorTemplates.Tests.csproj @@ -10,10 +10,6 @@ true true - - - true - false true diff --git a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs index a1922ddfe908..3ee874e8dac6 100644 --- a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs +++ b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs @@ -40,7 +40,7 @@ public override Task InitializeAsync() return InitializeAsync(isolationContext: Guid.NewGuid().ToString()); } - [Fact] + [Fact(Skip = "Certificate issue: https://github.com/dotnet/aspnetcore/issues/25826")] public async Task BlazorWasmStandaloneTemplate_Works() { // Additional arguments are needed. See: https://github.com/dotnet/aspnetcore/issues/24278 diff --git a/src/Shared/E2ETesting/BrowserFixture.cs b/src/Shared/E2ETesting/BrowserFixture.cs index 557fbf37fc15..3e6fa36b6720 100644 --- a/src/Shared/E2ETesting/BrowserFixture.cs +++ b/src/Shared/E2ETesting/BrowserFixture.cs @@ -11,7 +11,10 @@ using System.Threading.Tasks; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Edge; +using OpenQA.Selenium.IE; using OpenQA.Selenium.Remote; +using OpenQA.Selenium.Safari; using Xunit; using Xunit.Abstractions; @@ -234,55 +237,75 @@ private string UserProfileDirectory(string context) name = $"{name} - {context}"; } - var capabilities = new DesiredCapabilities(); + DriverOptions options; + + switch (sauce.BrowserName.ToLower()) + { + case "chrome": + options = new ChromeOptions(); + break; + case "safari": + options = new SafariOptions(); + break; + case "internet explorer": + options = new InternetExplorerOptions(); + break; + case "microsoftedge": + options = new EdgeOptions(); + break; + default: + throw new InvalidOperationException($"Browser name {sauce.BrowserName} not recognized"); + } // Required config - capabilities.SetCapability("username", sauce.Username); - capabilities.SetCapability("accessKey", sauce.AccessKey); - capabilities.SetCapability("tunnelIdentifier", sauce.TunnelIdentifier); - capabilities.SetCapability("name", name); + options.AddAdditionalOption("username", sauce.Username); + options.AddAdditionalOption("accessKey", sauce.AccessKey); + options.AddAdditionalOption("tunnelIdentifier", sauce.TunnelIdentifier); + options.AddAdditionalOption("name", name); if (!string.IsNullOrEmpty(sauce.BrowserName)) { - capabilities.SetCapability("browserName", sauce.BrowserName); + options.AddAdditionalOption("browserName", sauce.BrowserName); } if (!string.IsNullOrEmpty(sauce.PlatformVersion)) { - capabilities.SetCapability("platformName", sauce.PlatformName); - capabilities.SetCapability("platformVersion", sauce.PlatformVersion); + options.PlatformName = sauce.PlatformName; + options.AddAdditionalOption("platformVersion", sauce.PlatformVersion); } else { // In some cases (like macOS), SauceLabs expects us to set "platform" instead of "platformName". - capabilities.SetCapability("platform", sauce.PlatformName); + options.AddAdditionalOption("platform", sauce.PlatformName); } if (!string.IsNullOrEmpty(sauce.BrowserVersion)) { - capabilities.SetCapability("browserVersion", sauce.BrowserVersion); + options.BrowserVersion = sauce.BrowserVersion; } if (!string.IsNullOrEmpty(sauce.DeviceName)) { - capabilities.SetCapability("deviceName", sauce.DeviceName); + options.AddAdditionalOption("deviceName", sauce.DeviceName); } if (!string.IsNullOrEmpty(sauce.DeviceOrientation)) { - capabilities.SetCapability("deviceOrientation", sauce.DeviceOrientation); + options.AddAdditionalOption("deviceOrientation", sauce.DeviceOrientation); } if (!string.IsNullOrEmpty(sauce.AppiumVersion)) { - capabilities.SetCapability("appiumVersion", sauce.AppiumVersion); + options.AddAdditionalOption("appiumVersion", sauce.AppiumVersion); } if (!string.IsNullOrEmpty(sauce.SeleniumVersion)) { - capabilities.SetCapability("seleniumVersion", sauce.SeleniumVersion); + options.AddAdditionalOption("seleniumVersion", sauce.SeleniumVersion); } + var capabilities = options.ToCapabilities(); + await SauceConnectServer.StartAsync(output); var attempt = 0; diff --git a/src/Shared/E2ETesting/selenium-config.json b/src/Shared/E2ETesting/selenium-config.json index 95c078e2c712..0682e48d3201 100644 --- a/src/Shared/E2ETesting/selenium-config.json +++ b/src/Shared/E2ETesting/selenium-config.json @@ -1,6 +1,8 @@ { "drivers": { - "chrome": {} + "chrome": { + "version" : "85.0.4183.87" + } }, - "ignoreExtraDrivers": true + "ignoreExtraDrivers": true } \ No newline at end of file From adcd8d63a91b86bdcccd15fb14abd7a016e4ce88 Mon Sep 17 00:00:00 2001 From: John Luo Date: Sat, 12 Sep 2020 18:12:49 -0700 Subject: [PATCH 2/4] Disable tests using browser log --- src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs index f4d06eeda4e1..d60d665623a7 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyICUShardingTest.cs @@ -100,7 +100,7 @@ public void LoadingApp_KannadaLanguage_Works() Assert.Equal("ಹಲೋ", localizedDisplay.Text); } - [Fact] + [Fact(Skip = "Browser logs cannot be retrieved: https://github.com/dotnet/aspnetcore/issues/25803")] public void LoadingApp_DynamicallySetLanguageThrows() { // Arrange From b3fd941469f0dc9ac5e7d6ac40d4ef1e82071d53 Mon Sep 17 00:00:00 2001 From: John Luo Date: Mon, 14 Sep 2020 14:05:15 -0700 Subject: [PATCH 3/4] Disable template test --- .../BlazorTemplates.Tests/BlazorWasmTemplateTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs index 3ee874e8dac6..fa4c091d5400 100644 --- a/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs +++ b/src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs @@ -178,7 +178,7 @@ public async Task BlazorWasmStandalonePwaTemplate_Works() } } - [Fact] + [Fact(Skip = "Certificate issue: https://github.com/dotnet/aspnetcore/issues/25826")] public async Task BlazorWasmHostedPwaTemplate_Works() { // Additional arguments are needed. See: https://github.com/dotnet/aspnetcore/issues/24278 From f1439048469a4e21cd743e7b00688b963d44741c Mon Sep 17 00:00:00 2001 From: Pranav K Date: Mon, 14 Sep 2020 15:51:41 -0700 Subject: [PATCH 4/4] Avoid using .NET formatted strings in tests * Annotate BasicTestApp suggesting that it needs the all globalization data * Culture specific formatting relies on the ICU data carried by the OS. This causes issues in our tests if WebAssembly carries a different set than the OS. Instead updating these tests to use hardcoded strings. * Additionally fixing an issue where some projects in the solution were using tasks from the .dotnet SDK rather than the local copy of the SDK. This was causing issues building locally. --- eng/targets/CSharp.Common.props | 9 ++++++-- .../Tests/WebAssemblyICUShardingTest.cs | 22 +++++++++---------- .../Tests/WebAssemblyLocalizationTest.cs | 1 - .../BasicTestApp/BasicTestApp.csproj | 3 +++ 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/eng/targets/CSharp.Common.props b/eng/targets/CSharp.Common.props index ff77d1b86aab..097769897b73 100644 --- a/eng/targets/CSharp.Common.props +++ b/eng/targets/CSharp.Common.props @@ -12,7 +12,12 @@ - + + <_ReferenceLocalRazorSDK + Condition="'$(UsingMicrosoftNETSdkWeb)' == 'true' OR '$(UsingMicrosoftNETSdkBlazorWebAssembly)' == 'true' OR '$(RazorSdkCurrentVersionProps)' != '' OR '$(_RazorSdkImportsMicrosoftNetSdk)' == 'true'">true + + + false + + + true