From a575ec81053ac533449901e7f53bcd3283bca730 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Mon, 26 Aug 2019 11:58:26 -0400 Subject: [PATCH 1/7] [ci] Move Windows build and test to VS2019 We've been encountering a handful of issues recently with the Windows machine pool we're using for building and testing. There have been quite a few failures caused by files being locked, or dependencies being out of date or missing. Moving to the VSEng VS2019 Windows pool will hopefully resolve some of these issues. Unfortunately the size of our build dependencies presents an issue when attempting to use the Microsoft hosted Windows VS2019 agents, as our Win job ultimately runs out of space when executing on those VMS. Using the VSEng VS2019 pool will allow us to migrate to VS2019 while avoiding disk space limitations. --- build-tools/automation/azure-pipelines.yaml | 19 +++++++++++++------ .../mac/setup-test-environment.yaml | 4 ++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index 493695beaaa..c521b66c337 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -21,7 +21,9 @@ variables: InstallerArtifactName: installers TestAssembliesArtifactName: test-assemblies NUnitConsoleVersion: 3.9.0 + DotNetCoreVersion: 2.1.701 HostedWinVS2019: Hosted Windows 2019 with VS2019 + VSEngWinVS2019: VSEng-MicroBuildVS2019 # Stage and Job "display names" are shortened because they are combined to form the name of the corresponding GitHub check. stages: @@ -54,9 +56,9 @@ stages: submodules: recursive - task: UseDotNet@2 - displayName: install .NET Core 2.1.701 + displayName: install .NET Core $(DotNetCoreVersion) inputs: - version: 2.1.701 + version: $(DotNetCoreVersion) # Update Mono in a separate step since xaprepare uses it as well and it will crash when Mono it runs with is updated # The 'prepare' step creates the bundle @@ -103,9 +105,9 @@ stages: submodules: recursive - task: UseDotNet@2 - displayName: install .NET Core 2.1.701 + displayName: install .NET Core $(DotNetCoreVersion) inputs: - version: 2.1.701 + version: $(DotNetCoreVersion) - task: DownloadPipelineArtifact@1 inputs: @@ -198,7 +200,7 @@ stages: - job: queue_vsix_signing displayName: Queue Vsix Signing dependsOn: mac_build_create_installers - pool: $(XA.Build.Win.Pool) + pool: $(VSEngWinVS2019) condition: and(eq(dependencies.mac_build_create_installers.result, 'Succeeded'), eq(variables['XA.Commercial.Build'], 'true'), ne(variables['Build.Reason'], 'PullRequest')) timeoutInMinutes: 45 cancelTimeoutInMinutes: 1 @@ -226,7 +228,7 @@ stages: jobs: - job: win_build_test displayName: Build and Test - pool: $(XA.Build.Win.Pool) + pool: $(VSEngWinVS2019) timeoutInMinutes: 360 cancelTimeoutInMinutes: 5 workspace: @@ -240,6 +242,11 @@ stages: - checkout: self submodules: recursive + - task: UseDotNet@2 + displayName: install .NET Core $(DotNetCoreVersion) + inputs: + version: $(DotNetCoreVersion) + - task: DownloadPipelineArtifact@1 inputs: artifactName: $(BundleArtifactName) diff --git a/build-tools/automation/yaml-templates/mac/setup-test-environment.yaml b/build-tools/automation/yaml-templates/mac/setup-test-environment.yaml index 2b1a31ee08d..339ff9324fa 100644 --- a/build-tools/automation/yaml-templates/mac/setup-test-environment.yaml +++ b/build-tools/automation/yaml-templates/mac/setup-test-environment.yaml @@ -18,9 +18,9 @@ steps: displayName: install xaprepare dependencies - task: UseDotNet@2 - displayName: install .NET Core 2.1.701 + displayName: install .NET Core $(DotNetCoreVersion) inputs: - version: 2.1.701 + version: $(DotNetCoreVersion) # Restore solutions for Xamarin.Android.Tools.sln, Xamarin.Android.sln, and Xamarin.Android-Tests.sln - task: NuGetCommand@2 From ed079dc367ae32e7e86be0d98421938ec684eb3f Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Wed, 28 Aug 2019 16:15:03 -0400 Subject: [PATCH 2/7] [java-runtime] Always pass `$(JAVA_HOME) to `dx` As of commit 8cdda115, `$(JavaSdkDirectory)` will have a default value of the path to the Corretto JDK we install on both Windows and macOS. We should try to use this JDK path for all Android/Java tool invocations which require `$(JAVA_HOME)` to be set. --- src/java-runtime/java-runtime.targets | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/java-runtime/java-runtime.targets b/src/java-runtime/java-runtime.targets index c756e383362..756ba7d3224 100644 --- a/src/java-runtime/java-runtime.targets +++ b/src/java-runtime/java-runtime.targets @@ -56,12 +56,9 @@ AfterTargets="Build" Inputs="@(_RuntimeOutput->'%(Identity)')" Outputs="@(_RuntimeOutput->'%(OutputDex)')"> - - <_JavaHome Condition=" '$(JAVA_HOME)' == '' And '$(JavaSdkDirectory)' != '' ">JAVA_HOME=$(JavaSdkDirectory) - From 8d81905e6d08199ee758df159a50b578705450b2 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 29 Aug 2019 09:18:55 -0400 Subject: [PATCH 3/7] [tests] Fix dotnet lookup path on Windows Versions 2+ of .NET Core are installed into `C:\Program Files\dotnet` rather than `C:\Program Files (x86)\Microsoft Visual Studio\$(Version)`. --- .../Xamarin.ProjectTools/Common/Builder.cs | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index d6a3cfcced2..479d8f9e02b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -119,24 +119,25 @@ public string FrameworkLibDirectory { public string MicrosoftNetSdkDirectory { get { - string path; - if (IsUnix) { - path = FindLatestDotNetSdk (frameworkSDKRoot); - if (!string.IsNullOrEmpty (path)) - return path; - - return string.Empty; - } - var visualStudioDirectory = TestEnvironment.GetVisualStudioDirectory (); - if (!string.IsNullOrEmpty (visualStudioDirectory)) { - path = Path.Combine (visualStudioDirectory, "MSBuild", "Sdks", "Microsoft.NET.Sdk"); + string dotnetSearchPath = IsUnix ? frameworkSDKRoot + : Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.ProgramFiles), "dotnet"); + + string path = FindLatestDotNetSdk (dotnetSearchPath); + if (!string.IsNullOrEmpty (path)) + return path; + + if (!IsUnix) { + var visualStudioDirectory = TestEnvironment.GetVisualStudioDirectory (); + if (!string.IsNullOrEmpty (visualStudioDirectory)) { + path = Path.Combine (visualStudioDirectory, "MSBuild", "Sdks", "Microsoft.NET.Sdk"); + if (File.Exists (Path.Combine (path, "Sdk", "Sdk.props"))) + return path; + } + var x86 = Environment.GetFolderPath (Environment.SpecialFolder.ProgramFilesX86); + path = Path.Combine (x86, "MSBuild", "Sdks", "Microsoft.NET.Sdk"); if (File.Exists (Path.Combine (path, "Sdk", "Sdk.props"))) return path; } - var x86 = Environment.GetFolderPath (Environment.SpecialFolder.ProgramFilesX86); - path = Path.Combine (x86, "MSBuild", "Sdks", "Microsoft.NET.Sdk"); - if (File.Exists (Path.Combine (path, "Sdk", "Sdk.props"))) - return path; return string.Empty; } } @@ -505,7 +506,8 @@ string FindLatestDotNetSdk (string dotNetPath) Version latest = new Version (0,0); string Sdk = null; foreach (var dir in Directory.EnumerateDirectories (dotNetPath)) { - var version = GetVersionFromDirectory (dir); + // Preview versions such as `3.0.100-preview9-013775` will be ignored. + Version.TryParse (Path.GetFileName (dir), out Version version); var sdksDir = Path.Combine (dir, "Sdks"); if (!Directory.Exists (sdksDir)) sdksDir = Path.Combine (dir, "bin", "Sdks"); @@ -521,12 +523,6 @@ string FindLatestDotNetSdk (string dotNetPath) return null; } - static Version GetVersionFromDirectory (string dir) - { - Version v; - Version.TryParse (Path.GetFileName (dir), out v); - return v; - } } } From 5b893e47383bc15a0b97001694beb5e79e9c8f7b Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 29 Aug 2019 09:39:54 -0400 Subject: [PATCH 4/7] [ci] Add process monitor/kill task --- build-tools/automation/azure-pipelines.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index c521b66c337..f1eed92f4be 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -207,6 +207,8 @@ stages: workspace: clean: all steps: + - task: xamops.azdevex.lingering-process-task.lingering-process-task@1 + - task: JenkinsQueueJob@2 displayName: xamarin vsix codesign - run jenkins job inputs: @@ -237,11 +239,12 @@ stages: AndroidTargetAbiArgs: >- /p:AndroidSupportedTargetJitAbis=armeabi-v7a:arm64-v8a:x86:x86_64 /p:AndroidSupportedTargetAotAbis=armeabi-v7a:arm64:x86:x86_64:win-armeabi-v7a:win-arm64:win-x86:win-x86_64 - JAVA_HOME: '%HOMEDRIVE%%HOMEPATH%\android-toolchain\jdk' steps: - checkout: self submodules: recursive + - task: xamops.azdevex.lingering-process-task.lingering-process-task@1 + - task: UseDotNet@2 displayName: install .NET Core $(DotNetCoreVersion) inputs: From 242eecc29974b04207cbe4f2f670776e9223ba57 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 29 Aug 2019 12:18:20 -0400 Subject: [PATCH 5/7] [tests] Log potential .NET core paths --- .../Tests/Xamarin.ProjectTools/Common/Builder.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index 479d8f9e02b..daf7273c6d8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -506,6 +506,7 @@ string FindLatestDotNetSdk (string dotNetPath) Version latest = new Version (0,0); string Sdk = null; foreach (var dir in Directory.EnumerateDirectories (dotNetPath)) { + Console.WriteLine ($"Found potential .NET dir: {dir}"); // Preview versions such as `3.0.100-preview9-013775` will be ignored. Version.TryParse (Path.GetFileName (dir), out Version version); var sdksDir = Path.Combine (dir, "Sdks"); From d01f844f0b06c57c3c261c3c523cbcae59b0eb6c Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 29 Aug 2019 18:27:08 -0400 Subject: [PATCH 6/7] [tests] Actually fix .NET Core lookup path on Win --- .../Tests/Xamarin.ProjectTools/Common/Builder.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index daf7273c6d8..5ba9d4322cc 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -120,7 +120,7 @@ public string FrameworkLibDirectory { public string MicrosoftNetSdkDirectory { get { string dotnetSearchPath = IsUnix ? frameworkSDKRoot - : Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.ProgramFiles), "dotnet"); + : Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.ProgramFiles), "dotnet", "sdk"); string path = FindLatestDotNetSdk (dotnetSearchPath); if (!string.IsNullOrEmpty (path)) @@ -506,7 +506,6 @@ string FindLatestDotNetSdk (string dotNetPath) Version latest = new Version (0,0); string Sdk = null; foreach (var dir in Directory.EnumerateDirectories (dotNetPath)) { - Console.WriteLine ($"Found potential .NET dir: {dir}"); // Preview versions such as `3.0.100-preview9-013775` will be ignored. Version.TryParse (Path.GetFileName (dir), out Version version); var sdksDir = Path.Combine (dir, "Sdks"); From 69fa843f3db7d6424bd4d3230903b5165ad36637 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Fri, 30 Aug 2019 12:26:46 -0400 Subject: [PATCH 7/7] [tests] Remove unnecessary .NET SDK lookup --- .../PackagingTest.cs | 2 - .../Xamarin.ProjectTools/Common/Builder.cs | 74 ------------------- 2 files changed, 76 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 4ece2ab17ed..43cb26f0d74 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -609,8 +609,6 @@ protected override void OnCreate (Bundle bundle) }; var path = Path.Combine ("temp", TestContext.CurrentContext.Test.Name); using (var builder = CreateDllBuilder (Path.Combine (path, netStandardProject.ProjectName), cleanupOnDispose: false)) { - if (!Directory.Exists (builder.MicrosoftNetSdkDirectory)) - Assert.Fail ($"Microsoft.NET.Sdk not found: {builder.MicrosoftNetSdkDirectory}"); using (var ab = CreateApkBuilder (Path.Combine (path, app.ProjectName), cleanupOnDispose: false)) { builder.RequiresMSBuild = ab.RequiresMSBuild = true; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index 5ba9d4322cc..95df420b844 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -17,7 +17,6 @@ public class Builder : IDisposable { const string SigSegvError = "Got a SIGSEGV while executing native code"; const string ConsoleLoggerError = "[ERROR] FATAL UNHANDLED EXCEPTION: System.ArgumentException: is negative"; - static string frameworkSDKRoot = null; string root; string buildLogFullPath; @@ -117,31 +116,6 @@ public string FrameworkLibDirectory { } } - public string MicrosoftNetSdkDirectory { - get { - string dotnetSearchPath = IsUnix ? frameworkSDKRoot - : Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.ProgramFiles), "dotnet", "sdk"); - - string path = FindLatestDotNetSdk (dotnetSearchPath); - if (!string.IsNullOrEmpty (path)) - return path; - - if (!IsUnix) { - var visualStudioDirectory = TestEnvironment.GetVisualStudioDirectory (); - if (!string.IsNullOrEmpty (visualStudioDirectory)) { - path = Path.Combine (visualStudioDirectory, "MSBuild", "Sdks", "Microsoft.NET.Sdk"); - if (File.Exists (Path.Combine (path, "Sdk", "Sdk.props"))) - return path; - } - var x86 = Environment.GetFolderPath (Environment.SpecialFolder.ProgramFilesX86); - path = Path.Combine (x86, "MSBuild", "Sdks", "Microsoft.NET.Sdk"); - if (File.Exists (Path.Combine (path, "Sdk", "Sdk.props"))) - return path; - } - return string.Empty; - } - } - public bool CrossCompilerAvailable (string supportedAbis) { var crossCompilerLookup = new Dictionary { @@ -270,31 +244,6 @@ public Builder () BuildLogFile = "build.log"; Console.WriteLine ($"Using {XABuildExe}"); Console.WriteLine ($"Using {(RunningMSBuild ? "msbuild" : "xbuild")}"); - if (IsUnix && string.IsNullOrEmpty (frameworkSDKRoot)) { - var psi = new ProcessStartInfo ("msbuild") { - RedirectStandardOutput = true, - UseShellExecute = false, - CreateNoWindow = true, - WindowStyle = ProcessWindowStyle.Hidden, - WorkingDirectory = Root, - Arguments = $" {Path.Combine (Root, "FrameworkPath.targets")} /v:minimal /nologo", - }; - using (var process = Process.Start (psi)) { - process.WaitForExit (); - frameworkSDKRoot = process.StandardOutput.ReadToEnd ().Trim (); - } - - //NOTE: some machines aren't returning /msbuild/ on the end - // macOS should be /Library/Frameworks/Mono.framework/Versions/5.18.0/lib/mono/msbuild/ - var dir = Path.GetFileName (frameworkSDKRoot.TrimEnd (Path.DirectorySeparatorChar)); - if (dir != "msbuild") { - var path = Path.Combine (frameworkSDKRoot, "msbuild"); - if (Directory.Exists (path)) - frameworkSDKRoot = path; - } - } - if (!string.IsNullOrEmpty (frameworkSDKRoot)) - Console.WriteLine ($"Using $(FrameworkSDKRoot): {frameworkSDKRoot}"); } public void Dispose () @@ -500,29 +449,6 @@ string QuoteFileName (string fileName) return fileName.Contains (" ") ? $"\"{fileName}\"" : fileName; } - string FindLatestDotNetSdk (string dotNetPath) - { - if (Directory.Exists (dotNetPath)) { - Version latest = new Version (0,0); - string Sdk = null; - foreach (var dir in Directory.EnumerateDirectories (dotNetPath)) { - // Preview versions such as `3.0.100-preview9-013775` will be ignored. - Version.TryParse (Path.GetFileName (dir), out Version version); - var sdksDir = Path.Combine (dir, "Sdks"); - if (!Directory.Exists (sdksDir)) - sdksDir = Path.Combine (dir, "bin", "Sdks"); - if (version != null && version > latest) { - if (Directory.Exists (sdksDir) && File.Exists (Path.Combine (sdksDir, "Microsoft.NET.Sdk", "Sdk", "Sdk.props"))) { - latest = version; - Sdk = Path.Combine (sdksDir, "Microsoft.NET.Sdk"); - } - } - } - return Sdk; - } - return null; - } - } }