Skip to content

Commit 1bb9b02

Browse files
pjcollinspjcollins
authored
[tests/ci] Add MSBuild test jobs to Azure Pipelines (#3347)
Adds new `mac_msbuild_tests`, `mac_msbuilddevice_tests`, and `mac_timezonedevice_tests` jobs to azure-pipelines.yaml to run the msbuild tests against our installers. Other changes and fixes: * Removes the dependency on the generated XABuildPaths.cs file so that test assemblies can be built and executed on different environments. * Standardizes Android SDK and NDK path and OS information lookup to reduce some code duplication and better allow for toolchain path overriding. * Updates Mono to 6.0.0.313 and .NET Core to 2.1.701 for our macOS building and test stages. * Introduces a new `UsesDevice` test category to allow us to filter execution of tests which will be skipped or behave differently when a device or emulator is attached. * Updates clean tests to ignore the `FileListAbsolute.txt` file which is no longer deleted by the clean target on macOS. * Updates the ParallelScope attribute for some multi source tests to avoid NuGet restore and intermittent failures occurring on macOS. * Attaches relevant build log file paths to the test result xml rather than dumping the full log to the test output. It's a bit painful to scroll through diagnostic build output to see which tests failed. Build logs can now be viewed directly in the 'attachments' tab of each test failure in Azure Pipelines:
1 parent 63710e6 commit 1bb9b02

File tree

35 files changed

+542
-296
lines changed

35 files changed

+542
-296
lines changed

‎.external‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
xamarin/monodroid:master@237e0bd9f105b9842778ef82161a20c6d4497a40
1+
xamarin/monodroid:master@63bbeb076d809c74811a8001d38bf2e9e8672627
22
mono/mono:2019-06@761220d914e28b3ebd67e5642cf8c200110f62ec

‎build-tools/automation/azure-pipelines.yaml‎

Lines changed: 224 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ resources:
1919
variables:
2020
BundleArtifactName: bundle
2121
InstallerArtifactName: installers
22+
NUnitTestAssembliesArtifactName: test-assemblies
23+
NUnitConsoleVersion: 3.9.0
2224

2325
# Stage and Job "display names" are shortened because they are combined to form the name of the corresponding GitHub check.
2426
# Check - "Xamarin.Android (Prepare bundle)"
@@ -37,6 +39,11 @@ stages:
3739
- checkout: self
3840
submodules: recursive
3941

42+
- task: UseDotNet@2
43+
displayName: install .NET Core 2.1.701
44+
inputs:
45+
version: 2.1.701
46+
4047
# Update Mono in a separate step since xaprepare uses it as well and it will crash when Mono it runs with is updated
4148
# The 'prepare' step creates the bundle
4249
- script: |
@@ -90,6 +97,11 @@ stages:
9097
- checkout: self
9198
submodules: recursive
9299

100+
- task: UseDotNet@2
101+
displayName: install .NET Core 2.1.701
102+
inputs:
103+
version: 2.1.701
104+
93105
- task: DownloadPipelineArtifact@1
94106
inputs:
95107
artifactName: $(BundleArtifactName)
@@ -115,6 +127,12 @@ stages:
115127
- script: make jenkins V=1 CONFIGURATION=$(XA.Build.Configuration) PREPARE_CI=1 PREPARE_AUTOPROVISION=1 PREPARE_ARGS="--bundle-path=$(System.DefaultWorkingDirectory)"
116128
displayName: make jenkins
117129

130+
- task: PublishPipelineArtifact@0
131+
displayName: upload test assemblies
132+
inputs:
133+
artifactName: $(NUnitTestAssembliesArtifactName)
134+
targetPath: bin/Test$(XA.Build.Configuration)
135+
118136
- script: make create-installers V=1 CONFIGURATION=$(XA.Build.Configuration)
119137
displayName: make create-installers
120138

@@ -304,43 +322,9 @@ stages:
304322
variables:
305323
ApkTestConfiguration: Release
306324
steps:
307-
- task: DownloadPipelineArtifact@1
308-
inputs:
309-
artifactName: $(InstallerArtifactName)
310-
itemPattern: "*.pkg"
311-
downloadPath: $(System.DefaultWorkingDirectory)
312-
313-
- template: yaml-templates/run-installer.yaml
314-
315-
- task: MSBuild@1
316-
displayName: build xaprepare
317-
inputs:
318-
solution: build-tools/xaprepare/xaprepare.sln
319-
configuration: $(ApkTestConfiguration)
320-
msbuildArguments: /t:Restore,Build
321-
322-
- script: |
323-
mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=UpdateMono --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
324-
mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=Required --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
325-
mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=AndroidToolchain --no-emoji --run-mode=CI
326-
displayName: provision dependencies
327-
328-
- task: NuGetCommand@2
329-
displayName: nuget restore Xamarin.Android.Tools.sln
330-
inputs:
331-
restoreSolution: external/xamarin-android-tools/Xamarin.Android.Tools.sln
332-
333-
- task: MSBuild@1
334-
displayName: build Xamarin.Android.Tools.BootstrapTasks.csproj
335-
inputs:
336-
solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
325+
- template: yaml-templates/mac/setup-test-environment.yaml
326+
parameters:
337327
configuration: $(ApkTestConfiguration)
338-
msbuildArguments: /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/BootstrapTasks.binlog
339-
340-
- task: NuGetCommand@2
341-
displayName: nuget restore Xamarin.Android-Tests.sln
342-
inputs:
343-
restoreSolution: Xamarin.Android-Tests.sln
344328

345329
- template: yaml-templates/apk-instrumentation.yaml
346330
parameters:
@@ -493,6 +477,210 @@ stages:
493477
targetPath: $(Build.ArtifactStagingDirectory)
494478
condition: always()
495479

480+
# Check - "Xamarin.Android (Test MSBuild Mac)"
481+
- job: mac_msbuild_tests
482+
displayName: MSBuild Mac
483+
pool: $(XA.Build.Mac.Pool)
484+
timeoutInMinutes: 180
485+
cancelTimeoutInMinutes: 5
486+
workspace:
487+
clean: all
488+
steps:
489+
- template: yaml-templates/mac/setup-test-environment.yaml
490+
491+
- task: DownloadPipelineArtifact@1
492+
inputs:
493+
artifactName: $(NUnitTestAssembliesArtifactName)
494+
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
495+
496+
- script: >
497+
mono packages/NUnit.ConsoleRunner.$(NUnitConsoleVersion)/tools/nunit3-console.exe
498+
$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.dll
499+
--result TestResult-MSBuildTests-$(XA.Build.Configuration).xml
500+
displayName: run Xamarin.Android.Build.Tests
501+
502+
- task: PublishTestResults@2
503+
displayName: publish test results
504+
inputs:
505+
testResultsFormat: NUnit
506+
testResultsFiles: TestResult-MSBuildTests-$(XA.Build.Configuration).xml
507+
testRunTitle: MSBuildTestsMac
508+
condition: succeededOrFailed()
509+
510+
- script: >
511+
mono packages/NUnit.ConsoleRunner.$(NUnitConsoleVersion)/tools/nunit3-console.exe
512+
$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.Commercial.dll
513+
--result TestResult-MSBuildTestsCommercial-$(XA.Build.Configuration).xml
514+
displayName: run Xamarin.Android.Build.Tests.Commercial
515+
condition: and(succeededOrFailed(), eq(variables['XA.Commercial.Build'], 'true'))
516+
517+
- task: PublishTestResults@2
518+
displayName: publish test results
519+
inputs:
520+
testResultsFormat: NUnit
521+
testResultsFiles: TestResult-MSBuildTestsCommercial-$(XA.Build.Configuration).xml
522+
testRunTitle: MSBuildTestsMac
523+
condition: and(succeededOrFailed(), eq(variables['XA.Commercial.Build'], 'true'))
524+
525+
- task: MSBuild@1
526+
displayName: package results
527+
inputs:
528+
solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
529+
configuration: $(XA.Build.Configuration)
530+
msbuildArguments: /t:Build,ZipBuildStatus,ZipTestResults /p:BuildStatusZipOutputPath=$(Build.ArtifactStagingDirectory) /p:TestResultZipOutputPath=$(Build.ArtifactStagingDirectory)
531+
condition: always()
532+
533+
- task: PublishPipelineArtifact@0
534+
displayName: upload artifacts
535+
inputs:
536+
artifactName: mac-msbuild-testresults
537+
targetPath: $(Build.ArtifactStagingDirectory)
538+
condition: always()
539+
540+
# Check - "Xamarin.Android (Test MSBuild With Emulator Mac)"
541+
- job: mac_msbuilddevice_tests
542+
displayName: MSBuild With Emulator Mac
543+
pool: $(XA.Build.Mac.Pool)
544+
timeoutInMinutes: 120
545+
cancelTimeoutInMinutes: 5
546+
workspace:
547+
clean: all
548+
steps:
549+
- template: yaml-templates/mac/setup-test-environment.yaml
550+
551+
- task: DownloadPipelineArtifact@1
552+
inputs:
553+
artifactName: $(NUnitTestAssembliesArtifactName)
554+
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
555+
556+
- task: MSBuild@1
557+
displayName: start emulator
558+
inputs:
559+
solution: src/Mono.Android/Test/Mono.Android-Tests.csproj
560+
configuration: $(XA.Build.Configuration)
561+
msbuildArguments: >
562+
/t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
563+
564+
- script: >
565+
mono packages/NUnit.ConsoleRunner.$(NUnitConsoleVersion)/tools/nunit3-console.exe
566+
$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.dll
567+
--where "cat == UsesDevice"
568+
--result TestResult-MSBuildDeviceTests-$(XA.Build.Configuration).xml
569+
displayName: run on-device Xamarin.Android.Build.Tests
570+
571+
- task: PublishTestResults@2
572+
displayName: publish test results
573+
inputs:
574+
testResultsFormat: NUnit
575+
testResultsFiles: TestResult-MSBuildDeviceTests-$(XA.Build.Configuration).xml
576+
testRunTitle: MSBuildDeviceTestsMac
577+
condition: always()
578+
579+
- script: >
580+
mono packages/NUnit.ConsoleRunner.$(NUnitConsoleVersion)/tools/nunit3-console.exe
581+
$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/MSBuildDeviceIntegration.dll
582+
--where "test != Xamarin.Android.Build.Tests.DeploymentTest.CheckTimeZoneInfoIsCorrect"
583+
--result TestResult-MSBuildDeviceIntegration-$(XA.Build.Configuration).xml
584+
displayName: run on-device MSBuildDeviceIntegration tests
585+
condition: succeededOrFailed()
586+
587+
- task: PublishTestResults@2
588+
displayName: publish test results
589+
inputs:
590+
testResultsFormat: NUnit
591+
testResultsFiles: TestResult-MSBuildDeviceIntegration-$(XA.Build.Configuration).xml
592+
testRunTitle: MSBuildDeviceIntegration
593+
condition: always()
594+
595+
- task: MSBuild@1
596+
displayName: shut down emulator
597+
inputs:
598+
solution: src/Mono.Android/Test/Mono.Android-Tests.csproj
599+
configuration: $(XA.Build.Configuration)
600+
msbuildArguments: >
601+
/t:AcquireAndroidTarget,ReleaseAndroidTarget
602+
/bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
603+
condition: always()
604+
605+
- task: MSBuild@1
606+
displayName: package results
607+
inputs:
608+
solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
609+
configuration: $(XA.Build.Configuration)
610+
msbuildArguments: /t:Build,ZipBuildStatus,ZipTestResults /p:BuildStatusZipOutputPath=$(Build.ArtifactStagingDirectory) /p:TestResultZipOutputPath=$(Build.ArtifactStagingDirectory)
611+
condition: always()
612+
613+
- task: PublishPipelineArtifact@0
614+
displayName: upload artifacts
615+
inputs:
616+
artifactName: mac-msbuild-device-testresults
617+
targetPath: $(Build.ArtifactStagingDirectory)
618+
condition: always()
619+
620+
# Check - "Xamarin.Android (Test TimeZoneInfo With Emulator Mac)"
621+
- job: mac_timezonedevice_tests
622+
displayName: TimeZoneInfo With Emulator Mac
623+
pool: $(XA.Build.Mac.Pool)
624+
timeoutInMinutes: 240
625+
cancelTimeoutInMinutes: 5
626+
workspace:
627+
clean: all
628+
steps:
629+
- template: yaml-templates/mac/setup-test-environment.yaml
630+
631+
- task: DownloadPipelineArtifact@1
632+
inputs:
633+
artifactName: $(NUnitTestAssembliesArtifactName)
634+
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
635+
636+
- task: MSBuild@1
637+
displayName: start emulator
638+
inputs:
639+
solution: src/Mono.Android/Test/Mono.Android-Tests.csproj
640+
configuration: $(XA.Build.Configuration)
641+
msbuildArguments: >
642+
/t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
643+
644+
- script: >
645+
mono packages/NUnit.ConsoleRunner.$(NUnitConsoleVersion)/tools/nunit3-console.exe
646+
$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/MSBuildDeviceIntegration.dll
647+
--where "test == Xamarin.Android.Build.Tests.DeploymentTest.CheckTimeZoneInfoIsCorrect"
648+
--result TestResult-TimeZoneInfoTests-$(XA.Build.Configuration).xml
649+
displayName: run TimeZoneInfo tests
650+
651+
- task: PublishTestResults@2
652+
displayName: publish test results
653+
inputs:
654+
testResultsFormat: NUnit
655+
testResultsFiles: TestResult-TimeZoneInfoTests-$(XA.Build.Configuration).xml
656+
testRunTitle: TimeZoneInfoTestsMac
657+
condition: always()
658+
659+
- task: MSBuild@1
660+
displayName: shut down emulator
661+
inputs:
662+
solution: src/Mono.Android/Test/Mono.Android-Tests.csproj
663+
configuration: $(XA.Build.Configuration)
664+
msbuildArguments: >
665+
/t:AcquireAndroidTarget,ReleaseAndroidTarget
666+
/bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
667+
condition: always()
668+
669+
- task: MSBuild@1
670+
displayName: package results
671+
inputs:
672+
solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
673+
configuration: $(XA.Build.Configuration)
674+
msbuildArguments: /t:Build,ZipBuildStatus,ZipTestResults /p:BuildStatusZipOutputPath=$(Build.ArtifactStagingDirectory) /p:TestResultZipOutputPath=$(Build.ArtifactStagingDirectory)
675+
condition: always()
676+
677+
- task: PublishPipelineArtifact@0
678+
displayName: upload artifacts
679+
inputs:
680+
artifactName: mac-timezoneinfo-testresults
681+
targetPath: $(Build.ArtifactStagingDirectory)
682+
condition: always()
683+
496684
# Check - "Xamarin.Android (Test Designer Mac)"
497685
- job: designer_integration_mac
498686
displayName: Designer Mac

‎build-tools/automation/yaml-templates/mac/setup-test-environment.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
parameters:
2+
configuration: $(XA.Build.Configuration)
3+
4+
steps:
5+
- task: DownloadPipelineArtifact@1
6+
inputs:
7+
artifactName: $(InstallerArtifactName)
8+
itemPattern: "*.pkg"
9+
downloadPath: $(System.DefaultWorkingDirectory)
10+
11+
- template: ../run-installer.yaml
12+
13+
- task: MSBuild@1
14+
displayName: build xaprepare
15+
inputs:
16+
solution: build-tools/xaprepare/xaprepare.sln
17+
configuration: ${{ parameters.configuration }}
18+
msbuildArguments: /t:Restore,Build
19+
20+
- script: |
21+
mono build-tools/xaprepare/xaprepare/bin/${{ parameters.configuration }}/xaprepare.exe --s=UpdateMono --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
22+
mono build-tools/xaprepare/xaprepare/bin/${{ parameters.configuration }}/xaprepare.exe --s=Required --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
23+
mono build-tools/xaprepare/xaprepare/bin/${{ parameters.configuration }}/xaprepare.exe --s=AndroidToolchain --no-emoji --run-mode=CI
24+
displayName: install xaprepare dependencies
25+
26+
- task: UseDotNet@2
27+
displayName: install .NET Core 2.1.701
28+
inputs:
29+
version: 2.1.701
30+
31+
# Restore solutions for Xamarin.Android.Tools.sln, Xamarin.Android.sln, and Xamarin.Android-Tests.sln
32+
- task: NuGetCommand@2
33+
displayName: nuget restore Xamarin.Android solutions
34+
inputs:
35+
restoreSolution: '**/Xamarin.Android*.sln'
36+
37+
- task: MSBuild@1
38+
displayName: build Xamarin.Android.Tools.BootstrapTasks.csproj
39+
inputs:
40+
solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
41+
configuration: ${{ parameters.configuration }}
42+
msbuildArguments: /bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/BootstrapTasks.binlog
43+

‎build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,7 @@ partial class Configurables
77
partial class Urls
88
{
99
public static readonly Uri Corretto = new Uri ("https://d3pxv6yz143wms.cloudfront.net/8.212.04.2/amazon-corretto-8.212.04.2-macosx-x64.tar.gz");
10-
11-
// 6.0: https://download.mono-project.com/archive/6.0.0/macos-10-universal/MonoFramework-MDK-6.0.0.241.macos10.xamarin.universal.pkg
12-
// Currently breaks the build, thus we use the latest stable below
13-
public static readonly Uri MonoPackage = new Uri ("https://download.mono-project.com/archive/5.20.1/macos-10-universal/MonoFramework-MDK-5.20.1.19.macos10.xamarin.universal.pkg");
10+
public static readonly Uri MonoPackage = new Uri ("https://download.mono-project.com/archive/6.0.0/macos-10-universal/MonoFramework-MDK-6.0.0.313.macos10.xamarin.universal.pkg");
1411
}
1512

1613
partial class Defaults

‎build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/MacOS.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ partial class MacOS
3434

3535
// If you change the minimum Mono version here, please change the URL as well
3636
new MonoPkgProgram ("Mono", "com.xamarin.mono-MDK.pkg", Configurables.Urls.MonoPackage) {
37-
MinimumVersion = "5.20.1.19",
38-
MaximumVersion = "5.99.0.0",
37+
MinimumVersion = "6.0.0.313",
38+
MaximumVersion = "6.99.0.0",
3939
},
4040
};
4141

0 commit comments

Comments
 (0)