Skip to content

Commit aa54170

Browse files
pjcollinspjcollins
authored
[Xamarin.Android.Build.Tests] Import regression tests from job 2 (#5082)
Context: https://github.com/xamarin/QualityAssurance/pull/3644 CI and Xamarin.Android.Build.Tests have been updated to reflect the second round of regression test auditing. All QA regression tests have been merged into two total jobs, one per operating system type. The following regression test categories have been removed, see https://github.com/xamarin/QualityAssurance/pull/3644 for more info: * Proguard - Deprecated tool. * Multidex - Deprecated tool. * ExplicitCrunch - Obsolete build option. * TargetFrameworkTests - Missing coverage partially migrated to BuildBasicApplication and LibraryReferenceWithHigherTFVShouldDisplayWarning. * ResourceCacheTests - Category was previously removed. * PropertyCacheTests - Missing coverage partially migrated to BuildPropsBreaksConvertResourcesCases. * CodeAnalysisTests - Deprecated feature. * HttpClientAndTlsProviderPackageTests - Category was previously removed. * AndroidApiInfoTests - Mostly duplicated in https://github.com/xamarin/xamarin-android/blob/c31ad9f1e7549c632150e58fc99fe7a2c190c79f/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs#L3250 A minor fix has also been applied to Xamarin.Android.Build.Tests to bring in a missing [Test] attribute and fix some asserts that were caught while importing and looking for duplicate tests.
1 parent 079197a commit aa54170

File tree

5 files changed

+147
-196
lines changed

5 files changed

+147
-196
lines changed

‎build-tools/automation/azure-pipelines.yaml‎

Lines changed: 6 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1109,9 +1109,9 @@ stages:
11091109
condition: and(eq(dependencies.mac_build.result, 'Succeeded'), eq(variables['RunAllTests'], true))
11101110
jobs:
11111111

1112-
# Check - "Xamarin.Android (Regression Tests Mac-1)"
1113-
- job: integrated_regression_mac_1
1114-
displayName: Mac-1
1112+
# Check - "Xamarin.Android (Regression Tests Mac)"
1113+
- job: integrated_regression_mac
1114+
displayName: Mac
11151115
pool:
11161116
name: VSEng-Xamarin-Mac-Devices
11171117
demands:
@@ -1122,50 +1122,10 @@ stages:
11221122
clean: all
11231123
steps:
11241124
- template: yaml-templates/run-integrated-regression-tests.yaml
1125-
parameters:
1126-
node_id: 1
1127-
1128-
# Check - "Xamarin.Android (Regression Tests Mac-2)"
1129-
- job: integrated_regression_mac_2
1130-
displayName: Mac-2
1131-
pool:
1132-
name: VSEng-Xamarin-Mac-Devices
1133-
demands:
1134-
- android
1135-
timeoutInMinutes: 240
1136-
cancelTimeoutInMinutes: 5
1137-
workspace:
1138-
clean: all
1139-
steps:
1140-
- template: yaml-templates/run-integrated-regression-tests.yaml
1141-
parameters:
1142-
node_id: 2
1143-
1144-
# Check - "Xamarin.Android (Regression Tests Windows-1)"
1145-
- job: integrated_regression_Win_1
1146-
displayName: Windows-1
1147-
pool:
1148-
name: VSEng-Xamarin-Win-XMA
1149-
demands:
1150-
- android
1151-
timeoutInMinutes: 240
1152-
cancelTimeoutInMinutes: 5
1153-
workspace:
1154-
clean: all
1155-
variables:
1156-
XQA_VISUALSTUDIO_LOCATION: '%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise'
1157-
steps:
1158-
- template: remove-visualstudio.yml@yaml
1159-
1160-
- template: yaml-templates\run-integrated-regression-tests.yaml
1161-
parameters:
1162-
node_id: 1
1163-
1164-
- template: remove-visualstudio.yml@yaml
11651125

1166-
# Check - "Xamarin.Android (Regression Tests Windows-2)"
1167-
- job: integrated_regression_Win_2
1168-
displayName: Windows-2
1126+
# Check - "Xamarin.Android (Regression Tests Windows)"
1127+
- job: integrated_regression_win
1128+
displayName: Windows
11691129
pool:
11701130
name: VSEng-Xamarin-Win-XMA
11711131
demands:
@@ -1180,8 +1140,6 @@ stages:
11801140
- template: remove-visualstudio.yml@yaml
11811141

11821142
- template: yaml-templates\run-integrated-regression-tests.yaml
1183-
parameters:
1184-
node_id: 2
11851143

11861144
- template: remove-visualstudio.yml@yaml
11871145

‎build-tools/automation/yaml-templates/run-integrated-regression-tests.yaml‎

Lines changed: 42 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
parameters:
2-
node_id: 0
3-
41
steps:
52
- task: xamops.azdevex.lingering-process-task.lingering-process-task@1
63

@@ -67,112 +64,47 @@ steps:
6764
}
6865
displayName: Test Environment Setup
6966

70-
- ${{ if eq(parameters.node_id, 1) }}:
71-
- template: run-nunit-tests.yaml
72-
parameters:
73-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
74-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
75-
testRunTitle: Mono.Android-Tests Debug on Device
76-
nunitConsoleExtraArgs: --where "cat == RuntimeTests"
77-
78-
- template: run-nunit-tests.yaml
79-
parameters:
80-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
81-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
82-
testRunTitle: Hello Tests on Device
83-
nunitConsoleExtraArgs: --where "cat == Hello"
84-
condition: and(succeeded(), eq(variables['XA.Commercial.Build'], 'true'))
85-
86-
- template: run-nunit-tests.yaml
87-
parameters:
88-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
89-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
90-
testRunTitle: Smoke Tests on Device
91-
nunitConsoleExtraArgs: --where "cat == RegressionDeviceTests"
92-
93-
- template: run-nunit-tests.yaml
94-
parameters:
95-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
96-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
97-
testRunTitle: Incremental Build Tests
98-
nunitConsoleExtraArgs: --where "cat == BuildPerformance"
99-
100-
- template: run-nunit-tests.yaml
101-
parameters:
102-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
103-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
104-
testRunTitle: AOT Tests
105-
nunitConsoleExtraArgs: --where "cat == AotSupport"
106-
107-
- ${{ if eq(parameters.node_id, 2) }}:
108-
- template: run-nunit-tests.yaml
109-
parameters:
110-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
111-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
112-
testRunTitle: Proguard Tests
113-
nunitConsoleExtraArgs: --where "cat == Proguard"
114-
115-
- template: run-nunit-tests.yaml
116-
parameters:
117-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
118-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
119-
testRunTitle: Multidex Tests
120-
nunitConsoleExtraArgs: --where "cat == Multidex"
121-
122-
- template: run-nunit-tests.yaml
123-
parameters:
124-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
125-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
126-
testRunTitle: ExplicitCrunch Tests
127-
nunitConsoleExtraArgs: --where "cat == ExplicitCrunch"
128-
129-
- template: run-nunit-tests.yaml
130-
parameters:
131-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
132-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
133-
testRunTitle: TargetFrameworkVersion Tests
134-
nunitConsoleExtraArgs: --where "cat == TargetFrameworkTests"
135-
136-
- template: run-nunit-tests.yaml
137-
parameters:
138-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
139-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
140-
testRunTitle: Resource Cache Tests
141-
nunitConsoleExtraArgs: --where "cat == ResourceCacheTests"
142-
143-
- template: run-nunit-tests.yaml
144-
parameters:
145-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
146-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
147-
testRunTitle: Property Cache Tests
148-
nunitConsoleExtraArgs: --where "cat == PropertyCacheTests"
149-
150-
- template: run-nunit-tests.yaml
151-
parameters:
152-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
153-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
154-
testRunTitle: Code Analysis Tests
155-
nunitConsoleExtraArgs: --where "cat == CodeAnalysisTests"
156-
157-
- template: run-nunit-tests.yaml
158-
parameters:
159-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
160-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
161-
testRunTitle: Networking Options Tests
162-
nunitConsoleExtraArgs: --where "cat == HttpClientAndTlsProviderPackageTests"
163-
164-
- template: run-nunit-tests.yaml
165-
parameters:
166-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
167-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
168-
testRunTitle: AndroidApiInfo Tests
169-
nunitConsoleExtraArgs: --where "cat == AndroidApiInfoTests"
170-
171-
- template: run-nunit-tests.yaml
172-
parameters:
173-
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
174-
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
175-
testRunTitle: Test Environment Cleanup
176-
nunitConsoleExtraArgs: --where "cat == XATestCleanUp"
67+
- template: run-nunit-tests.yaml
68+
parameters:
69+
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
70+
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
71+
testRunTitle: Mono.Android-Tests Debug on Device
72+
nunitConsoleExtraArgs: --where "cat == RuntimeTests"
73+
74+
- template: run-nunit-tests.yaml
75+
parameters:
76+
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
77+
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
78+
testRunTitle: Hello Tests on Device
79+
nunitConsoleExtraArgs: --where "cat == Hello"
80+
condition: and(succeeded(), eq(variables['XA.Commercial.Build'], 'true'))
81+
82+
- template: run-nunit-tests.yaml
83+
parameters:
84+
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
85+
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
86+
testRunTitle: Smoke Tests on Device
87+
nunitConsoleExtraArgs: --where "cat == RegressionDeviceTests"
88+
89+
- template: run-nunit-tests.yaml
90+
parameters:
91+
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
92+
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
93+
testRunTitle: Incremental Build Tests
94+
nunitConsoleExtraArgs: --where "cat == BuildPerformance"
95+
96+
- template: run-nunit-tests.yaml
97+
parameters:
98+
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
99+
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
100+
testRunTitle: AOT Tests
101+
nunitConsoleExtraArgs: --where "cat == AotSupport"
102+
103+
- template: run-nunit-tests.yaml
104+
parameters:
105+
nunitConsole: $(System.DefaultWorkingDirectory)/NUnit.ConsoleRunner/tools/nunit3-console.exe
106+
testAssembly: $(System.DefaultWorkingDirectory)/XQA.Android/XQA.Android.dll
107+
testRunTitle: Test Environment Cleanup
108+
nunitConsoleExtraArgs: --where "cat == XATestCleanUp"
177109

178110
- template: fail-on-issue.yaml

‎src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs‎

Lines changed: 49 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,23 @@ public void CompressedWithoutLinker ()
3535
}
3636
}
3737

38+
public static string [] SupportedTargetFrameworks ()
39+
{
40+
using (var b = new Builder ()) {
41+
if (Builder.UseDotNet)
42+
return new string [] { b.LatestTargetFrameworkVersion () };
43+
else
44+
return b.GetAllSupportedTargetFrameworkVersions ();
45+
}
46+
}
47+
3848
[Test]
3949
[Category ("dotnet")]
40-
public void BuildBasicApplication ([Values (true, false)] bool isRelease)
50+
public void BuildBasicApplication ([ValueSource (nameof (SupportedTargetFrameworks))] string tfv, [Values (true, false)] bool isRelease)
4151
{
4252
var proj = new XamarinAndroidApplicationProject {
4353
IsRelease = isRelease,
54+
TargetFrameworkVersion = tfv,
4455
};
4556
using (var b = CreateApkBuilder ()) {
4657
Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
@@ -637,31 +648,6 @@ public CustomTextView(Context context, IAttributeSet attributes) : base(context,
637648
}
638649
}
639650

640-
[Test]
641-
public void BuildPropsBreaksConvertResourcesCases ([Values (true, false)] bool useAapt2)
642-
{
643-
var proj = new XamarinAndroidApplicationProject () {
644-
AndroidResources = {
645-
new AndroidItem.AndroidResource (() => "Resources\\drawable\\IMALLCAPS.png") {
646-
BinaryContent = () => XamarinAndroidApplicationProject.icon_binary_mdpi,
647-
},
648-
new AndroidItem.AndroidResource ("Resources\\layout\\test.axml") {
649-
TextContent = () => {
650-
return "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<ImageView xmlns:android=\"http://schemas.android.com/apk/res/android\" android:src=\"@drawable/IMALLCAPS\" />";
651-
}
652-
}
653-
}
654-
};
655-
proj.SetProperty ("AndroidUseAapt2", useAapt2.ToString ());
656-
using (var b = CreateApkBuilder (Path.Combine ("temp", TestName))) {
657-
Assert.IsTrue (b.Build (proj), "first build should have succeeded.");
658-
//Invalidate build.props with newer timestamp, you could also modify anything in @(_PropertyCacheItems)
659-
var props = b.Output.GetIntermediaryPath("build.props");
660-
File.SetLastWriteTimeUtc(props, DateTime.UtcNow);
661-
Assert.IsTrue (b.Build (proj), "second build should have succeeded.");
662-
}
663-
}
664-
665651
[Test]
666652
public void AndroidResourceNotExist ()
667653
{
@@ -1366,6 +1352,9 @@ public override void OnCreate()
13661352
using (var b = CreateApkBuilder ("temp/CustomApplicationClassAndMultiDex")) {
13671353
Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
13681354
Assert.IsFalse (b.LastBuildOutput.ContainsText ("Duplicate zip entry"), "Should not get warning about [META-INF/MANIFEST.MF]");
1355+
var customAppContent = File.ReadAllText (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "src", "com", "foxsports", "test", "CustomApp.java"));
1356+
Assert.IsTrue (customAppContent.Contains ("extends android.support.multidex.MultiDexApplication"),
1357+
"Custom App class should have inherited from android.support.multidex.MultiDexApplication.");
13691358
}
13701359
}
13711360

@@ -1691,6 +1680,7 @@ public void BuildBasicApplicationCheckConfigFiles ()
16911680
}
16921681
}
16931682

1683+
[Test]
16941684
public void BuildApplicationCheckItEmitsAWarningWithContentItems ()
16951685
{
16961686
var proj = new XamarinAndroidApplicationProject ();
@@ -1703,12 +1693,10 @@ public void BuildApplicationCheckItEmitsAWarningWithContentItems ()
17031693
TextContent = () => "Test Content 1"
17041694
});
17051695
Assert.IsTrue (b.Build (proj), "Build should have built successfully");
1706-
Assert.IsTrue (
1707-
b.LastBuildOutput.Contains ("TestContent.txt: warning XA0101: @(Content) build action is not supported"),
1708-
"Build Output did not contain the correct error message");
1709-
Assert.IsTrue (
1710-
b.LastBuildOutput.Contains ("TestContent1.txt: warning XA0101: @(Content) build action is not supported"),
1711-
"Build Output did not contain the correct error message");
1696+
StringAssertEx.Contains ("TestContent.txt : warning XA0101: @(Content) build action is not supported", b.LastBuildOutput,
1697+
"Build Output did not contain 'TestContent.txt : warning XA0101'.");
1698+
StringAssertEx.Contains ("TestContent1.txt : warning XA0101: @(Content) build action is not supported", b.LastBuildOutput,
1699+
"Build Output did not contain 'TestContent1.txt : warning XA0101'.");
17121700
}
17131701
}
17141702

@@ -2474,7 +2462,7 @@ public void CheckTargetFrameworkVersion ([Values (true, false)] bool isRelease)
24742462
};
24752463
proj.SetProperty ("AndroidUseLatestPlatformSdk", "False");
24762464
using (var builder = CreateApkBuilder ()) {
2477-
builder.GetTargetFrameworkVersionRange (out var _, out string firstFrameworkVersion, out var _, out string lastFrameworkVersion);
2465+
builder.GetTargetFrameworkVersionRange (out var _, out string firstFrameworkVersion, out var _, out string lastFrameworkVersion, out string[] _);
24782466
proj.SetProperty ("TargetFrameworkVersion", firstFrameworkVersion);
24792467
if (!Directory.Exists (Path.Combine (builder.FrameworkLibDirectory, firstFrameworkVersion)))
24802468
Assert.Ignore ("This is a Pull Request Build. Ignoring test.");
@@ -4003,6 +3991,34 @@ public void LibraryProjectsShouldSkipGetPrimaryCpuAbi ()
40033991
}
40043992
}
40053993

3994+
[Test]
3995+
[Category ("Commercial")]
3996+
public void LibraryReferenceWithHigherTFVShouldDisplayWarning ([Values (true, false)] bool isRelease)
3997+
{
3998+
if (!CommercialBuildAvailable || Builder.UseDotNet)
3999+
Assert.Ignore ("Not applicable to One .NET or single framework OSS builds.");
4000+
4001+
var libproj = new XamarinAndroidLibraryProject () {
4002+
IsRelease = isRelease,
4003+
ProjectName = "Library1",
4004+
};
4005+
var proj = new XamarinAndroidApplicationProject () {
4006+
IsRelease = isRelease,
4007+
ProjectName = "App1",
4008+
UseLatestPlatformSdk = false,
4009+
TargetFrameworkVersion = "v9.0",
4010+
References = {
4011+
new BuildItem ("ProjectReference", $"..\\{libproj.ProjectName}\\{Path.GetFileName (libproj.ProjectFilePath)}")
4012+
},
4013+
};
4014+
using (var libBuilder = CreateDllBuilder (Path.Combine ("temp", TestName, libproj.ProjectName)))
4015+
using (var appBuilder = CreateApkBuilder (Path.Combine ("temp", TestName, proj.ProjectName))) {
4016+
Assert.IsTrue (libBuilder.Build (libproj), "Library build should have succeeded.");
4017+
Assert.IsTrue (appBuilder.Build (proj), "App build should have succeeded.");
4018+
StringAssertEx.Contains ("warning XA0105", appBuilder.LastBuildOutput, "Build should have produced warning XA0105.");
4019+
}
4020+
}
4021+
40064022
[Test]
40074023
public void AllResourcesInClassLibrary ([Values (true, false)] bool useAapt2)
40084024
{

0 commit comments

Comments
 (0)