Skip to content

Commit c4e8165

Browse files
grendellojonpryor
grendello
authored and
jonpryor
committed
[BCL-Tests] New unit test runner (#1075)
This is a rewrite of our (very old) NUnit test runner to support tests using other test suites. The reason for this is that Mono BCL tests are starting to use xUnit tests and we want to be able to run all of them. At the same time this code drops support for the "GUI" we previously had to run tests (not maintained, makes the actual runner code messy) and restructures the runner infrastructure so that it's possible to add any number of runners in the future without rewriting any base code. We now use only Android instrumentations to execute the code. A few changes to the test running code were necessary since we now support more than one instrumentation per apk, with different result outputs, options etc. Support for granting Android permissions on apk install time was added as well (might be necessary when running tests on devices with Oreo)
1 parent 6ea761f commit c4e8165

File tree

71 files changed

+3348
-400
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+3348
-400
lines changed

‎Xamarin.Android-Tests.sln‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Core", "Core", "{EFBC4DC0-D
1111
EndProject
1212
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Locales", "Locales", "{D6BFEDF6-2F48-44B2-9553-F2F6F92531BD}"
1313
EndProject
14+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BCL-Tests", "BCL-Tests", "{6C86878D-9EBF-45B3-9368-C0EA4026F706}"
15+
EndProject
1416
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibraryResources", "tests\locales\LibraryResources\LibraryResources.csproj", "{05768F39-7BAF-43E6-971E-712F5771E88E}"
1517
EndProject
1618
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.Locale-Tests", "tests\locales\Xamarin.Android.Locale-Tests\Xamarin.Android.Locale-Tests.csproj", "{9D5C83B5-70D5-4CC2-9DB7-78B23DC8F255}"
@@ -49,7 +51,15 @@ Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "Xamarin.Forms.Performance.I
4951
EndProject
5052
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Forms.Performance.Integration.Droid", "tests\Xamarin.Forms-Performance-Integration\Droid\Xamarin.Forms.Performance.Integration.Droid.csproj", "{576312CC-83FF-48B1-A473-488CDC7121AD}"
5153
EndProject
52-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.Bcl-Tests", "tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj", "{E865C28E-32AF-4210-A41D-5791C39A9D85}"
54+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.Bcl-Tests", "tests\BCL-Tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj", "{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}"
55+
EndProject
56+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestRunner.Core", "tests\TestRunner.Core\TestRunner.Core.csproj", "{3CC4E384-4985-4D93-A34C-73F69A379FA7}"
57+
EndProject
58+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestRunner.NUnit", "tests\TestRunner.NUnit\TestRunner.NUnit.csproj", "{CB2335CB-0050-4020-8A05-E9614EDAA05E}"
59+
EndProject
60+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestRunner.xUnit", "tests\TestRunner.xUnit\TestRunner.xUnit.csproj", "{57DC8529-2628-40C4-B27E-BAC1AE44A706}"
61+
EndProject
62+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalTests.NUnit", "tests\BCL-Tests\LocalTests.NUnit\LocalTests.NUnit.csproj", "{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}"
5363
EndProject
5464
Global
5565
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -125,10 +135,26 @@ Global
125135
{576312CC-83FF-48B1-A473-488CDC7121AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
126136
{576312CC-83FF-48B1-A473-488CDC7121AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
127137
{576312CC-83FF-48B1-A473-488CDC7121AD}.Release|Any CPU.Build.0 = Release|Any CPU
128-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
129-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Debug|Any CPU.Build.0 = Debug|Any CPU
130-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Release|Any CPU.ActiveCfg = Release|Any CPU
131-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Release|Any CPU.Build.0 = Release|Any CPU
138+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
139+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
140+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
141+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Release|Any CPU.Build.0 = Release|Any CPU
142+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
143+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Debug|Any CPU.Build.0 = Debug|Any CPU
144+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Release|Any CPU.ActiveCfg = Release|Any CPU
145+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Release|Any CPU.Build.0 = Release|Any CPU
146+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
147+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Debug|Any CPU.Build.0 = Debug|Any CPU
148+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Release|Any CPU.ActiveCfg = Release|Any CPU
149+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Release|Any CPU.Build.0 = Release|Any CPU
150+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
151+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Debug|Any CPU.Build.0 = Debug|Any CPU
152+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Release|Any CPU.ActiveCfg = Release|Any CPU
153+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Release|Any CPU.Build.0 = Release|Any CPU
154+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
155+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
156+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
157+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Release|Any CPU.Build.0 = Release|Any CPU
132158
EndGlobalSection
133159
GlobalSection(NestedProjects) = preSolution
134160
{2305B00D-DE81-4744-B0DA-357835CAFE5A} = {43A4FB09-279A-4138-8027-EC1E1CED2E8A}
@@ -147,5 +173,10 @@ Global
147173
{9802CB35-0BC0-4EE1-9A81-BB40BD97945A} = {E49089E4-4CE0-475E-BE9C-0AB4E4D56EE9}
148174
{5BDCEF07-E3D7-4E73-B025-6E43A5A7D7F1} = {E49089E4-4CE0-475E-BE9C-0AB4E4D56EE9}
149175
{B297008B-C313-455E-B230-E119589D2D79} = {E49089E4-4CE0-475E-BE9C-0AB4E4D56EE9}
176+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
177+
{3CC4E384-4985-4D93-A34C-73F69A379FA7} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
178+
{CB2335CB-0050-4020-8A05-E9614EDAA05E} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
179+
{57DC8529-2628-40C4-B27E-BAC1AE44A706} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
180+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
150181
EndGlobalSection
151182
EndGlobal

‎build-tools/scripts/RunTests.targets‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
<_ApkTestProject Include="$(_TopDir)\tests\CodeGen-Binding\Xamarin.Android.JcwGen-Tests\Xamarin.Android.JcwGen-Tests.csproj" />
2020
<_ApkTestProject Include="$(_TopDir)\tests\CodeGen-MkBundle\Xamarin.Android.MakeBundle-Tests\Xamarin.Android.MakeBundle-Tests.csproj" />
2121
<_ApkTestProject Include="$(_TopDir)\tests\locales\Xamarin.Android.Locale-Tests\Xamarin.Android.Locale-Tests.csproj" />
22-
<_ApkTestProject Include="$(_TopDir)\tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj" />
22+
<_ApkTestProject Include="$(_TopDir)\tests\BCL-Tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj" />
2323
<_ApkTestProject Include="$(_TopDir)\tests\Xamarin.Forms-Performance-Integration\Droid\Xamarin.Forms.Performance.Integration.Droid.csproj" />
2424
<_ApkTestProjectAot Include="$(_TopDir)\src\Mono.Android\Test\Mono.Android-Tests.csproj" />
2525
</ItemGroup>

‎build-tools/scripts/TestApks.targets‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -153,16 +153,22 @@
153153

154154
<Target Name="RunTestApks"
155155
Condition=" '@(TestApk)' != '' ">
156+
<Xamarin.Android.Tools.BootstrapTasks.Adb
157+
Condition=" '@(TestApkPermission)' != '' "
158+
Arguments="$(_AdbTarget) $(AdbOptions) shell pm grant %(TestApkPermission.Package) android.permission.%(TestApkPermission.Identity)"
159+
ToolExe="$(AdbToolExe)"
160+
ToolPath="$(AdbToolPath)"
161+
/>
156162
<PropertyGroup>
157163
<_IncludeCategories Condition=" '$(IncludeCategories)' != '' ">include=$(IncludeCategories)</_IncludeCategories>
158164
<_ExcludeCategories Condition=" '$(ExcludeCategories)' != '' ">exclude=$(ExcludeCategories)</_ExcludeCategories>
159165
</PropertyGroup>
160166
<RunInstrumentationTests
161-
Condition=" '%(TestApk.InstrumentationType)' != ''"
167+
Condition=" '%(TestApkInstrumentation.Identity)' != ''"
162168
AdbTarget="$(_AdbTarget)"
163169
AdbOptions="$(AdbOptions)"
164-
Component="%(TestApk.Package)/%(TestApk.InstrumentationType)"
165-
NUnit2TestResultsFile="%(TestApk.ResultsPath)"
170+
Component="%(TestApkInstrumentation.Package)/%(TestApkInstrumentation.Identity)"
171+
NUnit2TestResultsFile="%(TestApkInstrumentation.ResultsPath)"
166172
InstrumentationArguments="$(_IncludeCategories);$(_ExcludeCategories)"
167173
TestFixture="$(TestFixture)"
168174
ToolExe="$(AdbToolExe)"
@@ -223,11 +229,11 @@
223229
<Target Name="RenameApkTestCases"
224230
Condition=" '@(TestApk)' != '' ">
225231
<RenameTestCases
226-
Condition=" '%(TestApk.ResultsPath)' != '' "
232+
Condition=" '%(TestApkInstrumentation.ResultsPath)' != '' "
227233
Configuration="$(Configuration)$(TestsAotName)"
228234
DeleteSourceFiles="True"
229235
DestinationFolder="$(MSBuildThisFileDirectory)..\.."
230-
SourceFile="%(TestApk.ResultsPath)"
236+
SourceFile="%(TestApkInstrumentation.ResultsPath)"
231237
/>
232238
</Target>
233239
<Target Name="RecordApkSizes"

‎src/Mono.Android/Test/Mono.Android-Tests.projitems‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
33
<PropertyGroup>
44
<_MonoAndroidTestPackage>Mono.Android_Tests</_MonoAndroidTestPackage>
5+
<_PackageName>Mono.Android_Tests</_PackageName>
56
</PropertyGroup>
67
<ItemGroup>
78
<TestApk Include="$(OutputPath)Mono.Android_Tests-Signed.apk">
@@ -15,4 +16,11 @@
1516
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Mono.Android_Tests-values.csv</ApkSizesResultsFilename>
1617
</TestApk>
1718
</ItemGroup>
19+
20+
<ItemGroup>
21+
<TestApkInstrumentation Include="xamarin.android.runtimetests.TestInstrumentation">
22+
<Package>Mono.Android_Tests</Package>
23+
<ResultsPath>$(OutputPath)TestResult-Mono.Android_Tests.xml</ResultsPath>
24+
</TestApkInstrumentation>
25+
</ItemGroup>
1826
</Project>

‎tests/BCL-Tests/LocalTests.NUnit/LocalTests.NUnit.csproj‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<PropertyGroup>
4+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6+
<ProjectGuid>{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}</ProjectGuid>
7+
<ProjectTypeGuids>{EFBA0AD7-5A72-4C68-AF49-83D382785DCF};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
8+
<OutputType>Library</OutputType>
9+
<RootNamespace>LocalTests.NUnit</RootNamespace>
10+
<AssemblyName>LocalTests.NUnit</AssemblyName>
11+
<TargetFrameworkVersion>v8.1</TargetFrameworkVersion>
12+
<AndroidResgenFile>Resources\Resource.designer.cs</AndroidResgenFile>
13+
<AndroidResgenClass>Resource</AndroidResgenClass>
14+
<MonoAndroidResourcePrefix>Resources</MonoAndroidResourcePrefix>
15+
<MonoAndroidAssetsPrefix>Assets</MonoAndroidAssetsPrefix>
16+
<AndroidUseLatestPlatformSdk>true</AndroidUseLatestPlatformSdk>
17+
</PropertyGroup>
18+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
19+
<DebugSymbols>true</DebugSymbols>
20+
<DebugType>full</DebugType>
21+
<Optimize>false</Optimize>
22+
<OutputPath>bin\Debug</OutputPath>
23+
<DefineConstants>DEBUG;</DefineConstants>
24+
<ErrorReport>prompt</ErrorReport>
25+
<WarningLevel>4</WarningLevel>
26+
<AndroidLinkMode>None</AndroidLinkMode>
27+
</PropertyGroup>
28+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
29+
<DebugSymbols>true</DebugSymbols>
30+
<DebugType>pdbonly</DebugType>
31+
<Optimize>true</Optimize>
32+
<OutputPath>bin\Release</OutputPath>
33+
<ErrorReport>prompt</ErrorReport>
34+
<WarningLevel>4</WarningLevel>
35+
<AndroidManagedSymbols>true</AndroidManagedSymbols>
36+
<AndroidUseSharedRuntime>false</AndroidUseSharedRuntime>
37+
</PropertyGroup>
38+
<ItemGroup>
39+
<Reference Include="System" />
40+
<Reference Include="System.Xml" />
41+
<Reference Include="System.Core" />
42+
<Reference Include="Mono.Android" />
43+
<Reference Include="Xamarin.Android.NUnitLite" />
44+
<Reference Include="System.Net" />
45+
<Reference Include="System.Net.Http" />
46+
</ItemGroup>
47+
<ItemGroup>
48+
<Compile Include="Resources\Resource.designer.cs" />
49+
<Compile Include="Properties\AssemblyInfo.cs" />
50+
<Compile Include="System.Net\NetworkChangeTest.cs" />
51+
<Compile Include="System.Net.Http\HttpClientTest.cs" />
52+
</ItemGroup>
53+
<ItemGroup>
54+
<None Include="Resources\AboutResources.txt" />
55+
</ItemGroup>
56+
<ItemGroup>
57+
<AndroidResource Include="Resources\values\Strings.xml" />
58+
</ItemGroup>
59+
<Import Project="$(MSBuildExtensionsPath)\Xamarin\Android\Xamarin.Android.CSharp.targets" />
60+
</Project>

‎tests/Xamarin.Android.Bcl-Tests/Properties/AssemblyInfo.cs‎ renamed to ‎tests/BCL-Tests/LocalTests.NUnit/Properties/AssemblyInfo.cs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
using System.Reflection;
22
using System.Runtime.CompilerServices;
3-
using Android.App;
43

54
// Information about this assembly is defined by the following attributes.
65
// Change them to the values specific to your project.
76

8-
[assembly: AssemblyTitle ("BCL")]
7+
[assembly: AssemblyTitle ("LocalTests.NUnit")]
98
[assembly: AssemblyDescription ("")]
109
[assembly: AssemblyConfiguration ("")]
11-
[assembly: AssemblyCompany ("Microsoft Corporation")]
10+
[assembly: AssemblyCompany ("")]
1211
[assembly: AssemblyProduct ("")]
13-
[assembly: AssemblyCopyright ("")]
12+
[assembly: AssemblyCopyright ("Microsoft, Inc")]
1413
[assembly: AssemblyTrademark ("")]
1514
[assembly: AssemblyCulture ("")]
1615

‎tests/Xamarin.Android.Bcl-Tests/Resources/AboutResources.txt‎ renamed to ‎tests/BCL-Tests/LocalTests.NUnit/Resources/AboutResources.txt‎

File renamed without changes.

‎tests/BCL-Tests/LocalTests.NUnit/Resources/values/Strings.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<resources>
3+
<string name="library_name">LocalTests.NUnit</string>
4+
</resources>

‎tests/Xamarin.Android.Bcl-Tests/System.Net.Http/HttpClientTest.cs‎ renamed to ‎tests/BCL-Tests/LocalTests.NUnit/System.Net.Http/HttpClientTest.cs‎

File renamed without changes.

‎tests/Xamarin.Android.Bcl-Tests/System.Net/NetworkChangeTest.cs‎ renamed to ‎tests/BCL-Tests/LocalTests.NUnit/System.Net/NetworkChangeTest.cs‎

File renamed without changes.

0 commit comments

Comments
 (0)