Skip to content

Commit 09ba1ee

Browse files
radekdoulikjonpryor
radekdoulik
authored and
jonpryor
committed
[tests] Xamarin.Forms-Performance-Integration test (#818)
Add `tests/Xamarin.Forms-Performance-Integration` to measure UI app startup times. Add a new `<RunUITests/>` task to start an app/activity. Rename `@(UnitTestApk)` to `@(TestApk)`, as it also handles UI tests. Update `<ProcessLogcatTiming/>` to be able to process the `adb logcat` output of started Android Activity processes.
1 parent 0340725 commit 09ba1ee

File tree

53 files changed

+1136
-47
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+1136
-47
lines changed

‎Documentation/DevelopmentTips.md‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -148,39 +148,39 @@ make variable:
148148
## Running Individual `.apk` Projects
149149

150150
See also the [`tests/RunApkTests.targets`](../tests/RunApkTests.targets) and
151-
[`build-tools/scripts/UnitTestApks.targets`](../build-tools/scripts/UnitTestApks.targets)
151+
[`build-tools/scripts/TestApks.targets`](../build-tools/scripts/TestApks.targets)
152152
files.
153153

154154
All `.apk`-based unit test projects provide the following targets:
155155

156-
* `DeployUnitTestApks`: Installs the associated `.apk` to an Android device.
157-
* `UndeployUnitTestApks`: Uninstalls the associated `.apk` from an Android device.
158-
* `RunUnitTestApks`: Executes the unit tests contained within a `.apk`.
159-
Must be executed *after* the `DeployUnitTestApks` target.
156+
* `DeployTestApks`: Installs the associated `.apk` to an Android device.
157+
* `UndeployTestApks`: Uninstalls the associated `.apk` from an Android device.
158+
* `RunTestApks`: Executes the unit tests contained within a `.apk`.
159+
Must be executed *after* the `DeployTestApks` target.
160160

161161
To run an individual `.apk`-based test project, a package must be built, using the
162162
`SignAndroidPackage` target, installed, and executed.
163163

164164
For example:
165165

166166
$ tools/scripts/xabuild /t:SignAndroidPackage tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
167-
$ tools/scripts/xabuild /t:DeployUnitTestApks tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
168-
$ tools/scripts/xabuild /t:RunUnitTestApks tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
167+
$ tools/scripts/xabuild /t:DeployTestApks tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
168+
$ tools/scripts/xabuild /t:RunTestApks tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
169169

170170
### Running A Single Test Fixture
171171

172172
A single NUnit *Test Fixture* -- a class with the `[TestFixture]`
173173
custom attribute -- may be executed instead of executing *all* test fixtures.
174174

175-
The `RunUnitTestApks` target accepts a `TestFixture` MSBuild property
175+
The `RunTestApks` target accepts a `TestFixture` MSBuild property
176176
to specify the test fixture class to execute:
177177

178-
$ tools/scripts/xabuild /t:RunUnitTestApks \
178+
$ tools/scripts/xabuild /t:RunTestApks \
179179
/p:TestFixture=Xamarin.Android.LocaleTests.SatelliteAssemblyTests \
180180
tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
181181

182182
If using `Xamarin.Android.NUnitLite` for projects outside the `xamarin-android`
183-
repository, such as NUnit tests for a custom app, the `RunUnitTestApks` target
183+
repository, such as NUnit tests for a custom app, the `RunTestApks` target
184184
will not exist. In such scenarios, the [`adb shell am`][adb-shell-am]
185185
`instrument` command can be used instead. It follows the format:
186186

‎Xamarin.Android-Tests.sln‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.BindingReso
4545
EndProject
4646
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.BindingResolveImports-Tests", "tests\ResolveImports\Xamarin.Android.BindingResolveImports-Tests\Xamarin.Android.BindingResolveImports-Tests.csproj", "{B297008B-C313-455E-B230-E119589D2D79}"
4747
EndProject
48+
Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "Xamarin.Forms.Performance.Integration", "tests\Xamarin.Forms-Performance-Integration\Xamarin.Forms.Performance.Integration.shproj", "{195BE9C2-1F91-40DC-BD6D-DE860BF083FB}"
49+
EndProject
50+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Forms.Performance.Integration.Droid", "tests\Xamarin.Forms-Performance-Integration\Droid\Xamarin.Forms.Performance.Integration.Droid.csproj", "{576312CC-83FF-48B1-A473-488CDC7121AD}"
51+
EndProject
4852
Global
4953
GlobalSection(SolutionConfigurationPlatforms) = preSolution
5054
Debug|Any CPU = Debug|Any CPU
@@ -115,6 +119,10 @@ Global
115119
{B297008B-C313-455E-B230-E119589D2D79}.Debug|Any CPU.Build.0 = Debug|Any CPU
116120
{B297008B-C313-455E-B230-E119589D2D79}.Release|Any CPU.ActiveCfg = Release|Any CPU
117121
{B297008B-C313-455E-B230-E119589D2D79}.Release|Any CPU.Build.0 = Release|Any CPU
122+
{576312CC-83FF-48B1-A473-488CDC7121AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
123+
{576312CC-83FF-48B1-A473-488CDC7121AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
124+
{576312CC-83FF-48B1-A473-488CDC7121AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
125+
{576312CC-83FF-48B1-A473-488CDC7121AD}.Release|Any CPU.Build.0 = Release|Any CPU
118126
EndGlobalSection
119127
GlobalSection(NestedProjects) = preSolution
120128
{2305B00D-DE81-4744-B0DA-357835CAFE5A} = {43A4FB09-279A-4138-8027-EC1E1CED2E8A}

‎build-tools/scripts/UnitTestApks.targets‎ renamed to ‎build-tools/scripts/TestApks.targets‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@
44
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget" />
55
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.CreateAndroidEmulator" />
66
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.RunInstrumentationTests" />
7+
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.RunUITests" />
78
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.StartAndroidEmulator" />
89
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.KillProcess" />
910
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.Sleep" />
1011
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.ProcessLogcatTiming" />
1112

1213
<PropertyGroup>
13-
<_TestImageName>XamarinAndroidUnitTestRunner</_TestImageName>
14+
<_TestImageName>XamarinAndroidTestRunner</_TestImageName>
1415
<_AdbEmulatorPort>5570</_AdbEmulatorPort>
1516
</PropertyGroup>
1617

@@ -107,54 +108,62 @@
107108

108109
<!--
109110
<ItemGroup>
110-
<UnitTestApk Include="ApkFile">
111+
<TestApk Include="ApkFile">
111112
<Package></Package>
112113
<InstrumentationType></InstrumentationType>
113114
<ResultsPath></ResultsPath>
114-
</UnitTestApk>
115+
</TestApk>
115116
</ItemGroup>
116117
-->
117118

118-
<Target Name="DeployUnitTestApks"
119-
Condition=" '@(UnitTestApk)' != '' ">
119+
<Target Name="DeployTestApks"
120+
Condition=" '@(TestApk)' != '' ">
120121
<Xamarin.Android.Tools.BootstrapTasks.Adb
121-
Arguments="$(_AdbTarget) $(AdbOptions) install &quot;%(UnitTestApk.Identity)&quot;"
122+
Arguments="$(_AdbTarget) $(AdbOptions) install &quot;%(TestApk.Identity)&quot;"
122123
ToolExe="$(AdbToolExe)"
123124
ToolPath="$(AdbToolPath)"
124125
/>
125126
</Target>
126127

127-
<Target Name="UndeployUnitTestApks"
128-
Condition=" '@(UnitTestApk)' != '' ">
128+
<Target Name="UndeployTestApks"
129+
Condition=" '@(TestApk)' != '' ">
129130
<Xamarin.Android.Tools.BootstrapTasks.Adb
130-
Arguments="$(_AdbTarget) $(AdbOptions) uninstall &quot;%(UnitTestApk.Package)&quot;"
131+
Arguments="$(_AdbTarget) $(AdbOptions) uninstall &quot;%(TestApk.Package)&quot;"
131132
ToolExe="$(AdbToolExe)"
132133
ToolPath="$(AdbToolPath)"
133134
/>
134135
</Target>
135136

136-
<Target Name="RunUnitTestApks"
137-
Condition=" '@(UnitTestApk)' != '' ">
137+
<Target Name="RunTestApks"
138+
Condition=" '@(TestApk)' != '' ">
138139
<RunInstrumentationTests
140+
Condition=" '%(TestApk.InstrumentationType)' != ''"
139141
AdbTarget="$(_AdbTarget)"
140142
AdbOptions="$(AdbOptions)"
141-
Component="%(UnitTestApk.Package)/%(UnitTestApk.InstrumentationType)"
142-
NUnit2TestResultsFile="%(UnitTestApk.ResultsPath)"
143+
Component="%(TestApk.Package)/%(TestApk.InstrumentationType)"
144+
NUnit2TestResultsFile="%(TestApk.ResultsPath)"
143145
TestFixture="$(TestFixture)"
144146
ToolExe="$(AdbToolExe)"
145147
ToolPath="$(AdbToolPath)">
146148
<Output TaskParameter="FailedToRun" ItemName="_FailedComponent"/>
147149
</RunInstrumentationTests>
150+
<RunUITests
151+
Condition=" '%(TestApk.Activity)' != '' "
152+
AdbTarget="$(_AdbTarget)"
153+
AdbOptions="$(AdbOptions)"
154+
Activity="%(TestApk.Activity)">
155+
</RunUITests>
148156
<PropertyGroup>
149157
<_LogcatFilename>logcat-$(Configuration)$(_AotName).txt</_LogcatFilename>
150158
</PropertyGroup>
151159
<Exec Command="&quot;$(AdbToolPath)\$(AdbToolExe)&quot; $(_AdbTarget) $(AdbOptions) logcat -v threadtime -d > $(_LogcatFilename)" />
152160
<ProcessLogcatTiming
153-
Condition=" '%(UnitTestApk.TimingDefinitionsFilename)' != '' "
161+
Condition=" '%(TestApk.TimingDefinitionsFilename)' != '' "
154162
InputFilename="$(_LogcatFilename)"
155-
ApplicationPackageName="%(UnitTestApk.Package)"
156-
ResultsFilename="%(UnitTestApk.ResultsPath)"
157-
DefinitionsFilename="%(UnitTestApk.TimingDefinitionsFilename)"
158-
AddResults="true" />
163+
ApplicationPackageName="%(TestApk.Package)"
164+
ResultsFilename="%(TestApk.ResultsPath)"
165+
DefinitionsFilename="%(TestApk.TimingDefinitionsFilename)"
166+
AddResults="true"
167+
Activity="%(TestApk.Activity)" />
159168
</Target>
160169
</Project>

‎build-tools/xa-prep-tasks/Xamarin.Android.BuildTools.PrepTasks/ProcessLogcatTiming.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@ namespace Xamarin.Android.BuildTools.PrepTasks
1010
{
1111
public class ProcessLogcatTiming : ProcessPlotInput
1212
{
13+
public string Activity { get; set; }
14+
1315
public override bool Execute ()
1416
{
1517
LoadDefinitions ();
1618
using (var reader = new StreamReader (InputFilename)) {
1719
string line;
1820
int pid = -1;
19-
var procStartRegex = new Regex ($@"^(?<timestamp>\d+-\d+\s+[\d:\.]+)\s+.*ActivityManager: Start proc.*for added application {ApplicationPackageName}: pid=(?<pid>\d+)");
21+
var procIdentification = string.IsNullOrEmpty (Activity) ? $"added application {ApplicationPackageName}" : $"activity {Activity}";
22+
var procStartRegex = new Regex ($@"^(?<timestamp>\d+-\d+\s+[\d:\.]+)\s+.*ActivityManager: Start proc.*for {procIdentification}: pid=(?<pid>\d+)");
2023
Regex timingRegex = null;
2124
DateTime start = DateTime.Now;
2225
DateTime last = start;

‎src/Mono.Android/Test/Mono.Android-Tests.projitems‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@
88
<_MonoAndroidTestApkSizesInput>apk-sizes-$(_MonoAndroidTestPackage)-$(Configuration)$(_AotName).txt</_MonoAndroidTestApkSizesInput>
99
</PropertyGroup>
1010
<ItemGroup>
11-
<UnitTestApk Include="$(_MonoAndroidTestApkFile)">
11+
<TestApk Include="$(_MonoAndroidTestApkFile)">
1212
<Package>$(_MonoAndroidTestPackage)</Package>
1313
<InstrumentationType>xamarin.android.runtimetests.TestInstrumentation</InstrumentationType>
1414
<ResultsPath>$(_MonoAndroidTestResultsPath)</ResultsPath>
1515
<TimingDefinitionsFilename>$(MSBuildThisFileDirectory)timing-definitions-$(Configuration)$(_AotName).txt</TimingDefinitionsFilename>
16-
</UnitTestApk>
16+
</TestApk>
1717
</ItemGroup>
18-
<Target Name="_RecordApkSizes" AfterTargets="DeployUnitTestApks">
18+
<Target Name="_RecordApkSizes" AfterTargets="DeployTestApks">
1919
<Delete Files="$(MSBuildThisFileDirectory)..\..\..\TestResult-Mono.Android_Tests-values.csv;$(MSBuildThisFileDirectory)..\..\..\TestResult-Mono.Android_Tests-times.csv" Condition=" '$(Configuration)' == 'Debug' " />
2020
<Exec Condition=" '$(HostOS)' == 'Darwin' " Command="stat -f &quot;stat: %z %N&quot; &quot;$(_MonoAndroidTestApkFile)&quot; > $(_MonoAndroidTestApkSizesInput)" />
2121
<Exec Condition=" '$(HostOS)' == 'Linux' " Command="stat -c &quot;stat: %s %N&quot; &quot;$(_MonoAndroidTestApkFile)&quot; > $(_MonoAndroidTestApkSizesInput)" />

‎src/Mono.Android/Test/Mono.Android-Tests.targets‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
<AndroidNativeLibrary Include="libs\x86_64\libreuse-threads.so" />
99
</ItemGroup>
1010
<Import Project="Mono.Android-Tests.projitems" />
11-
<Import Project="..\..\..\build-tools\scripts\UnitTestApks.targets" />
11+
<Import Project="..\..\..\build-tools\scripts\TestApks.targets" />
1212
<Target Name="BuildNativeLibs"
1313
BeforeTargets="Build"
1414
DependsOnTargets="AndroidPrepareForBuild"

‎src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,12 @@
4646
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\GenerateMonoDroidIncludes.cs" />
4747
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\GenerateProfile.cs" />
4848
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\GetNugetPackageBasePath.cs" />
49-
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\RunInstrumentationTests.cs" />
5049
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\StartAndroidEmulator.cs" />
5150
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\UnzipDirectoryChildren.cs" />
5251
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\Zip.cs" />
5352
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\KillProcess.cs" />
53+
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\RunUITests.cs" />
54+
<Compile Include="Xamarin.Android.Tools.BootstrapTasks\RunInstrumentationTests.cs" />
5455
</ItemGroup>
5556
<ItemGroup>
5657
<ProjectReference Include="..\..\external\LibZipSharp\libZipSharp.csproj">

‎src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/CreateAndroidEmulator.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public class CreateAndroidEmulator : Task
2121

2222
public string TargetId {get; set;}
2323

24-
public string ImageName {get; set;} = "XamarinAndroidUnitTestRunner";
24+
public string ImageName {get; set;} = "XamarinAndroidTestRunner";
2525

2626
public override bool Execute ()
2727
{

‎src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunUITests.cs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
using Microsoft.Build.Framework;
2+
3+
namespace Xamarin.Android.Tools.BootstrapTasks
4+
{
5+
public class RunUITests : Adb
6+
{
7+
public string AdbTarget { get; set; }
8+
public string AdbOptions { get; set; }
9+
10+
[Required]
11+
public string Activity { get; set; }
12+
13+
protected override bool LogTaskMessages {
14+
get { return false; }
15+
}
16+
17+
public override bool Execute ()
18+
{
19+
Log.LogMessage (MessageImportance.Low, $"Task {nameof (RunUITests)}");
20+
Log.LogMessage (MessageImportance.Low, $" {nameof (AdbTarget)}: {AdbTarget}");
21+
Log.LogMessage (MessageImportance.Low, $" {nameof (AdbOptions)}: {AdbOptions}");
22+
Log.LogMessage (MessageImportance.Low, $" {nameof (Activity)}: {Activity}");
23+
24+
base.Execute ();
25+
26+
Log.LogMessage(MessageImportance.Low, $" going to wait for 15 seconds");
27+
System.Threading.Thread.Sleep (15000);
28+
29+
return !Log.HasLoggedErrors;
30+
}
31+
32+
protected override string GenerateCommandLineCommands ()
33+
{
34+
return $"{AdbTarget} {AdbOptions} shell am start -n \"{Activity}\"";
35+
}
36+
}
37+
}

‎src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class StartAndroidEmulator : Task
1919

2020
public string AndroidSdkHome {get; set;}
2121
public string Port {get; set;}
22-
public string ImageName {get; set;} = "XamarinAndroidUnitTestRunner";
22+
public string ImageName {get; set;} = "XamarinAndroidTestRunner";
2323
public string ToolPath {get; set;}
2424
public string ToolExe {get; set;}
2525

0 commit comments

Comments
 (0)