Skip to content

Commit e8b9ee2

Browse files
radekdoulikjonpryor
radekdoulik
authored and
jonpryor
committed
[tests] Check apk tests sizes to avoid regressions (#3594)
Fixes: #3593 Added new `<ProcessApkSizes/>` task based on the `<ProcessPlotInput/>` task, which takes additional argument with apk sizes reference file. The file is used to check the current apk sizes and fails when the size difference is over a given threshold; we use 100KiB now. There's one additional threshold to warn about smalled differences, with current threshold 50KiB. The reference files can be update with > make update-apk-sizes-reference CONFIGURATION=<Debug or Release> which copies the current apk sizes files to `tests/apk-sizes-reference` directory, these are then used by `<ProcessApkSizes/>` task. The apk sizes files were split by configuration and contain `$(Configuration)` in the name. The reason for that is an easier update of the references after failed PR build, after the size difference is checked to be valid and desired. This way it is possible to just download the results artifacts file and copy the sizes recorded by CI build into the reference directory.
1 parent 2d400aa commit e8b9ee2

13 files changed

+127
-5
lines changed

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,3 +209,9 @@ prepare-update-mono: prepare-build
209209

210210
prepare-external-git-dependencies: prepare-build
211211
mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) -s:PrepareExternalGitDependencies
212+
213+
APK_SIZES_REFERENCE_DIR=tests/apk-sizes-reference
214+
215+
update-apk-sizes-reference:
216+
-mkdir -p $(APK_SIZES_REFERENCE_DIR)
217+
cp -v *values-$(CONFIGURATION).csv $(APK_SIZES_REFERENCE_DIR)/

‎build-tools/scripts/TestApks.targets‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@
1111
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.KillProcess" />
1212
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.Sleep" />
1313
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.ProcessLogcatTiming" />
14-
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.ProcessPlotInput" />
14+
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.ProcessApkSizes" />
1515

1616
<PropertyGroup>
1717
<_TestImageName>XamarinAndroidTestRunner</_TestImageName>
1818
<_AdbEmulatorPort>5570</_AdbEmulatorPort>
19+
<_ApkSizesReferenceDirectory>$(MSBuildThisFileDirectory)..\..\tests\apk-sizes-reference</_ApkSizesReferenceDirectory>
1920
</PropertyGroup>
2021

2122
<ItemGroup>
@@ -334,11 +335,12 @@
334335
Condition=" '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
335336
Command="unzip -l &quot;%(_AllArchives.Identity)&quot; >> &quot;$(OutputPath)%(_AllArchives.ApkSizesInputFilename)&quot;"
336337
ContinueOnError="ErrorAndContinue" />
337-
<ProcessPlotInput
338+
<ProcessApkSizes
338339
Condition=" '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
339340
InputFilename="$(OutputPath)%(_AllArchives.ApkSizesInputFilename)"
340341
ApplicationPackageName="%(_AllArchives.Package)"
341342
ResultsFilename="%(_AllArchives.ApkSizesResultsFilename)"
343+
ReferenceFilename="$(_ApkSizesReferenceDirectory)\$([System.IO.Path]::GetFileName(%(_AllArchives.ApkSizesResultsFilename)))"
342344
DefinitionsFilename="%(_AllArchives.ApkSizesDefinitionFilename)"
343345
AddResults="True"
344346
LabelSuffix="-$(Configuration)$(TestsFlavor)"

‎build-tools/xa-prep-tasks/Xamarin.Android.BuildTools.PrepTasks/ProcessApkSizes.cs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
5+
namespace Xamarin.Android.BuildTools.PrepTasks
6+
{
7+
public class ProcessApkSizes : ProcessPlotInput
8+
{
9+
public string ReferenceFilename { get; set; }
10+
11+
Dictionary<string, string> referenceResults = new Dictionary<string, string> ();
12+
13+
public override bool Execute ()
14+
{
15+
if (!base.Execute ())
16+
return false;
17+
18+
19+
if (string.IsNullOrEmpty (ReferenceFilename) || !File.Exists (ReferenceFilename))
20+
return true;
21+
22+
if (!ReadReference ()) {
23+
Log.LogError ($"unexpected reference results file {ReferenceFilename}");
24+
25+
return false;
26+
}
27+
28+
return CheckSizes ();
29+
}
30+
31+
bool ReadReference ()
32+
{
33+
string line1 = null, line2 = null;
34+
using (var reader = new StreamReader (ReferenceFilename)) {
35+
try {
36+
line1 = reader.ReadLine ();
37+
line2 = reader.ReadLine ();
38+
} catch (Exception e) {
39+
Log.LogError ($"unable to read reference results from {ReferenceFilename}\n{e}");
40+
41+
return false;
42+
}
43+
}
44+
45+
if (string.IsNullOrEmpty (line1) || string.IsNullOrEmpty (line2))
46+
return false;
47+
48+
var keys = line1.Split (new char [] { ',' });
49+
var values = line2.Split (new char [] { ',' });
50+
51+
if (keys.Length != values.Length || keys.Length < 1)
52+
return false;
53+
54+
for (int i = 0; i < keys.Length; i++)
55+
referenceResults [keys [i]] = values [i];
56+
57+
return true;
58+
}
59+
60+
string SizesMessage (int size, int refSize, int difference, string key)
61+
{
62+
return $"the reference {refSize} and actual size {size} for {key} differs too much (>{difference}kb).\nplease check if it is correct and update the reference apk sizes files if it is.\nit can be done by running `make CONFIGURATION=<Debug or Release> update-apk-sizes-reference` in XA root directory";
63+
}
64+
65+
const int errorThreshold = 100;
66+
const int warningThreshold = 50;
67+
68+
bool CheckSizes ()
69+
{
70+
foreach (var size in results) {
71+
var key = size.Key + LabelSuffix;
72+
var refSizeString = referenceResults [key];
73+
74+
int refSize, resSize;
75+
76+
if (!int.TryParse (refSizeString, out refSize)) {
77+
Log.LogError ($"unable to parse size of {key} in reference file {ReferenceFilename}");
78+
79+
return false;
80+
}
81+
82+
if (!int.TryParse (size.Value, out resSize)) {
83+
Log.LogError ($"unable to parse size of {key} in current results");
84+
85+
return false;
86+
}
87+
88+
if (Math.Abs (resSize - refSize) > errorThreshold * 1024) {
89+
Log.LogError (SizesMessage (resSize, refSize, errorThreshold, key));
90+
91+
return false;
92+
}
93+
94+
if (Math.Abs (resSize - refSize) > warningThreshold * 1024)
95+
Log.LogWarning (SizesMessage (resSize, refSize, warningThreshold, key));
96+
}
97+
98+
return true;
99+
}
100+
}
101+
}

‎build-tools/xa-prep-tasks/xa-prep-tasks.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<Compile Include="Xamarin.Android.BuildTools.PrepTasks\NDKInfo.cs" />
6363
<Compile Include="Xamarin.Android.BuildTools.PrepTasks\ErrorResultsHelper.cs" />
6464
<Compile Include="Xamarin.Android.BuildTools.PrepTasks\CheckoutExternalGitDependency.cs" />
65+
<Compile Include="Xamarin.Android.BuildTools.PrepTasks\ProcessApkSizes.cs" />
6566
</ItemGroup>
6667
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
6768
<Import Project="xa-prep-tasks.targets" />

‎src/Mono.Android/Test/Mono.Android-Tests.projitems‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
<TimingResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Mono.Android_Tests-times.csv</TimingResultsFilename>
1414
<ApkSizesInputFilename>apk-sizes-$(_MonoAndroidTestPackage)-$(Configuration)$(TestsFlavor).txt</ApkSizesInputFilename>
1515
<ApkSizesDefinitionFilename>$(MSBuildThisFileDirectory)..\..\..\build-tools\scripts\ApkSizesDefinitions.txt</ApkSizesDefinitionFilename>
16-
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Mono.Android_Tests-values.csv</ApkSizesResultsFilename>
16+
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Mono.Android_Tests-values-$(Configuration).csv</ApkSizesResultsFilename>
1717
</TestApk>
1818
</ItemGroup>
1919

‎tests/Runtime-MultiDex/Mono.Android-TestsMultiDex.projitems‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
<TimingResultsFilename>$(MSBuildThisFileDirectory)..\..\TestResult-$(_PackageName)-times.csv</TimingResultsFilename>
1313
<ApkSizesInputFilename>apk-sizes-$(_PackageName)-$(Configuration)$(TestsFlavor).txt</ApkSizesInputFilename>
1414
<ApkSizesDefinitionFilename>$(MSBuildThisFileDirectory)apk-sizes-definitions.txt</ApkSizesDefinitionFilename>
15-
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\TestResult-$(_PackageName)-values.csv</ApkSizesResultsFilename>
15+
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\TestResult-$(_PackageName)-values-$(Configuration).csv</ApkSizesResultsFilename>
1616
</TestApk>
1717
</ItemGroup>
1818

‎tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.projitems‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
<TimingResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Xamarin.Forms_Test-times.csv</TimingResultsFilename>
1313
<ApkSizesInputFilename>apk-sizes-$(_XamarinFormsIntergationPackage)-$(Configuration)$(TestsFlavor).txt</ApkSizesInputFilename>
1414
<ApkSizesDefinitionFilename>$(MSBuildThisFileDirectory)..\..\..\build-tools\scripts\ApkSizesDefinitions.txt</ApkSizesDefinitionFilename>
15-
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Xamarin.Forms_Tests-values.csv</ApkSizesResultsFilename>
15+
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Xamarin.Forms_Tests-values-$(Configuration).csv</ApkSizesResultsFilename>
1616
</TestApk>
1717
</ItemGroup>
1818
</Project>

‎tests/apk-sizes-reference/TestResult-Mono.Android_Tests-values-Debug.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
apk-Debug,Java.Interop.dll-Debug,Mono.Android.dll-Debug,mscorlib.dll-Debug,monosgen-armeabi-v7a-Debug,apk-Debug-Aot,Java.Interop.dll-Debug-Aot,Mono.Android.dll-Debug-Aot,mscorlib.dll-Debug-Aot,monosgen-armeabi-v7a-Debug-Aot,Java.Interop.dll.so-Debug-Aot,mscorlib.dll.so-Debug-Aot,Mono.Android.dll.so-Debug-Aot,apk-Debug-Profiled-Aot,Java.Interop.dll-Debug-Profiled-Aot,Mono.Android.dll-Debug-Profiled-Aot,mscorlib.dll-Debug-Profiled-Aot,monosgen-armeabi-v7a-Debug-Profiled-Aot,Mono.Android.dll.so-Debug-Profiled-Aot,Java.Interop.dll.so-Debug-Profiled-Aot,mscorlib.dll.so-Debug-Profiled-Aot,apk-Debug-Bundle,monosgen-armeabi-v7a-Debug-Bundle
2+
74038851,216064,28933632,4487680,5500576,131375051,216064,28933632,4487680,5500576,1066756,16432748,92736548,79474635,216064,28933632,4487680,5500576,3169540,241804,3471724,32317290,5500576

‎tests/apk-sizes-reference/TestResult-Mono.Android_Tests-values-Release.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
apk-Release,Java.Interop.dll-Release,Mono.Android.dll-Release,mscorlib.dll-Release,monosgen-armeabi-v7a-Release,apk-Release-Aot,Java.Interop.dll-Release-Aot,Mono.Android.dll-Release-Aot,mscorlib.dll-Release-Aot,monosgen-armeabi-v7a-Release-Aot,Java.Interop.dll.so-Release-Aot,Mono.Android.dll.so-Release-Aot,mscorlib.dll.so-Release-Aot,apk-Release-Profiled-Aot,Java.Interop.dll-Release-Profiled-Aot,Mono.Android.dll-Release-Profiled-Aot,mscorlib.dll-Release-Profiled-Aot,monosgen-armeabi-v7a-Release-Profiled-Aot,Mono.Android.dll.so-Release-Profiled-Aot,mscorlib.dll.so-Release-Profiled-Aot,Java.Interop.dll.so-Release-Profiled-Aot,apk-Release-Bundle,monosgen-armeabi-v7a-Release-Bundle
2+
16540799,161792,1400832,2112000,3843248,32985861,161792,1400832,2112000,3843248,842432,5908196,8641772,18506501,161792,1400832,2112000,3843248,421036,1685856,220904,14229016,3843248

‎tests/apk-sizes-reference/TestResult-Mono.Android_TestsMultiDex-values-Debug.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
apk-Debug,Mono.Android.dll-Debug,mscorlib.dll-Debug,monosgen-armeabi-v7a-Debug,apk-Debug-Aot,Mono.Android.dll-Debug-Aot,mscorlib.dll-Debug-Aot,monosgen-armeabi-v7a-Debug-Aot,apk-Debug-Profiled-Aot,Mono.Android.dll-Debug-Profiled-Aot,mscorlib.dll-Debug-Profiled-Aot,monosgen-armeabi-v7a-Debug-Profiled-Aot,apk-Debug-Bundle,Mono.Android.dll-Debug-Bundle,mscorlib.dll-Debug-Bundle,monosgen-armeabi-v7a-Debug-Bundle
2+
74116842,28933632,4487680,5500576,74116842,28933632,4487680,5500576,74116842,28933632,4487680,5500576,74116842,28933632,4487680,5500576

0 commit comments

Comments
 (0)