Skip to content

Commit 75dc590

Browse files
radekdoulikradekdoulik
authored
[tests] apk size regression testing (#4545)
Use `apkdiff` (2e28f2e) and `.apkdesc` description files to check for `.apk` size regressions. This allows for more informative and useful messages when `.apk` size regressions occur, e.g. Saving apk description to '../../../bin/TestRelease/Xamarin.Forms_Performance_Integration-Signed.apkdesc' Size difference in bytes ([*1] apk1 only, [*2] apk2 only): 100,000 assemblies/Mono.Android.dll Error: apkdiff: Assembly 'assemblies/Mono.Android.dll' size increase 100,000 is 48,800 bytes more than the threshold 51,200. 492 lib/armeabi-v7a/libmonodroid.so 216 lib/x86/libmonodroid.so Summary: 100,000 Assemblies 0.71% (of 14,080,480) 708 Shared libraries 0.01% (of 11,855,444) 0 Package size difference 0.00% (of 21,007,581) Error: apkdiff: Size regression occured, 1 check(s) failed. The new `<ApkDiffCheckRegression/>` task handles running the `apkdiff` tool and creates `TestResult*.xml` files for convenient CI reporting of `.apk` size regressions. Add `tests/apk-sizes-reference/*.apkdesc` files to contain "expected sizes" for verification.
1 parent f0afa43 commit 75dc590

File tree

10 files changed

+13357
-34
lines changed

10 files changed

+13357
-34
lines changed

‎build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/ApkDiffCheckRegression.cs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
using System;
2+
using System.IO;
3+
using System.Text;
4+
using Microsoft.Build.Framework;
5+
using Microsoft.Build.Utilities;
6+
using Xamarin.Android.BuildTools.PrepTasks;
7+
8+
9+
namespace Xamarin.Android.Tools.BootstrapTasks
10+
{
11+
public class ApkDiffCheckRegression : ToolTask
12+
{
13+
[Required]
14+
public string ApkDiffTool { get; set; }
15+
16+
[Required]
17+
public string Package { get; set; }
18+
19+
[Required]
20+
public string ReferenceDescription { get; set; }
21+
22+
[Required]
23+
public string TestResultDirectory { get; set; }
24+
25+
protected override string ToolName => Path.GetFileName (ApkDiffTool);
26+
protected override string GenerateFullPathToTool () => ApkDiffTool;
27+
28+
const int ApkSizeThreshold = 50*1024;
29+
const int AssemblySizeThreshold = 50*1024;
30+
31+
StringBuilder logCopy = new StringBuilder ();
32+
33+
protected override string GenerateCommandLineCommands ()
34+
{
35+
var cmd = new CommandLineBuilder ();
36+
cmd.AppendSwitch ("-s");
37+
cmd.AppendSwitch ($"--test-apk-size-regression={ApkSizeThreshold}");
38+
cmd.AppendSwitch ($"--test-assembly-size-regression={AssemblySizeThreshold}");
39+
cmd.AppendFileNameIfNotNull (ReferenceDescription);
40+
cmd.AppendFileNameIfNotNull (Package);
41+
42+
return cmd.ToString ();
43+
}
44+
45+
protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance)
46+
{
47+
base.LogEventsFromTextOutput (singleLine, messageImportance);
48+
49+
logCopy.AppendLine (singleLine);
50+
}
51+
52+
public override bool Execute ()
53+
{
54+
LogStandardErrorAsError = false;
55+
StandardOutputImportance = "Low";
56+
57+
var succes = base.Execute ();
58+
if (ExitCode == 0)
59+
return succes;
60+
61+
var errorMessage = $"apkdiff exited with error code: {ExitCode}.";
62+
var testResultPath = Path.Combine (TestResultDirectory, $"TestResult-apkdiff-{Path.GetFileNameWithoutExtension (ReferenceDescription)}.xml");
63+
ErrorResultsHelper.CreateErrorResultsFile (
64+
testResultPath,
65+
nameof (ApkDiffCheckRegression),
66+
"check apk size regression",
67+
new Exception (errorMessage),
68+
$"apkdiff output:\n{logCopy}");
69+
70+
return false;
71+
}
72+
}
73+
}

‎build-tools/automation/azure-pipelines.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -640,6 +640,7 @@ stages:
640640
configuration: $(ApkTestConfiguration)
641641
testName: Xamarin.Forms-Performance-Integration
642642
project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
643+
testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration).xml
643644
artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
644645
artifactFolder: Default
645646

@@ -648,6 +649,7 @@ stages:
648649
configuration: $(ApkTestConfiguration)
649650
testName: Xamarin.Forms-Performance-Integration-Aot
650651
project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
652+
testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration)-Aot.xml
651653
extraBuildArgs: /p:AotAssemblies=true
652654
artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
653655
artifactFolder: Aot
@@ -657,6 +659,7 @@ stages:
657659
configuration: $(ApkTestConfiguration)
658660
testName: Xamarin.Forms-Performance-Integration-Profiled-Aot
659661
project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
662+
testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration)-Profiled-Aot.xml
660663
extraBuildArgs: /p:AotAssemblies=True /p:AndroidEnableProfiledAot=true
661664
artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
662665
artifactFolder: Profiled-Aot
@@ -666,6 +669,7 @@ stages:
666669
configuration: $(ApkTestConfiguration)
667670
testName: Xamarin.Forms-Performance-Integration-Bundle
668671
project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
672+
testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration)-Bundle.xml
669673
extraBuildArgs: /p:BundleAssemblies=true
670674
artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
671675
artifactFolder: Bundle

‎build-tools/automation/yaml-templates/apk-instrumentation.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ steps:
1818
configuration: ${{ parameters.configuration }}
1919
msbuildArguments: >
2020
/restore
21-
/t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,RecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
21+
/t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
2222
/bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run${{ parameters.testName }}.binlog
2323
${{ parameters.extraBuildArgs }}
2424
condition: ${{ parameters.condition }}

‎build-tools/installers/create-installers.targets‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,13 +116,19 @@
116116
<ItemGroup>
117117
<_MSBuildFiles Include="$(MSBuildSrcDir)\android-support-multidex.jar" />
118118
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\apkdiff.exe" />
119+
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\apkdiff.exe.config" />
119120
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\apkdiff.pdb" />
120121
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\Mono.Options.dll" />
121122
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\libZipSharp.dll" />
122123
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\libZipSharp.dll.config" />
123124
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\Newtonsoft.Json.dll" />
124125
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\Newtonsoft.Json-LICENSE.md" />
125126
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Buffers.dll" />
127+
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Collections.Immutable.dll" />
128+
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Memory.dll" />
129+
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Numerics.Vectors.dll" />
130+
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Reflection.Metadata.dll" />
131+
<_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Runtime.CompilerServices.Unsafe.dll" />
126132
<_MSBuildFiles Include="$(MSBuildSrcDir)\aprofutil.exe" />
127133
<_MSBuildFiles Include="$(MSBuildSrcDir)\aprofutil.pdb" />
128134
<_MSBuildFiles Include="$(MSBuildSrcDir)\cil-strip.exe" />

‎build-tools/scripts/TestApks.targets‎

Lines changed: 12 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
33
<UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.Adb" />
4+
<UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.ApkDiffCheckRegression" />
45
<UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.BundleTool" />
56
<UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget" />
67
<UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.CreateAndroidEmulator" />
@@ -320,40 +321,19 @@
320321
TestsFlavor="$(TestsFlavor)"
321322
/>
322323
</Target>
323-
<Target Name="RecordApkSizes"
324-
Condition=" '$(HostOS)' != 'Windows' ">
325-
<Exec
326-
Condition=" '$(HostOS)' == 'Darwin' And '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
327-
Command="stat -f &quot;stat: %z %N&quot; &quot;%(_AllArchives.Identity)&quot; > &quot;$(OutputPath)%(_AllArchives.ApkSizesInputFilename)&quot;"
328-
ContinueOnError="ErrorAndContinue"
329-
/>
330-
<Exec
331-
Condition=" '$(HostOS)' == 'Linux' And '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
332-
Command="stat -c &quot;stat: %s %N&quot; &quot;%(_AllArchives.Identity)&quot; > &quot;$(OutputPath)%(_AllArchives.ApkSizesInputFilename)&quot;"
333-
ContinueOnError="ErrorAndContinue"
334-
/>
335-
<Exec
336-
Condition=" '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
337-
Command="unzip -l &quot;%(_AllArchives.Identity)&quot; >> &quot;$(OutputPath)%(_AllArchives.ApkSizesInputFilename)&quot;"
338-
ContinueOnError="ErrorAndContinue" />
339-
<ProcessApkSizes
340-
Condition=" '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
341-
InputFilename="$(OutputPath)%(_AllArchives.ApkSizesInputFilename)"
342-
ApplicationPackageName="%(_AllArchives.Package)"
343-
ResultsFilename="%(_AllArchives.ApkSizesResultsFilename)"
344-
ReferenceFilename="$(_ApkSizesReferenceDirectory)\$([System.IO.Path]::GetFileName(%(_AllArchives.ApkSizesResultsFilename)))"
345-
DefinitionsFilename="%(_AllArchives.ApkSizesDefinitionFilename)"
346-
AddResults="True"
347-
LabelSuffix="-$(Configuration)$(TestsFlavor)"
348-
ContinueOnError="ErrorAndContinue"
349-
/>
324+
<Target Name="CheckAndRecordApkSizes">
350325
<PropertyGroup>
351-
<ApkDiffPath>$(XAInstallPrefix)xbuild\Xamarin\Android\$(HostOS)\apkdiff</ApkDiffPath>
352-
<ApkDiffPath Condition=" !Exists('$(ApkDiffPath)') ">$(MonoAndroidBinDirectory)\apkdiff</ApkDiffPath>
326+
<ApkDiffExtension Condition=" '$(HostOS)' == 'Windows' ">.exe</ApkDiffExtension>
327+
<ApkDiffPath Condition=" '$(HostOS)' == 'Windows' ">$(XAInstallPrefix)xbuild\Xamarin\Android\apkdiff\apkdiff$(ApkDiffExtension)</ApkDiffPath>
328+
<ApkDiffPath Condition=" !Exists('$(ApkDiffPath)') ">$(XAInstallPrefix)xbuild\Xamarin\Android\$(HostOS)\apkdiff$(ApkDiffExtension)</ApkDiffPath>
329+
<ApkDiffPath Condition=" !Exists('$(ApkDiffPath)') ">$(MonoAndroidBinDirectory)\apkdiff$(ApkDiffExtension)</ApkDiffPath>
353330
</PropertyGroup>
354-
<Exec
355-
Condition=" Exists('$(ApkDiffPath)') "
356-
Command="&quot;$(ApkDiffPath)&quot; -v -c &quot;HEAD/`git branch --show-current`: `git rev-parse HEAD`&quot; -s &quot;%(_AllArchives.Identity)&quot;;mv &quot;%(_AllArchives.RelativeDir)\%(_AllArchives.Filename)%(_AllArchives.Extension)desc&quot; &quot;%(_AllArchives.RelativeDir)\%(_AllArchives.Filename)-$(Configuration)$(TestsFlavor)%(_AllArchives.Extension)desc&quot;"
331+
<ApkDiffCheckRegression
332+
Condition="Exists('$(_ApkSizesReferenceDirectory)\$([System.IO.Path]::GetFileNameWithoutExtension(%(_AllArchives.Identity)))-$(Configuration)$(TestsFlavor)$([System.IO.Path]::GetExtension(%(_AllArchives.Identity)))desc')"
333+
ApkDiffTool="$(ApkDiffPath)"
334+
Package="%(_AllArchives.Identity)"
335+
ReferenceDescription="$(_ApkSizesReferenceDirectory)\$([System.IO.Path]::GetFileNameWithoutExtension(%(_AllArchives.Identity)))-$(Configuration)$(TestsFlavor)$([System.IO.Path]::GetExtension(%(_AllArchives.Identity)))desc"
336+
TestResultDirectory="$(MSBuildThisFileDirectory)..\.."
357337
ContinueOnError="ErrorAndContinue"
358338
/>
359339
</Target>

‎tests/RunApkTests.targets‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@
8383
UndeployTestApks;
8484
DeployTestApks;
8585
DeployTestAabs;
86-
RecordApkSizes;
86+
CheckAndRecordApkSizes;
8787
RunTestApks;
8888
ReleaseAndroidTarget;
8989
RenameApkTestCases;

0 commit comments

Comments
 (0)