[tests] New apk size regression testing

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/ApkDiffCheckRegression.cs

@@ -0,0 +1,73 @@
+using System;
+using System.IO;
+using System.Text;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Xamarin.Android.BuildTools.PrepTasks;
+
+
+namespace Xamarin.Android.Tools.BootstrapTasks
+{
+	public class ApkDiffCheckRegression : ToolTask
+	{
+		[Required]
+		public string ApkDiffTool { get; set; }
+
+		[Required]
+		public string Package { get; set; }
+
+		[Required]
+		public string ReferenceDescription { get; set; }
+
+		[Required]
+		public string TestResultDirectory { get; set; }
+
+		protected override string ToolName => Path.GetFileName (ApkDiffTool);
+		protected override string GenerateFullPathToTool () => ApkDiffTool;
+
+		const int ApkSizeThreshold = 50*1024;
+		const int AssemblySizeThreshold = 50*1024;
+
+		StringBuilder logCopy = new StringBuilder ();
+
+		protected override string GenerateCommandLineCommands ()
+		{
+			var cmd = new CommandLineBuilder ();
+			cmd.AppendSwitch ("-s");
+			cmd.AppendSwitch ($"--test-apk-size-regression={ApkSizeThreshold}");
+			cmd.AppendSwitch ($"--test-assembly-size-regression={AssemblySizeThreshold}");
+			cmd.AppendFileNameIfNotNull (ReferenceDescription);
+			cmd.AppendFileNameIfNotNull (Package);
+
+			return cmd.ToString ();
+		}
+
+		protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance)
+		{
+			base.LogEventsFromTextOutput (singleLine, messageImportance);
+
+			logCopy.AppendLine (singleLine);
+		}
+
+		public override bool Execute ()
+		{
+			LogStandardErrorAsError = false;
+			StandardOutputImportance = "Low";
+
+			var succes = base.Execute ();
+			if (ExitCode == 0)
+				return succes;
+
+			var errorMessage = $"apkdiff exited with error code: {ExitCode}.";
+			var testResultPath = Path.Combine (TestResultDirectory, $"TestResult-apkdiff-{Path.GetFileNameWithoutExtension (ReferenceDescription)}.xml");
+			ErrorResultsHelper.CreateErrorResultsFile (
+				testResultPath,
+				nameof (ApkDiffCheckRegression),
+				"check apk size regression",
+				new Exception (errorMessage),
+				$"apkdiff output:\n{logCopy}");
+
+			return false;
+		}
+	}
+}

build-tools/automation/azure-pipelines.yaml

@@ -640,6 +640,7 @@ stages:
         configuration: $(ApkTestConfiguration)
         testName: Xamarin.Forms-Performance-Integration
         project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration).xml
         artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Default
 
@@ -648,6 +649,7 @@ stages:
         configuration: $(ApkTestConfiguration)
         testName: Xamarin.Forms-Performance-Integration-Aot
         project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration)-Aot.xml
         extraBuildArgs: /p:AotAssemblies=true
         artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Aot
@@ -657,6 +659,7 @@ stages:
         configuration: $(ApkTestConfiguration)
         testName: Xamarin.Forms-Performance-Integration-Profiled-Aot
         project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration)-Profiled-Aot.xml
         extraBuildArgs: /p:AotAssemblies=True /p:AndroidEnableProfiledAot=true
         artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Profiled-Aot
@@ -666,6 +669,7 @@ stages:
         configuration: $(ApkTestConfiguration)
         testName: Xamarin.Forms-Performance-Integration-Bundle
         project: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration)-Bundle.xml
         extraBuildArgs: /p:BundleAssemblies=true
         artifactName: Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Bundle

build-tools/automation/yaml-templates/apk-instrumentation.yaml

@@ -18,7 +18,7 @@ steps:
     configuration: ${{ parameters.configuration }}
     msbuildArguments: >
       /restore
-      /t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,RecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
+      /t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
       /bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run${{ parameters.testName }}.binlog
       ${{ parameters.extraBuildArgs }}
   condition: ${{ parameters.condition }}

build-tools/installers/create-installers.targets

@@ -116,13 +116,19 @@
   <ItemGroup>
     <_MSBuildFiles Include="$(MSBuildSrcDir)\android-support-multidex.jar" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\apkdiff.exe" />
+    <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\apkdiff.exe.config" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\apkdiff.pdb" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\Mono.Options.dll" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\libZipSharp.dll" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\libZipSharp.dll.config" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\Newtonsoft.Json.dll" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\Newtonsoft.Json-LICENSE.md" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Buffers.dll" />
+    <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Collections.Immutable.dll" />
+    <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Memory.dll" />
+    <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Numerics.Vectors.dll" />
+    <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Reflection.Metadata.dll" />
+    <_MSBuildFiles Include="$(MSBuildSrcDir)\apkdiff\System.Runtime.CompilerServices.Unsafe.dll" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\aprofutil.exe" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\aprofutil.pdb" />
     <_MSBuildFiles Include="$(MSBuildSrcDir)\cil-strip.exe" />

build-tools/scripts/TestApks.targets

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.Adb" />
+  <UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.ApkDiffCheckRegression" />
   <UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.BundleTool" />
   <UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget" />
   <UsingTask AssemblyFile="$(BootstrapTasksAssembly)" TaskName="Xamarin.Android.Tools.BootstrapTasks.CreateAndroidEmulator" />
@@ -320,40 +321,19 @@
         TestsFlavor="$(TestsFlavor)"
     />
   </Target>
-  <Target Name="RecordApkSizes"
-      Condition=" '$(HostOS)' != 'Windows' ">
-    <Exec
-        Condition=" '$(HostOS)' == 'Darwin' And '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
-        Command="stat -f &quot;stat: %z %N&quot; &quot;%(_AllArchives.Identity)&quot; > &quot;$(OutputPath)%(_AllArchives.ApkSizesInputFilename)&quot;"
-        ContinueOnError="ErrorAndContinue"
-    />
-    <Exec
-        Condition=" '$(HostOS)' == 'Linux'  And '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
-        Command="stat -c &quot;stat: %s %N&quot; &quot;%(_AllArchives.Identity)&quot; > &quot;$(OutputPath)%(_AllArchives.ApkSizesInputFilename)&quot;"
-        ContinueOnError="ErrorAndContinue"
-    />
-    <Exec
-        Condition=" '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
-        Command="unzip -l &quot;%(_AllArchives.Identity)&quot; >> &quot;$(OutputPath)%(_AllArchives.ApkSizesInputFilename)&quot;"
-        ContinueOnError="ErrorAndContinue" />
-    <ProcessApkSizes
-        Condition=" '%(_AllArchives.ApkSizesDefinitionFilename)' != '' "
-        InputFilename="$(OutputPath)%(_AllArchives.ApkSizesInputFilename)"
-        ApplicationPackageName="%(_AllArchives.Package)"
-        ResultsFilename="%(_AllArchives.ApkSizesResultsFilename)"
-        ReferenceFilename="$(_ApkSizesReferenceDirectory)\$([System.IO.Path]::GetFileName(%(_AllArchives.ApkSizesResultsFilename)))"
-        DefinitionsFilename="%(_AllArchives.ApkSizesDefinitionFilename)"
-        AddResults="True"
-        LabelSuffix="-$(Configuration)$(TestsFlavor)"
-        ContinueOnError="ErrorAndContinue"
-    />
+  <Target Name="CheckAndRecordApkSizes">
     <PropertyGroup>
-      <ApkDiffPath>$(XAInstallPrefix)xbuild\Xamarin\Android\$(HostOS)\apkdiff</ApkDiffPath>
-      <ApkDiffPath Condition=" !Exists('$(ApkDiffPath)') ">$(MonoAndroidBinDirectory)\apkdiff</ApkDiffPath>
+      <ApkDiffExtension Condition=" '$(HostOS)' == 'Windows' ">.exe</ApkDiffExtension>
+      <ApkDiffPath Condition=" '$(HostOS)' == 'Windows' ">$(XAInstallPrefix)xbuild\Xamarin\Android\apkdiff\apkdiff$(ApkDiffExtension)</ApkDiffPath>
+      <ApkDiffPath Condition=" !Exists('$(ApkDiffPath)') ">$(XAInstallPrefix)xbuild\Xamarin\Android\$(HostOS)\apkdiff$(ApkDiffExtension)</ApkDiffPath>
+      <ApkDiffPath Condition=" !Exists('$(ApkDiffPath)') ">$(MonoAndroidBinDirectory)\apkdiff$(ApkDiffExtension)</ApkDiffPath>
     </PropertyGroup>
-    <Exec
-        Condition=" Exists('$(ApkDiffPath)') "
-        Command="&quot;$(ApkDiffPath)&quot; -v -c &quot;HEAD/`git branch --show-current`: `git rev-parse HEAD`&quot; -s &quot;%(_AllArchives.Identity)&quot;;mv &quot;%(_AllArchives.RelativeDir)\%(_AllArchives.Filename)%(_AllArchives.Extension)desc&quot; &quot;%(_AllArchives.RelativeDir)\%(_AllArchives.Filename)-$(Configuration)$(TestsFlavor)%(_AllArchives.Extension)desc&quot;"
+    <ApkDiffCheckRegression
+        Condition="Exists('$(_ApkSizesReferenceDirectory)\$([System.IO.Path]::GetFileNameWithoutExtension(%(_AllArchives.Identity)))-$(Configuration)$(TestsFlavor)$([System.IO.Path]::GetExtension(%(_AllArchives.Identity)))desc')"
+        ApkDiffTool="$(ApkDiffPath)"
+        Package="%(_AllArchives.Identity)"
+        ReferenceDescription="$(_ApkSizesReferenceDirectory)\$([System.IO.Path]::GetFileNameWithoutExtension(%(_AllArchives.Identity)))-$(Configuration)$(TestsFlavor)$([System.IO.Path]::GetExtension(%(_AllArchives.Identity)))desc"
+        TestResultDirectory="$(MSBuildThisFileDirectory)..\.."
         ContinueOnError="ErrorAndContinue"
     />
   </Target>

tests/RunApkTests.targets

@@ -83,7 +83,7 @@
       UndeployTestApks;
       DeployTestApks;
       DeployTestAabs;
-      RecordApkSizes;
+      CheckAndRecordApkSizes;
       RunTestApks;
       ReleaseAndroidTarget;
       RenameApkTestCases;