Build time changes (#22362)

* Build time changes
A few changes for build time
- Don't build tests with SkipTestBuild=true and use that for official
  build legs. This cuts 40%-50% off the msbuild invocations for build.
  The longest build leg drops by about 30 mins.
- Skip logging of some task parameters and their metadata.
  This reduces overall binlog size, which is a major contributor to
  build time.

Unfortunately, this does not mean we can yet turn binlogs back on. This
change can actually increase the overall binlog size due to logging of
more project started arguments. There is another optimization for this
in progress.

Co-authored-by: Doug Bunting <[email protected]>

Author: mmitche

.azure/pipelines/ci.yml

@@ -35,7 +35,7 @@ variables:
   - name: _UseHelixOpenQueues
     value: 'true'
   - name: _BuildArgs
-    value: ''
+    value: '/p:SkipTestBuild=true'
   - name: _PublishArgs
     value: ''
   - name: _SignType
@@ -66,6 +66,7 @@ variables:
     - name: _BuildArgs
       value: /p:TeamName=$(_TeamName)
              /p:OfficialBuildId=$(Build.BuildNumber)
+             /p:SkipTestBuild=true
     - name: _SignType
       value: real
 
@@ -81,7 +82,7 @@ variables:
 
   - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
     - name: _BuildArgs
-      value: ''
+      value: '/p:SkipTestBuild=true'
     - name: _SignType
       value: test
     - name: _PublishArgs

Directory.Build.props

@@ -36,6 +36,42 @@
     <!-- Workaround issue with ComponentsAnalyzer throwing for interfaces -->
     <DisableImplicitComponentsAnalyzers>true</DisableImplicitComponentsAnalyzers>
   </PropertyGroup>
+  
+  <!-- Disable logging of some task parameters or metadata to reduce binlog size.
+       Reenable logging of any particular item by changing the value of the property below to 'false'
+       The format is as follows:
+       DisableLogTaskParameter_[task name]_[parameter name] - Disable logging of a particular parameter
+       DisableLogTaskParameterItemMetadata_[task name]_[parameter name] - Disable logging of item metadata of the parameter
+  -->
+  <PropertyGroup>
+    <TrimTaskParameters Condition=" '$(TrimTaskParameters)' == '' ">true</TrimTaskParameters>
+    <!-- ItemsToHash is used for incremental building and hashes input properties to a file
+         This is not generally useful for day-to-day build debugging. -->
+    <DisableLogTaskParameter_Hash_ItemsToHash>$(TrimTaskParameters)</DisableLogTaskParameter_Hash_ItemsToHash>
+    
+    <!-- JoinItems takes input ItemGroups. The output ItemGroup is logged. -->
+    <DisableLogTaskParameter_JoinItems_Right>$(TrimTaskParameters)</DisableLogTaskParameter_JoinItems_Right>
+    <DisableLogTaskParameter_JoinItems_Left>$(TrimTaskParameters)</DisableLogTaskParameter_JoinItems_Left>
+    
+    <!-- ConvertToAbsolutePaths - The output parameter (AbsolutePaths) is interesting
+         while the input Path is not generally useful. The output itemgroup's metadata
+         is not altered by the task. -->
+    <DisableLogTaskParameter_ConvertToAbsolutePath_Paths>$(TrimTaskParameters)</DisableLogTaskParameter_ConvertToAbsolutePath_Paths>
+    <DisableLogTaskParameterItemMetadata_ConvertToAbsolutePath_Paths>$(TrimTaskParameters)</DisableLogTaskParameterItemMetadata_ConvertToAbsolutePath_Paths>
+    
+    <!-- The standard msbuild Copy task does not use Metadata and thus the input/outputs
+         item metadata is not relevant -->
+     <DisableLogTaskParameterItemMetadata_Copy_SourceFiles>$(TrimTaskParameters)</DisableLogTaskParameterItemMetadata_Copy_SourceFiles>
+     <DisableLogTaskParameterItemMetadata_Copy_DestinationFiles>$(TrimTaskParameters)</DisableLogTaskParameterItemMetadata_Copy_DestinationFiles>
+     
+     <!-- Reference metadata for GenerateDepsFile, Csc, RAR, etc. are sometimes useful, but extraordinarily large
+          when building against a shared framework where the number of input assemblies is very large.
+          Avoid logging these by default. -->
+     <DisableLogTaskParameterItemMetadata_GenerateDepsFile_ReferenceAssemblies>$(TrimTaskParameters)</DisableLogTaskParameterItemMetadata_GenerateDepsFile_ReferenceAssemblies>
+     <DisableLogTaskParameterItemMetadata_GenerateDepsFile_ReferencePaths>$(TrimTaskParameters)</DisableLogTaskParameterItemMetadata_GenerateDepsFile_ReferencePaths>
+     <DisableLogTaskParameterItemMetadata_ResolveAssemblyReference_Assemblies>$(TrimTaskParameters)</DisableLogTaskParameterItemMetadata_ResolveAssemblyReference_Assemblies>
+     <DisableLogTaskParameterItemMetadata_Csc_References>$(TrimTaskParameters)</DisableLogTaskParameterItemMetadata_Csc_References>
+  </PropertyGroup>
 
   <Import Project="eng\QuarantinedTests.BeforeArcade.props" />
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />

Directory.Build.targets

@@ -5,6 +5,16 @@
     <!-- Analyzer package are needed in source build for WebSDK -->
     <ExcludeFromSourceBuild
         Condition="'$(ExcludeFromSourceBuild)' == '' and '$(DotNetBuildFromSource)' == 'true' and '$(IsAspNetCoreApp)' != 'true' and '$(IsReferenceAssemblyProject)' != 'true' and '$(IsAnalyzersProject)' != 'true'">true</ExcludeFromSourceBuild>
+        
+    <!-- If the user has specified that they want to skip building any test related projects with SkipTestBuild,
+     suppress all targets for TestProjects using ExcludeFromBuild. -->
+    <ExcludeFromBuild Condition="'$(SkipTestBuild)' == 'true' and 
+        ('$(IsTestProject)' == 'true' or
+         '$(IsUnitTestProject)' == 'true' or
+         '$(IsTestAssetProject)' == 'true' or
+         '$(IsBenchmarkProject)' == 'true' or 
+         '$(IsSampleProject)' == 'true' or
+         '$(IsSpecificationTestProject)' == 'true')">true</ExcludeFromBuild>
   </PropertyGroup>
 
   <PropertyGroup Label="Resx settings">

src/Components/Web.JS/dist/Release/blazor.server.js

@@ -17,7 +17,7 @@
     <_BlazorToolsDir>$(MSBuildThisFileDirectory)bin\$(BlazorBuildConfiguration)\tools\</_BlazorToolsDir>
   </PropertyGroup>
 
-  <Target Name="Check_BlazorJSFiles" BeforeTargets="Build">
+  <Target Name="Check_BlazorJSFiles" BeforeTargets="Build" Condition="'$(ExcludeFromBuild)' != 'true'">
     <Error Text="blazor.webassembly.js file could not be found at $(_BlazorJsPath)" Condition="!Exists($(_BlazorJsPath))" />
   </Target>
 

src/Components/WebAssembly/Build/src/ReferenceBlazorBuildFromSource.props

@@ -8,7 +8,7 @@
     <Content Include="..\Common.FunctionalTests\AppHostConfig\*.config" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 
-  <Target Name="BuildAssets" AfterTargets="Build">
+  <Target Name="BuildAssets" AfterTargets="Build" Condition="'$(ExcludeFromBuild)' != 'true'">
     <MSBuild Projects="@(ProjectReference)" Targets="PublishTestsAssets" SkipNonexistentTargets="true" BuildInParallel="True">
       <Output TaskParameter="TargetOutputs" ItemName="PublishedTestAsset" />
     </MSBuild>

src/Servers/IIS/IIS/test/FunctionalTest.props

@@ -29,7 +29,7 @@
     <EmbeddedResource Include="Http.config" />
   </ItemGroup>
 
-  <Target Name="ValidateNativeComponentsBuilt" AfterTargets="Build" Condition="'$(BuildIisNativeProjects)' == 'true'">
+  <Target Name="ValidateNativeComponentsBuilt" AfterTargets="Build" Condition="'$(BuildIisNativeProjects)' == 'true' and '$(SkipTestBuild)' != 'true'">
     <Error Text="Required dll from ANCM has not been built. To build ANCM, you must use MSBuild.exe."
           Condition="!Exists('$(AspNetCoreModuleV2ShimDll)') OR !Exists('$(AspNetCoreModuleV2OutOfProcessHandlerDll)')" />
   </Target>

src/Servers/IIS/IntegrationTesting.IIS/src/Microsoft.AspNetCore.Server.IntegrationTesting.IIS.csproj

@@ -42,15 +42,15 @@
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.targets))\Directory.Build.targets" />
 
   <!-- Define Target overrides after importing Directory.Build.targets so these don't get overridden -->
-  <Target Name="Pack" DependsOnTargets="$(PackDependsOn)" Condition="'$(IsPackable)' == 'true'">
+  <Target Name="Pack" DependsOnTargets="$(PackDependsOn)" Condition="'$(IsPackable)' == 'true' and '$(SkipTestBuild)' != 'true'">
     <Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=Pack" />
     <Message Text="> gradlew $(GradleOptions) createPackage" Importance="high" />
     <Exec Command="./gradlew $(GradleOptions) createPackage" />
     <Message Importance="high" Text="java:signalr -> $(PackageOutputPath)%(JavaBuildFiles.Identity)" />
     <Copy SourceFiles="build\libs\%(JavaBuildFiles.Identity)" DestinationFolder="$(PackageOutputPath)" />
   </Target>
 
-  <Target Name="Build">
+  <Target Name="Build" Condition="'$(SkipTestBuild)' != 'true'">
     <Telemetry EventName="NETCORE_ENGINEERING_TELEMETRY" EventData="Category=Build" />
     <Exec Command="./gradlew $(GradleOptions) compileJava" />
   </Target>

src/SignalR/clients/java/signalr/signalr.client.java.Tests.javaproj

@@ -40,11 +40,11 @@
     <RemoveDir Directories="$(TargetDir)TestProjects" Condition="Exists('$(TargetDir)TestProjects')" />
   </Target>
 
-  <Target Name="PublishDotNetOpenApiOnBuild" BeforeTargets="Build" Condition="'$(DotNetBuildFromSource)' != 'true'">
+  <Target Name="PublishDotNetOpenApiOnBuild" BeforeTargets="Build" Condition="'$(DotNetBuildFromSource)' != 'true' and '$(ExcludeFromBuild)' != 'true'">
     <MSBuild Projects="$(OpenAPIToolCSProjPath)" Targets="Publish" Properties="PublishDir=$(OutputPath)\tool\;Configuration=$(Configuration)" />
   </Target>
 
-  <Target Name="PublishDotNetOpenApiOnPublish" BeforeTargets="Publish"  Condition="'$(DotNetBuildFromSource)' != 'true'">
+  <Target Name="PublishDotNetOpenApiOnPublish" BeforeTargets="Publish"  Condition="'$(DotNetBuildFromSource)' != 'true' and '$(ExcludeFromBuild)' != 'true'">
     <MSBuild Projects="$(OpenAPIToolCSProjPath)" Targets="Publish" Properties="PublishDir=$(PublishDir)\tool\;Configuration=$(Configuration)" />
   </Target>
 </Project>

src/Tools/Microsoft.dotnet-openapi/test/dotnet-microsoft.openapi.Tests.csproj

@@ -28,14 +28,14 @@
   </Target>
 
   <!-- Do not publish in source build -->
-  <Target Name="PublishDotNetWatchOnBuild" BeforeTargets="Build" Condition="'$(DotNetBuildFromSource)' != 'true'">
+  <Target Name="PublishDotNetWatchOnBuild" BeforeTargets="Build" Condition="'$(DotNetBuildFromSource)' != 'true' and '$(ExcludeFromBuild)' != 'true'">
     <MSBuild Projects="..\src\dotnet-watch.csproj"
       Targets="Publish"
       Properties="PublishDir=$(OutputPath)\tool\;Configuration=$(Configuration)" />
   </Target>
 
   <!-- Do not publish in source build -->
-  <Target Name="PublishDotNetWatchOnPublish" BeforeTargets="Publish" Condition="'$(DotNetBuildFromSource)' != 'true'">
+  <Target Name="PublishDotNetWatchOnPublish" BeforeTargets="Publish" Condition="'$(DotNetBuildFromSource)' != 'true' and '$(ExcludeFromBuild)' != 'true'">
     <MSBuild Projects="..\src\dotnet-watch.csproj"
       Targets="Publish"
       Properties="PublishDir=$(PublishDir)\tool\;Configuration=$(Configuration)" />