Reduce build duplication in pipelines

dougbu · dougbu · commit 58f970a3e639 · 2020-05-26T16:28:06.000-07:00
- build native assets and repo tasks once per CI job
- only cleanup framework references after packing managed projects

nits:
- remove default `-arch x64` option
- wrap a few long lines
- remove extra `-forceCoreMsbuild` options in SiteExtensions' build.cmd
diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml
@@ -47,7 +47,8 @@ variables:
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNet-MSRC-Storage
   - name: _InternalRuntimeDownloadArgs
-    value: -DotNetRuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet -DotNetRuntimeSourceFeedKey $(dotnetclimsrc-read-sas-token-base64) /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)'
+    value: -DotNetRuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet -DotNetRuntimeSourceFeedKey
+           $(dotnetclimsrc-read-sas-token-base64) /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)'
   # The code signing doesn't use the aspnet build scripts, so the msbuild parameters have
   # to be passed directly. This is awkward, since we pass the same info above, but we have
   # to have it in two different forms
@@ -140,32 +141,35 @@ stages:
       - script: ./build.cmd
                 -ci
                 -nobl
+                -noBuildRepoTasks
                 -arch x86
                 -pack
                 -all
                 -noBuildJava
+                -noBuildNative
                 /p:OnlyPackPlatformSpecificPackages=true
                 $(_BuildArgs)
                 $(_InternalRuntimeDownloadArgs)
         displayName: Build x86
 
-      # This is in a separate build step with -forceCoreMsbuild to workaround MAX_PATH limitations - https://github.com/Microsoft/msbuild/issues/53
       - script: .\src\SiteExtensions\build.cmd
                 -ci
                 -nobl
+                -noBuildRepoTasks
                 -pack
                 -noBuildDeps
                 $(_BuildArgs)
                 $(_InternalRuntimeDownloadArgs)
         condition: ne(variables['Build.Reason'], 'PullRequest')
         displayName: Build SiteExtension
 
-      # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If https://github.com/dotnet/arcade/issues/1957 is resolved,
-      # consider running code-signing inline with the other previous steps.
-      # Sign check is disabled because it is run in a separate step below, after installers are built.
+      # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If
+      # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other
+      # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built.
       - script: ./build.cmd
                 -ci
                 -nobl
+                -noBuildRepoTasks
                 -noBuild
                 -noRestore
                 -sign
@@ -177,6 +181,7 @@ stages:
       - script: ./build.cmd
                 -ci
                 -nobl
+                -noBuildRepoTasks
                 -sign
                 -buildInstallers
                 /p:DotNetSignType=$(_SignType)
@@ -305,7 +310,6 @@ stages:
       - script: ./build.sh
             --ci
             --nobl
-            --arch x64
             --pack
             --all
             --no-build-nodejs
@@ -319,7 +323,6 @@ stages:
           ./dockerbuild.sh bionic \
             --ci \
             --nobl \
-            --arch x64 \
             --build-installers \
             --no-build-deps \
             --no-build-nodejs \
@@ -334,7 +337,6 @@ stages:
           ./dockerbuild.sh rhel \
             --ci \
             --nobl \
-            --arch x64 \
             --build-installers \
             --no-build-deps \
             --no-build-nodejs \
@@ -427,7 +429,6 @@ stages:
       agentOs: Linux
       buildScript: ./dockerbuild.sh alpine
       buildArgs:
-        --arch x64
         --os-name linux-musl
         --pack
         --all
@@ -492,7 +493,9 @@ stages:
       jobDisplayName: "Test: Windows Server 2016 x64"
       agentOs: Windows
       isTestingJob: true
-      buildArgs: -all -pack -test "/p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false" $(_InternalRuntimeDownloadArgs)
+      buildArgs: -all -pack -test /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true
+                 /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false
+                 $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
         displayName: Setup IISExpress test certificates and schema
@@ -530,9 +533,9 @@ stages:
       steps:
       - script: ./build.cmd -ci -nobl -all -pack $(_InternalRuntimeDownloadArgs)
         displayName: Build Repo
-      - script: ./src/ProjectTemplates/build.cmd -ci -nobl -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true"
+      - script: ./src/ProjectTemplates/build.cmd -ci -nobl -noBuildRepoTasks -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true"
         displayName: Pack Templates
-      - script: ./src/ProjectTemplates/build.cmd -ci -nobl -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true"
+      - script: ./src/ProjectTemplates/build.cmd -ci -nobl -noBuildRepoTasks -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true"
         displayName: Test Templates
       artifacts:
       - name: Windows_Test_Templates_Dumps
@@ -630,11 +633,13 @@ stages:
       timeoutInMinutes: 180
       steps:
       # Build the shared framework
-      - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+      - script: ./build.cmd -ci -nobl -all -pack /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
         displayName: Build shared fx
       - script: .\restore.cmd -ci -nobl /p:BuildInteropProjects=true
         displayName: Restore interop projects
-      - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+      - script: ./build.cmd -ci -nobl -noBuildRepoTasks -noRestore -test -all -noBuildNative -projects eng\helix\helix.proj
+                /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true
+                /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
         displayName: Run build.cmd helix target
         env:
           HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
diff --git a/.azure/pipelines/devBuilds.yml b/.azure/pipelines/devBuilds.yml
@@ -31,7 +31,6 @@ stages:
       steps:
       - script: ./build.cmd
                 -ci
-                -arch x64
                 /bl:artifacts/log/build.components.x64.binlog
         displayName: Build x64
         workingDirectory: ./src/Components
@@ -54,7 +53,6 @@ stages:
       steps:
       - script: ./build.cmd
                 -ci
-                -arch x64
                 /bl:artifacts/log/build.servers.x64.binlog
         displayName: Build x64
         workingDirectory: ./src/Servers
@@ -77,7 +75,6 @@ stages:
       steps:
       - script: ./build.cmd
                 -ci
-                -arch x64
                 /bl:artifacts/log/build.projectTemplates.x64.binlog
         displayName: Build x64
         workingDirectory: ./src/ProjectTemplates
@@ -100,7 +97,6 @@ stages:
       steps:
       - script: ./build.cmd
                 -ci
-                -arch x64
                 /bl:artifacts/log/build.all.x64.binlog
         displayName: Build x64
       artifacts:
diff --git a/.azure/pipelines/helix-matrix.yml b/.azure/pipelines/helix-matrix.yml
@@ -7,16 +7,16 @@ schedules:
     include:
     - master
   always: true
-    
+
 variables:
 - ${{ if ne(variables['System.TeamProject'], 'internal') }}:
   - name: _UseHelixOpenQueues
     value: 'true'
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNet-HelixApi-Access
   - name: _UseHelixOpenQueues
-    value: 'false'    
-    
+    value: 'false'
+
 jobs:
 - template: jobs/default-build.yml
   parameters:
@@ -26,11 +26,13 @@ jobs:
     timeoutInMinutes: 240
     steps:
     # Build the shared framework
-    - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+    - script: ./build.cmd -ci -nobl -all -pack /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
       displayName: Build shared fx
     - script: .\restore.cmd -ci /p:BuildInteropProjects=true
       displayName: Restore interop projects
-    - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj /p:IsHelixDaily=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+    - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj
+              /p:IsHelixDaily=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true
+              /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
       displayName: Run build.cmd helix target
       env:
         HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
@@ -39,7 +41,7 @@ jobs:
     - name: Helix_logs
       path: artifacts/log/
       publishOnError: true
-      
+
 # Helix ARM64
 - template: jobs/default-build.yml
   parameters:
@@ -50,7 +52,9 @@ jobs:
     steps:
     - script: ./restore.sh -ci -nobl
       displayName: Restore
-    - script: ./build.sh -ci --nobl --arch arm64 -test --no-build-nodejs --all -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+    - script: ./build.sh -ci --nobl --arch arm64 -test --no-build-nodejs --all -projects
+              $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true
+              /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
       displayName: Run build.sh helix arm64 target
       env:
         HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
diff --git a/.azure/pipelines/quarantined-tests.yml b/.azure/pipelines/quarantined-tests.yml
@@ -31,11 +31,13 @@ jobs:
     timeoutInMinutes: 240
     steps:
     # Build the shared framework
-    - script: ./build.cmd -ci -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog
+    - script: ./build.cmd -ci -all -pack /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog
       displayName: Build shared fx
     - script: .\restore.cmd -ci /p:BuildInteropProjects=true
       displayName: Restore interop projects
-    - script: .\build.cmd -ci -NoRestore -test -noBuildJava -all -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+    - script: ./build.cmd -ci -noRestore -test -all -noBuildJava -noBuildNative -projects eng\helix\helix.proj
+              /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true
+              /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
       displayName: Run build.cmd helix target
       env:
         HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets
@@ -5,7 +5,8 @@
   <Import Project="SharedFramework.Local.props" />
 
   <!-- This is temporary until we can use FrameworkReference to build our own packages. -->
-  <Target Name="RemoveSharedFrameworkOnlyRefsFromNuspec" AfterTargets="Pack">
+  <Target Name="RemoveSharedFrameworkOnlyRefsFromNuspec" AfterTargets="Pack"
+      Condition=" '$(MSBuildRuntimeType)' == 'core' ">
     <ItemGroup>
       <_BuildOutput Include="$(ArtifactsShippingPackagesDir)*.nupkg"
                     Exclude="$(ArtifactsShippingPackagesDir)*.symbols.nupkg" />
diff --git a/src/SiteExtensions/build.cmd b/src/SiteExtensions/build.cmd
@@ -18,21 +18,21 @@ IF %ERRORLEVEL% NEQ 0 (
 ECHO Building x64 LoggingBranch
 REM /p:DisableTransitiveFrameworkReferences=true is needed to prevent SDK from picking up transitive references to
 REM Microsoft.AspNetCore.App as framework references https://github.com/dotnet/sdk/pull/3221
-CALL "%RepoRoot%\build.cmd" -forceCoreMsbuild -arch x64 -projects "%~dp0LoggingBranch\LB.csproj" ^
+CALL "%RepoRoot%\build.cmd" -arch x64 -projects "%~dp0LoggingBranch\LB.csproj" ^
     /p:DisableTransitiveFrameworkReferences=true /bl:artifacts/log/SiteExtensions-LoggingBranch-x64.binlog %*
 IF %ERRORLEVEL% NEQ 0 (
    EXIT /b %ErrorLevel%
 )
 
 ECHO Building x86 LoggingBranch
-CALL "%RepoRoot%\build.cmd" -forceCoreMsbuild -arch x86 -projects "%~dp0LoggingBranch\LB.csproj" ^
+CALL "%RepoRoot%\build.cmd" -arch x86 -projects "%~dp0LoggingBranch\LB.csproj" ^
     /p:DisableTransitiveFrameworkReferences=true /bl:artifacts/log/SiteExtensions-LoggingBranch-x86.binlog %*
 IF %ERRORLEVEL% NEQ 0 (
    EXIT /b %ErrorLevel%
 )
 
 ECHO Building Microsoft.AspNetCore.AzureAppServices.SiteExtension
-CALL "%RepoRoot%\build.cmd" -forceCoreMsbuild -projects ^
+CALL "%RepoRoot%\build.cmd" -projects ^
     "%~dp0LoggingAggregate\src\Microsoft.AspNetCore.AzureAppServices.SiteExtension\Microsoft.AspNetCore.AzureAppServices.SiteExtension.csproj" ^
     /bl:artifacts/log/SiteExtensions-LoggingAggregate.binlog %*
 IF %ERRORLEVEL% NEQ 0 (
