Arcade powered source build (#32790)

* Add source-build ci leg

* Remove patches

* Fixup project type logic

* Mark Components.E2ETests as NodeJsProject

* Exclude YarnMSBuild project in source build

* Update binlog parameter logic

* Modify source build global json script

Co-authored-by: Michael Simons <[email protected]>

Author: John Luo

.azure/pipelines/ci.yml

@@ -749,56 +749,13 @@ stages:
           includeForks: true
 
   # Source build
-  - job: Source_Build
-    displayName: 'Test: Linux Source Build'
-    container: centos:7
-    pool:
-      vmImage: 'ubuntu-18.04'
-    variables:
-      LC_ALL: 'en_US.UTF-8'
-      LANG: 'en_US.UTF-8'
-      LANGUAGE: 'en_US.UTF-8'
-      DotNetCoreSdkDir: $(Agent.ToolsDirectory)/dotnet
-      # This isn't needed in the path because build does not need to _use_ global tools.
-      DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory)
-      DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: true
-    steps:
-    - script: |
-        source eng/common/native/common-library.sh
-        mkdir -p $(System.DefaultWorkingDirectory)/.tools
-        GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $(System.DefaultWorkingDirectory)/.tools/jq
-        chmod +x $(System.DefaultWorkingDirectory)/.tools/jq
-        echo "##vso[task.prependpath]$(System.DefaultWorkingDirectory)/.tools"
-      displayName: Install jq
-    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
-        - task: Bash@3
-          displayName: Setup Private Feeds Credentials
-          inputs:
-            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
-          env:
-            Token: $(dn-bot-dnceng-artifact-feeds-rw)
-    - script: ./eng/scripts/ci-source-build.sh --ci --nobl --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false $(_InternalRuntimeDownloadArgs)
-      displayName: Run ci-source-build.sh
-    - task: PublishBuildArtifacts@1
-      displayName: Upload logs
-      condition: always()
-      continueOnError: true
-      inputs:
-        pathtoPublish: artifacts/log/
-        artifactName: Source_Build_Logs
-        artifactType: Container
-        parallel: true
-    - task: PublishBuildArtifacts@1
-      displayName: Upload package artifacts
-      # Only capture source build artifacts in PRs for the sake of inspecting
-      # changes that impact source-build. The artifacts from this build pipeline are never actually used.
-      condition: and(succeeded(), in(variables['Build.Reason'], 'PullRequest'))
-      inputs:
-        pathtoPublish: artifacts/packages/
-        artifactName: Source_Build_Packages
-        artifactType: Container
-        parallel: true
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      platform:
+        name: 'Managed'
+        container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df28-20191023143754'
+        buildScript: './eng/build.sh'
+        skipPublishValidation: true
 
   # Publish to the BAR
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -827,7 +784,7 @@ stages:
             - Linux_musl_arm_build
             - Linux_musl_arm64_build
           # In addition to the dependencies above, ensure the build was successful overall.
-          - Source_Build
+          - Source_Build_Managed
         pool:
           vmImage: vs2017-win2016
         publishUsingPipelines: ${{ variables._PublishUsingPipelines }}

Directory.Build.props

@@ -29,7 +29,7 @@
         ( $(MSBuildProjectName.EndsWith('Tests')) OR
           $(MSBuildProjectName.EndsWith('.Test')) OR
           $(MSBuildProjectName.EndsWith('.FunctionalTest')) ) ">true</IsUnitTestProject>
-    <IsTestAssetProject Condition=" $(RepoRelativeProjectDir.Contains('testassets')) ">true</IsTestAssetProject>
+    <IsTestAssetProject Condition=" $(RepoRelativeProjectDir.Contains('testassets')) OR $(MSBuildProjectName.Contains('TestCommon'))">true</IsTestAssetProject>
     <IsSampleProject Condition=" $(RepoRelativeProjectDir.ToUpperInvariant().Contains('SAMPLE')) ">true</IsSampleProject>
     <IsAnalyzersProject Condition="$(MSBuildProjectName.EndsWith('.Analyzers'))">true</IsAnalyzersProject>
     <IsShipping Condition=" '$(IsSampleProject)' == 'true' OR
@@ -171,7 +171,8 @@
       linux-musl-arm64;
       linux-x64;
       linux-arm;
-      linux-arm64
+      linux-arm64;
+      freebsd-x64
     </SupportedRuntimeIdentifiers>
 
     <!-- Make error messages clickable in VS Code's console -->

eng/Build.props

@@ -134,6 +134,14 @@
         <ProjectToBuild Condition=" '$(BuildNodeJS)' == 'true'" Include="@(NodeJsProjects)" Exclude="@(ProjectToExclude)" />
         <ProjectToExclude Condition=" '$(BuildNodeJS)' != 'true'" Include="@(NodeJsProjects)" />
 
+        <YarnMSBuildProjects Include="
+                          $(RepoRoot)src\Components\test\E2ETest\Microsoft.AspNetCore.Components.E2ETests.csproj;
+                          $(RepoRoot)src\Components\WebAssembly\Authentication.Msal\src\Microsoft.Authentication.WebAssembly.Msal.csproj;
+                          $(RepoRoot)src\Components\WebAssembly\WebAssembly.Authentication\src\Microsoft.AspNetCore.Components.WebAssembly.Authentication.csproj;
+                          "
+                        Exclude="@(ProjectToExclude)" />
+        <ProjectToExclude Condition=" '$(DotNetBuildFromSource)' == 'true'" Include="@(YarnMSBuildProjects)" />
+
         <JavaProjects Include="$(RepoRoot)src\SignalR\**\*.javaproj"
                       Exclude="@(ProjectToExclude)" />
 

eng/Common.props

@@ -3,6 +3,7 @@
     <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('Windows'))">win</TargetOsName>
     <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('OSX'))">osx</TargetOsName>
     <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('Linux'))">linux</TargetOsName>
+    <TargetOsName Condition=" '$(TargetOsName)' == '' AND $([MSBuild]::IsOSPlatform('FreeBSD'))">freebsd</TargetOsName>
     <TargetArchitecture Condition="'$(TargetArchitecture)' == ''">x64</TargetArchitecture>
     <TargetRuntimeIdentifier Condition="'$(TargetRuntimeIdentifier)' == ''">$(TargetOsName)-$(TargetArchitecture)</TargetRuntimeIdentifier>
   </PropertyGroup>

eng/SourceBuild.props

@@ -6,18 +6,12 @@
     <CloneSubmodulesToInnerSourceBuildRepo>false</CloneSubmodulesToInnerSourceBuildRepo>
   </PropertyGroup>
 
-  <Target Name="ApplySourceBuildPatchFiles"
+  <Target Name="PrepareGlobalJsonForSourceBuild"
           AfterTargets="PrepareInnerSourceBuildRepoRoot"
           BeforeTargets="RunInnerSourceBuildCommand">
-
-    <ItemGroup>
-      <SourceBuildPatchFile Include="$(RepositoryEngineeringDir)source-build-patches\*.patch" />
-    </ItemGroup>
-
     <Exec
-      Command="git apply --ignore-whitespace --whitespace=nowarn &quot;%(SourceBuildPatchFile.FullPath)&quot;"
-      WorkingDirectory="$(InnerSourceBuildRepoRoot)"
-      Condition="'@(SourceBuildPatchFile)' != ''" />
+      Command="./eng/scripts/prepare-sourcebuild-globaljson.sh"
+      WorkingDirectory="$(InnerSourceBuildRepoRoot)" />
   </Target>
 
   <!--

eng/Versions.props

@@ -183,6 +183,8 @@
     <MicrosoftBuildVersion>16.9.0</MicrosoftBuildVersion>
     <MicrosoftAzureSignalRVersion>1.2.0</MicrosoftAzureSignalRVersion>
     <MicrosoftBuildFrameworkVersion>16.9.0</MicrosoftBuildFrameworkVersion>
+    <MicrosoftBuildTaskCoreVersion>16.9.0</MicrosoftBuildTaskCoreVersion>
+    <MicrosoftBuildUtilitiesCoreVersion>16.9.0</MicrosoftBuildUtilitiesCoreVersion>
     <MicrosoftBuildLocatorVersion>1.2.6</MicrosoftBuildLocatorVersion>
     <MicrosoftBuildUtilitiesCoreVersion>16.9.0</MicrosoftBuildUtilitiesCoreVersion>
     <MicrosoftCodeAnalysisCommonVersion>3.8.0</MicrosoftCodeAnalysisCommonVersion>
@@ -201,6 +203,7 @@
     <MicrosoftWebWebView2Version>1.0.705.50</MicrosoftWebWebView2Version>
     <MicrosoftWebXdtVersion>1.4.0</MicrosoftWebXdtVersion>
     <SystemIdentityModelTokensJwtVersion>6.10.0</SystemIdentityModelTokensJwtVersion>
+    <NuGetPackagingVersion>5.10.0-rc.7240</NuGetPackagingVersion>
     <NuGetVersioningVersion>5.10.0-rc.7240</NuGetVersioningVersion>
     <NuGetFrameworksVersion>5.10.0-rc.7240</NuGetFrameworksVersion>
     <SystemComponentModelAnnotationsVersion>5.0.0</SystemComponentModelAnnotationsVersion>

eng/build.sh

@@ -342,6 +342,7 @@ if [[ "$binary_log" == true ]]; then
     if [[ "$found" == false ]]; then
         msbuild_args[${#msbuild_args[*]}]="/bl:$log_dir/Build.binlog"
     fi
+    toolset_build_args[${#toolset_build_args[*]}]="/bl:$log_dir/Build.repotasks.binlog"
 elif [[ "$ci" == true ]]; then
     # Ensure the artifacts/log directory isn't empty to avoid warnings.
     touch "$log_dir/empty.log"

eng/scripts/ci-source-build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+while IFS= read -r line
+do
+    # Remove dotnet/x64 runtimes
+    if [[ "$line" == *"dotnet/x86"* ]]
+    then
+        while IFS= read -r removeLine
+        do
+            if [[ "$removeLine" == *"]"* ]]
+            then
+                break
+            fi
+        done
+    else
+        # Change dotnet/x64 to dotnet
+        echo "${line/dotnet\/x64/dotnet}"
+    fi
+done < $DIR/../../global.json > global.json.swap
+mv global.json.swap $DIR/../../global.json