Add support for source-build

Author: John Luo

.azure/pipelines/ci.yml

@@ -471,3 +471,50 @@ jobs:
     - name: Linux_Test_Results
       path: artifacts/TestResults/
       publishOnError: true
+
+# Source build
+- job: Source_Build
+  displayName: 'Test: Linux Source Build'
+  container: centos:7
+  pool:
+    vmImage: 'ubuntu-16.04'
+  variables:
+    DotNetCoreSdkDir: $(Agent.ToolsDirectory)/dotnet
+    DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: true
+  steps:
+  - script: |
+      source eng/common/native/common-library.sh
+      mkdir -p $HOME/bin
+      GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $HOME/bin/jq
+      chmod +x $HOME/bin/jq
+      echo "##vso[task.prependpath]$HOME/bin"
+    displayName: Install jq
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk'
+    inputs:
+      packageType: sdk
+      # The SDK version selected here is intentionally supposed to use the latest release
+      # For the purpose of building Linux distros, we can't depend on features of the SDK
+      # which may not exist in pre-built versions of the SDK
+      version: 3.0.x
+      installationPath: $(DotNetCoreSdkDir)
+      includePreviewVersions: true
+  - script: ./eng/scripts/ci-source-build.sh --ci --configuration Release /p:BuildManaged=true
+    displayName: Run ci-source-build.sh
+  - task: PublishBuildArtifacts@1
+    displayName: Upload logs
+    condition: always()
+    continueOnError: true
+    inputs:
+      pathtoPublish: artifacts/log/
+      artifactName: Source_Build_Logs
+      artifactType: Container
+      parallel: true
+  - task: PublishBuildArtifacts@1
+    displayName: Upload package artifacts
+    condition: always()
+    inputs:
+      pathtoPublish: artifacts/packages/
+      artifactName: Source_Build_Packages
+      artifactType: Container
+      parallel: true

Directory.Build.targets

@@ -14,6 +14,9 @@
       By default, assemblies which are only in the Microsoft.AspNetCore.App shared framework are not available as NuGet packages.
     -->
     <IsPackable Condition="'$(IsAspNetCoreApp)' == 'true' AND '$(IsShippingPackage)' != 'true'">false</IsPackable>
+
+    <!-- Only build assemblies in Microsoft.AspNetCore.App in source build -->
+    <ExcludeFromSourceBuild Condition="'$(ExcludeFromSourceBuild)' == '' and '$(DotNetBuildFromSource)' == 'true' and '$(IsAspNetCoreApp)' != 'true'">true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />

eng/Build.props

@@ -121,6 +121,12 @@
           Use caution to avoid deep recursion. If the globbing pattern picks up something which exceeds MAX_PATH,
           the entire pattern will silently fail to evaluate correctly.
         -->
+
+        <!-- Tool projects except FirstRunCertGenerator are not neede in source build -->
+        <ProjectToExclude Condition="'$(DotNetBuildFromSource)' == 'true'"
+                          Include="$(RepoRoot)src\Tools\**\*.*proj"
+                          Exclude="$(RepoRoot)src\Tools\FirstRunCertGenerator\Microsoft.AspNetCore.DeveloperCertificates.XPlat.csproj"/>
+
         <DotNetProjects Include="
                           $(RepoRoot)src\Framework\ref\Microsoft.AspNetCore.App.Ref.csproj;
                           $(RepoRoot)src\Framework\src\Microsoft.AspNetCore.App.Runtime.csproj;
@@ -148,8 +154,8 @@
                           $(RepoRoot)src\Analyzers\**\*.csproj;
                           $(RepoRoot)src\ProjectTemplates\*\*.csproj;
                           $(RepoRoot)src\ProjectTemplates\testassets\*\*.csproj;
-                          "
-                        Exclude="
+                          " />
+        <DotNetProjects Remove="
                           @(ProjectToBuild);
                           @(ProjectToExclude);
                           $(RepoRoot)**\node_modules\**\*;

eng/Dependencies.props

@@ -21,10 +21,15 @@
   </ItemGroup>
 
   <!-- Workaround https://github.com/aspnet/AspNetCore/issues/7503. This chains GenerateSourceLinkFile before razor component targets run. -->
-  <Target Name="_EnsureSourceLinkHappensBeforeRazorComponentGeneration"
+  <!-- Workaround https://github.com/dotnet/source-build/issues/1112. Source link is currently disabled in source build so do not apply this worksaround. -->
+  <Target Condition="'$(DotNetBuildFromSource)' != 'true'"
+          Name="_EnsureSourceLinkHappensBeforeRazorComponentGeneration"
           BeforeTargets="PrepareForRazorComponentGenerate"
           DependsOnTargets="GenerateSourceLinkFile" />
 
+  <!-- Workaround https://github.com/dotnet/source-build/issues/1112. Source link is currently disabled in source build so define this dummy target which is required for pack. -->
+  <Import Condition="'$(DotNetBuildFromSource)' == 'true'" Project="WorkaroundsImported.targets" />
+
   <!-- Workaround https://github.com/aspnet/websdk/pull/646. If merged, once we update to a websdk with this fix, we can move the setting below to Directory.Build.props. -->
   <PropertyGroup>
     <!-- Ignore warning about calling the Pack target on Web SDK projects. Our build scripts call /t:pack on everything in this repo. -->

eng/Workarounds.targets

@@ -0,0 +1,5 @@
+<!-- Use this file to workaround issues. List the issue tracking the item to fix so we can remove the workaround when the issue is resolved. -->
+<Project>
+  <!-- Workaround https://github.com/dotnet/source-build/issues/1112. Source link is currently disabled in source build so define this dummy target which is required for pack. -->
+  <Target Name="InitializeSourceControlInformation"/>
+</Project>

eng/WorkaroundsImported.targets

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+ #
+# This script is meant for testing source build by imitating some of the input parameters and conditions.
+#
+
+set -euo pipefail
+
+scriptroot="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+reporoot="$(dirname "$(dirname "$scriptroot")")"
+
+ # For local development, make a backup copy of this file first
+if [ ! -f "$reporoot/global.bak.json" ]; then
+    mv "$reporoot/global.json" "$reporoot/global.bak.json"
+fi
+
+ # Detect the current version of .NET Core installed
+export SDK_VERSION=$(dotnet --version)
+echo "The ambient version of .NET Core SDK version = $SDK_VERSION"
+
+ # Update the global.json file to match the current .NET environment
+cat "$reporoot/global.bak.json" | \
+    jq '.sdk.version=env.SDK_VERSION' | \
+    jq '.tools.dotnet=env.SDK_VERSION' | \
+    jq 'del(.tools.runtimes)' \
+    > "$reporoot/global.json"
+
+ # Restore the original global.json file
+trap "{
+    mv "$reporoot/global.bak.json" "$reporoot/global.json"
+}" EXIT
+
+export DotNetBuildFromSource='true'
+
+ # Build repo tasks
+"$reporoot/eng/common/build.sh" --restore --build --ci --configuration Release /p:ProjectToBuild=$reporoot/eng/tools/RepoTasks/RepoTasks.csproj
+
+ # Build projects
+"$reporoot/eng/common/build.sh" --restore --build --pack "$@"

eng/scripts/ci-source-build.sh

@@ -11,7 +11,7 @@
     <!-- Required to exist in the NuGet package cache to enable code-signing. -->
     <PackageReference Include="MicroBuild.Core" Version="0.3.0" PrivateAssets="All" AllowExplicitReference="true" ExcludeAssets="All" />
 
-    <PackageReference Include="Microsoft.DotNet.GenAPI" PrivateAssets="All" Version="$(MicrosoftDotNetGenApiPackageVersion)" IsImplicitlyDefined="true" />
+    <PackageReference Condition="'$(DotNetBuildFromSource)' != 'true'" Include="Microsoft.DotNet.GenAPI" PrivateAssets="All" Version="$(MicrosoftDotNetGenApiPackageVersion)" IsImplicitlyDefined="true" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(UsingMicrosoftNETSdkWeb)' == 'true' OR '$(RazorSdkCurrentVersionProps)' != ''">

eng/targets/CSharp.Common.props

@@ -19,6 +19,8 @@
 <Project>
 
   <PropertyGroup>
+    <EnableCustomReferenceResolution Condition="'$(EnableCustomReferenceResolution)' == '' AND ('$(DotNetBuildFromSource)' != 'true' OR '$(ExcludeFromSourceBuild)' != 'true')">true</EnableCustomReferenceResolution>
+
     <ResolveReferencesDependsOn>
       ResolveCustomReferences;
       $(ResolveReferencesDependsOn);
@@ -60,7 +62,7 @@
     </Reference>
   </ItemDefinitionGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(EnableCustomReferenceResolution)' == 'true'">
     <Reference Update="@(Reference)">
       <IsSharedSource Condition="'%(IsSharedSource)' == '' AND $([System.String]::new('%(Identity)').EndsWith('.Sources'))">true</IsSharedSource>
     </Reference>
@@ -129,7 +131,7 @@
     This target resolves remaining Referene items to Packages, if possible. If not, they are left as Reference items fo the SDK to resolve.
     This executes on NuGet restore and during DesignTimeBuild. It should not run in the outer, cross-targeting build.
    -->
-  <Target Name="ResolveCustomReferences" BeforeTargets="CollectPackageReferences;ResolveAssemblyReferencesDesignTime;ResolveAssemblyReferences" Condition=" '$(TargetFramework)' != '' ">
+  <Target Name="ResolveCustomReferences" BeforeTargets="CollectPackageReferences;ResolveAssemblyReferencesDesignTime;ResolveAssemblyReferences" Condition=" '$(TargetFramework)' != '' AND '$(EnableCustomReferenceResolution)' == 'true' ">
     <ItemGroup>
       <!-- Ensure only content asset are consumed from .Sources packages -->
       <Reference>

eng/targets/ResolveReferences.targets

@@ -7,6 +7,8 @@
     <Optimize>false</Optimize>
     <DebugType>embedded</DebugType>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+    <!-- Need to build this project in source build -->
+    <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>