Fine tune first join point and VMR support for it

Directory.Build.targets

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!-- Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the MIT license. See License.txt in the project root for full license information. -->
 <Project>
+
   <ItemGroup>
     <SupportedPlatform Remove="Android" />
     <SupportedPlatform Remove="iOS" />
@@ -36,13 +37,18 @@
 
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 
+  <PropertyGroup>
+    <FullNugetVersion>$(VersionPrefix)-$(PreReleaseVersionLabel)</FullNugetVersion>
+    <FullNugetVersion Condition="'$(PreReleaseVersionIteration)' != ''">$(FullNugetVersion).$(PreReleaseVersionIteration)</FullNugetVersion>
+    <FullNugetVersion Condition=" '$(VersionSuffixDateStamp)' != '' And '$(VersionSuffixBuildOfTheDay)' != '' ">$(FullNugetVersion).$(VersionSuffixDateStamp).$(VersionSuffixBuildOfTheDay)</FullNugetVersion>
+  </PropertyGroup>
+
   <!-- Optionally override arcade's test target with one which will run the tests as tools.
        Conditionally overriding a target requires a conditional import of another (.targets)
        file. -->
   <Import Project="OverrideTest.targets"
           Condition="'$(RunTestsAsTool)' == 'true' And '$(CanRunTestAsTool)' == 'true'"/>
 
-
   <!-- Update KnownFrameworkReferences to target the right version of the runtime -->
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'
                           and $(MicrosoftNETCoreAppRefPackageVersion.StartsWith('$(_TargetFrameworkVersionWithoutV)'))

eng/Build.props

@@ -6,4 +6,11 @@
     <ProjectToBuild Include="$(RepoRoot)src/VirtualMonoRepo/InitializeVMR.proj" BuildInParallel="false" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildPass)' != '' and '$(DotNetBuildPass)' != '1'">
+    <!-- For product build, build SdkResolver and VSTemplateLocator only in the second build pass as they depend on
+         assets from other verticals that are built in the first build pass. -->
+    <ProjectToBuild Include="$(RepoRoot)src\Installer\redist-installer\projects\SdkResolver.csproj" DotNetBuildPass="2" />
+    <ProjectToBuild Include="$(RepoRoot)src\Installer\redist-installer\projects\VSTemplateLocator\VSTemplateLocator.csproj" DotNetBuildPass="2" />
+  </ItemGroup>
+
 </Project>

eng/DotNetBuild.props

@@ -16,8 +16,6 @@
       <InnerBuildArgs>$(InnerBuildArgs) /p:SkipBuildingInstallers=true</InnerBuildArgs>
       <InnerBuildArgs>$(InnerBuildArgs) /p:IncludeAdditionalSharedFrameworks=false</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(SourceBuildUseMonoRuntime)' == 'true'">$(InnerBuildArgs) /p:DISABLE_CROSSGEN=true</InnerBuildArgs>
-      <InnerBuildArgs Condition="'$(PgoInstrument)' == 'true'">$(InnerBuildArgs) /p:PgoInstrument=true</InnerBuildArgs>
-      <InnerBuildArgs Condition="'$(DotNetBuildPass)' != ''">$(InnerBuildArgs) /p:DotNetBuildPass=$(DotNetBuildPass)</InnerBuildArgs>
     </PropertyGroup>
   </Target>
 

eng/Publishing.props

@@ -10,7 +10,8 @@
     <Product>Sdk</Product>
     <BlobStoragePartialRelativePath>$(Product)</BlobStoragePartialRelativePath>
     <BlobStoragePartialRelativePath Condition="'$(IsNotOrchestratedPublish)' == 'false'">assets/$(Product)</BlobStoragePartialRelativePath>
-    <SdkAssetManifestFileName>$(OS)-$(PlatformName)-SdkAssets.xml</SdkAssetManifestFileName>
+    <SdkAssetManifestBuildPass Condition="'$(DotNetBuildPass)' != ''">-BuildPass$(DotNetBuildPass)</SdkAssetManifestBuildPass>
+    <SdkAssetManifestFileName>$(OS)-$(PlatformName)-SdkAssets$(SdkAssetManifestBuildPass).xml</SdkAssetManifestFileName>
     <SdkAssetsManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(SdkAssetManifestFileName)</SdkAssetsManifestFilePath>
 
     <TempWorkingDirectory>$(ArtifactsDir)\AssetsTmpDir\$([System.Guid]::NewGuid())</TempWorkingDirectory>
@@ -143,7 +144,7 @@
       <AssetManifestOS Condition="'$(SYSTEM_PHASENAME)' != ''">$(AssetManifestOS)-$(SYSTEM_PHASENAME)</AssetManifestOS>
       <BaseAssetManifestFileName>$(AssetManifestOS)</BaseAssetManifestFileName>
       <BaseAssetManifestFileName Condition="'$(SYSTEM_PHASENAME)' == '' and '$(Architecture)' != ''">$(AssetManifestOS)-$(Architecture)</BaseAssetManifestFileName>
-      <InstallersAssetManifestFileName>$(BaseAssetManifestFileName)-installers</InstallersAssetManifestFileName>
+      <InstallersAssetManifestFileName>$(BaseAssetManifestFileName)-installers$(SdkAssetManifestBuildPass)</InstallersAssetManifestFileName>
       <!-- Property AssetManifestFilePath would be reassigned by the Arcade SDK, so use a different name (InstallersAssetManifestFilePath) -->
       <InstallersAssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(InstallersAssetManifestFileName).xml</InstallersAssetManifestFilePath>
 
@@ -183,7 +184,10 @@
     <CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)*.tar.gz.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
   </ItemGroup>
 
-  <Target Name="PublishSdkAssetsAndChecksums" BeforeTargets="Publish" Condition="$(DotNetPublishUsingPipelines)">
+  <!-- Publish the sdk and the checksums only in the initial build pass. -->
+  <Target Name="PublishSdkAssetsAndChecksums"
+          BeforeTargets="Publish"
+          Condition="'$(DotNetPublishUsingPipelines)' == 'true' and ('$(DotNetBuildPass)' == '' or '$(DotNetBuildPass)' == '1')">
     <ReadLinesFromFile File="$(ArtifactsTmpDir)FullNugetVersion.version">
       <Output TaskParameter="Lines" PropertyName="FullNugetVersion" />
     </ReadLinesFromFile>

eng/pipelines/templates/jobs/vmr-build.yml

@@ -106,7 +106,7 @@ parameters:
   type: string
 
 jobs:
-- job: ${{ parameters.buildName }}_${{ parameters.architecture }}
+- job: ${{ parameters.buildName }}_${{ parameters.architecture }}${{ replace(format('_BuildPass{0}', coalesce(parameters.buildPass, '1')), '_BuildPass1', '') }}
   pool: ${{ parameters.pool }}
 
   # Currently, CodeQL slows the build down too much
@@ -117,9 +117,12 @@ jobs:
     timeoutInMinutes: 240
 
   ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
-    # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status.
-    # The build shortcuts when stage 1 build fails and doesn't produce the SDK.
-    condition: succeededOrFailed()
+    ${{ if eq(parameters.buildPass, '') }}:
+      # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status.
+      # The build shortcuts when stage 1 build fails and doesn't produce the SDK.
+      condition: succeededOrFailed()
+    ${{ else }}:
+      condition: succeeded()
     dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}
   variables:
   - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
@@ -150,13 +153,6 @@ jobs:
     - name: sourcesPath
       value: $(vmrPath)
 
-  - ${{ if ne(parameters.buildPass, '') }}:
-    - name: artifactsSuffix
-      value: '_buildpass${{ parameters.buildPass }}'
-  - ${{ else }}:
-    - name: artifactsSuffix
-      value: ''
-
   templateContext:
     outputs:
     - output: pipelineArtifact
@@ -168,7 +164,7 @@ jobs:
 
     - output: pipelineArtifact
       path: $(Build.ArtifactStagingDirectory)/publishing
-      artifact: $(Agent.JobName)_Artifacts$(artifactsSuffix)
+      artifact: $(Agent.JobName)_Artifacts
       displayName: Publish Artifacts
       sbomEnabled: true
 
@@ -207,15 +203,15 @@ jobs:
 
   - ${{ if ne(parameters.reuseBuildArtifactsFrom,'') }}:
     - ${{ each reuseBuildArtifacts in parameters.reuseBuildArtifactsFrom }}:
-      - download: current
-        artifact: ${{ reuseBuildArtifacts }}_Artifacts
-        patterns: |
-          **/Private.SourceBuilt.Artifacts.*.tar.gz
-          **/dotnet-sdk-*.tar.gz
-          **/*.nupkg
-        displayName: Download Previous Build (${{ reuseBuildArtifacts }})
-
       - ${{ if eq(parameters.buildSourceOnly, true) }}:
+        - download: current
+          artifact: ${{ reuseBuildArtifacts }}_Artifacts
+          patterns: |
+            **/Private.SourceBuilt.Artifacts.*.tar.gz
+            **/dotnet-sdk-*.tar.gz
+            **/*.nupkg
+          displayName: Download Previous Build (${{ reuseBuildArtifacts }})
+
         - task: CopyFiles@2
           displayName: Copy Previous Build (${{ reuseBuildArtifacts }} - Source Build artifacts)
           inputs:
@@ -224,13 +220,16 @@ jobs:
             TargetFolder: $(sourcesPath)/prereqs/packages/archive/
 
       - ${{ else }}:
+        - download: current
+          artifact: ${{ reuseBuildArtifacts }}_Artifacts
+          displayName: Download Previous Build (${{ reuseBuildArtifacts }})
+
         - task: CopyFiles@2
-          displayName: Copy Previous Build (${{ reuseBuildArtifacts }} - NuGet Packages)
+          displayName: Copy Previous Build (${{ reuseBuildArtifacts }})
           inputs:
-            SourceFolder: $(Pipeline.Workspace)/${{ reuseBuildArtifacts }}_Artifacts/packages
-            Contents: '**/*.nupkg'
+            SourceFolder: $(Pipeline.Workspace)/${{ reuseBuildArtifacts }}_Artifacts/
             OverWrite: false
-            TargetFolder: $(sourcesPath)/artifacts/packages/
+            TargetFolder: $(sourcesPath)/artifacts/
 
   - ${{ if eq(parameters.withPreviousSDK, 'true') }}:
     - script: |
@@ -502,11 +501,6 @@ jobs:
             CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "Updated*.txt"
             CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.trx"
         }
-
-        # check if we have assets to publish
-        if (Test-Path "artifacts/assets/Release/*") {
-          echo "##vso[task.setvariable variable=hasAssets]true"
-        }
       displayName: Prepare BuildLogs staging directory and check assets
       continueOnError: true
       condition: succeededOrFailed()
@@ -545,12 +539,6 @@ jobs:
         if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
           find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
         fi
-
-
-        # check if we have assets to publish
-        if [ -n "$(ls -A 'artifacts/assets/Release/')" ]; then
-          echo "##vso[task.setvariable variable=hasAssets]true"
-        fi
       displayName: Prepare BuildLogs staging directory and check assets
       continueOnError: true
       condition: succeededOrFailed()
@@ -592,8 +580,8 @@ jobs:
     inputs:
       SourceFolder: $(sourcesPath)/artifacts
       Contents: |
-        VerticalManifest.xml
         assets/**
+        manifests/**
       TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
     displayName: Copy artifacts to Artifact Staging Directory
 
@@ -607,6 +595,6 @@ jobs:
 
   - ${{ if or(ne(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}:
     - publish: $(Build.ArtifactStagingDirectory)/publishing
-      artifact: $(Agent.JobName)_Artifacts$(artifactsSuffix)
+      artifact: $(Agent.JobName)_Artifacts
       displayName: Publish Artifacts
       continueOnError: true

eng/pipelines/templates/stages/vmr-build.yml

@@ -391,11 +391,6 @@ stages:
         pool: ${{ parameters.pool_Windows }}
         targetOS: windows
         targetArchitecture: x64
-        ${{ if in(parameters.scope, 'full') }}:
-          buildPass: 2
-          reuseBuildArtifactsFrom:
-          - Windows_x86
-          - Windows_arm64
 
     - template: ../jobs/vmr-build.yml
       parameters:
@@ -1098,3 +1093,21 @@ stages:
             targetArchitecture: arm64
             extraProperties: /p:PgoInstrument=true
             runTests: false
+
+        # Build Pass 2 verticals
+
+        - template: ../jobs/vmr-build.yml
+          parameters:
+            buildName: Windows
+            isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
+            vmrBranch: ${{ variables.VmrBranch }}
+            architecture: x64
+            pool: ${{ parameters.pool_Windows }}
+            targetOS: windows
+            targetArchitecture: x64
+            runTests: false
+            buildPass: 2
+            reuseBuildArtifactsFrom:
+            - Windows_x64
+            - Windows_x86
+            - Windows_arm64

sdk.sln

@@ -500,7 +500,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "redist-installer", "src\Ins
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SdkResolver", "src\Installer\redist-installer\projects\SdkResolver.csproj", "{27E399C0-6F91-45D6-A63D-041DA41CCFDF}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VSTemplateLocator", "src\Installer\redist-installer\projects\VSTemplateLocator.csproj", "{0CBA5FB8-71A3-457A-89F3-E52B9602164A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VSTemplateLocator", "src\Installer\redist-installer\projects\VSTemplateLocator\VSTemplateLocator.csproj", "{0CBA5FB8-71A3-457A-89F3-E52B9602164A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "core-sdk-tasks.Tests", "test\core-sdk-tasks.Tests\core-sdk-tasks.Tests.csproj", "{21C21975-84C1-4A24-8E21-F7EC790A4584}"
 EndProject

src/Installer/redist-installer/projects/SdkResolver.csproj

@@ -7,7 +7,9 @@
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
     <GenerateRuntimeConfigurationFiles>false</GenerateRuntimeConfigurationFiles>
     <ExcludeFromSourceOnlyBuild>true</ExcludeFromSourceOnlyBuild>
-    <ExcludeFromDotNetBuild Condition="'$(DotNetBuildPass)' == ''">true</ExcludeFromDotNetBuild>
+    <!-- For product build, this project only builds in the second build pass as it depends on assets other
+         verticals that are built in the first build pass. -->
+    <ExcludeFromDotNetBuild Condition="'$(DotNetBuildPass)' != '2'">true</ExcludeFromDotNetBuild>
   </PropertyGroup>
 
   <ItemGroup>

src/Installer/redist-installer/projects/VSTemplateLocator.csproj → src/Installer/redist-installer/projects/VSTemplateLocator/VSTemplateLocator.csproj

@@ -1,13 +1,13 @@
-<Project Sdk="Microsoft.Build.NoTargets/3.7.0">
+<Project Sdk="Microsoft.Build.NoTargets">
 
   <PropertyGroup>
     <TargetFramework>$(SdkTargetFramework)</TargetFramework>
     <CopyBuildOutputToOutputDirectory>false</CopyBuildOutputToOutputDirectory>
-    <GenerateDependencyFile>false</GenerateDependencyFile>
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
-    <GenerateRuntimeConfigurationFiles>false</GenerateRuntimeConfigurationFiles>
     <ExcludeFromSourceOnlyBuild>true</ExcludeFromSourceOnlyBuild>
-    <ExcludeFromDotNetBuild Condition="'$(DotNetBuildPass)' == ''">true</ExcludeFromDotNetBuild>
+    <!-- For product build, this project only builds in the second build pass as it depends on assets other
+         verticals that are built in the first build pass. -->
+    <ExcludeFromDotNetBuild Condition="'$(DotNetBuildPass)' != '2'">true</ExcludeFromDotNetBuild>
   </PropertyGroup>
 
   <ItemGroup>
@@ -21,10 +21,13 @@
     <ProjectReference Include="$(RepoRoot)src\Microsoft.DotNet.TemplateLocator\Microsoft.DotNet.TemplateLocator.csproj" ReferenceOutputAssembly="false" />
   </ItemGroup>
 
-  <Target Name="GenerateLayout"
-          Condition="'$(OS)' == 'Windows_NT' and '$(_SuppressAllTargets)' != 'true'"
-          BeforeTargets="AfterBuild">
-    <Message Importance="High" Text="$(MSBuildProjectName) -&gt; $(OutputPath)" />
+  <Target Name="GenerateLayout">
+    <!-- Set these here early enough for inputs & outputs in the GenerateVSTemplateLocatorNupkg target. -->
+    <PropertyGroup>
+      <VSTemplateLocatorNuspecFile>$(MSBuildThisFileDirectory)VS.Redist.Common.Net.Core.SDK.VSTemplateLocator.nuspec</VSTemplateLocatorNuspecFile>
+      <VSTemplateLocatorNupkgFile>$(ArtifactsNonShippingPackagesDir)VS.Redist.Common.Net.Core.SDK.VSTemplateLocator.$(FullNugetVersion).nupkg</VSTemplateLocatorNupkgFile>
+      <GenerateNupkgPowershellScript>$(MSBuildThisFileDirectory)../../packaging/windows/generatenupkg.ps1</GenerateNupkgPowershellScript>
+    </PropertyGroup>
 
     <RemoveDir Directories="$(OutputPath)" />
     <MakeDir Directories="$(OutputPath)" />
@@ -41,6 +44,30 @@
     </ItemGroup>
 
     <Copy SourceFiles="@(_VSTemplateLocatorSrc)" DestinationFiles="@(_VSTemplateLocatorDst)" />
+
+    <Message Importance="High" Text="$(MSBuildProjectName) -&gt; $(OutputPath)" />
+  </Target>
+
+  <!-- For product build, this only builds in the second build pass as it depends on assets
+       from other verticals that are built in the first build pass. -->
+  <Target Name="GenerateVSTemplateLocatorNupkg"
+          DependsOnTargets="GenerateLayout"
+          Condition="'$(OS)' == 'Windows_NT' and
+                     '$(Architecture)' == 'x64' and
+                     '$(PgoInstrument)' != 'true' and
+                     ('$(DotNetBuild)' != 'true' or '$(DotNetBuildPass)' == '2') and
+                     '$(_SuppressAllTargets)' != 'true'"
+          Inputs="$(OutputPath)/**/*;
+                  $(VSTemplateLocatorNuspecFile);
+                  $(GenerateNupkgPowershellScript)"
+          Outputs="$(VSTemplateLocatorNupkgFile)"
+          AfterTargets="Build">
+    <Exec Command="powershell -NoProfile -NoLogo $(GenerateNupkgPowershellScript) ^
+                      '$(ArtifactsDir)' ^
+                      '$(OutputPath.TrimEnd('\'))' ^
+                      '$(FullNugetVersion)' ^
+                      '$(VSTemplateLocatorNuspecFile)' ^
+                      '$(VSTemplateLocatorNupkgFile)'" />
   </Target>
 
 </Project>

src/Installer/redist-installer/redist-installer.csproj

@@ -19,8 +19,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Layout\redist\redist.csproj" ReferenceOutputAssembly="false" />
     <ProjectReference Include="$(RepoRoot)template_feed\*\*.csproj" ReferenceOutputAssembly="false" />
-    <ProjectReference Include="projects\SdkResolver.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuild)' != 'true' or '$(DotNetBuildPass)' != ''" />
-    <ProjectReference Include="projects\VSTemplateLocator.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuild)' != 'true' or '$(DotNetBuildPass)' != ''" />
+    <ProjectReference Include="projects\SdkResolver.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuild)' != 'true'" />
   </ItemGroup>
 
   <ItemGroup>
@@ -30,7 +29,6 @@
   <Import Project="targets\BuildCoreSdkTasks.targets" />
   <Import Project="targets\GetRuntimeInformation.targets" />
   <Import Project="targets\SetBuildDefaults.targets" />
-  <Import Project="targets\Versions.targets" />
   <Import Project="targets\Branding.targets" />
   <Import Project="targets\BundledTemplates.targets" />
   <Import Project="targets\BundledManifests.targets" />

src/Installer/redist-installer/targets/GenerateLayout.targets

@@ -511,7 +511,7 @@
   </Target>
 
   <Target Name="GenerateVersionFile"
-          DependsOnTargets="SetupBundledComponents;GenerateFullNuGetVersion">
+          DependsOnTargets="SetupBundledComponents">
     <WriteLinesToFile File="$(SdkOutputDirectory).version"
                       Lines="$(SourceRevisionId);$(Version);$(Rid);$(FullNugetVersion);$(SdkFeatureBand)"
                       Overwrite="true" />
@@ -611,7 +611,6 @@
           DependsOnTargets="DownloadBundledComponents;
                             CleanLayoutPath;
                             LayoutBundledComponents;
-                            GenerateFullNuGetVersion;
                             GenerateVersionFile;
                             CopyKnownWorkloadManifestFile;
                             GenerateBundledVersions;