Skip to content

Commit ea38eba

Browse files
MilenaHristovammitcheViktorHofer
authored
Join assets from verticals in the final join point (#43627)
Co-authored-by: Matt Mitchell <[email protected]> Co-authored-by: Viktor Hofer <[email protected]>
1 parent ce0abad commit ea38eba

File tree

12 files changed

+755
-3
lines changed

12 files changed

+755
-3
lines changed

‎eng/pipelines/templates/jobs/vmr-build.yml‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,13 @@ jobs:
168168
displayName: Publish Artifacts
169169
sbomEnabled: true
170170

171+
# Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
172+
- output: buildArtifacts
173+
PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
174+
ArtifactName: VerticalManifests
175+
displayName: Publish Vertical Manifest
176+
sbomEnabled: false
177+
171178
- ${{ if not(parameters.isBuiltFromVmr) }}:
172179
- output: pipelineArtifact
173180
displayName: Upload failed patches
@@ -271,7 +278,7 @@ jobs:
271278
- script: |
272279
set extraBuildProperties=
273280
if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }}
274-
call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} %extraBuildProperties% ${{ parameters.extraProperties }}
281+
call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }}
275282
displayName: Build
276283
workingDirectory: ${{ variables.sourcesPath }}
277284
env:
@@ -282,7 +289,7 @@ jobs:
282289
283290
- ${{ if eq(parameters.runTests, 'True') }}:
284291
- script: |
285-
call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
292+
call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }}
286293
displayName: Run Tests
287294
workingDirectory: ${{ variables.sourcesPath }}
288295
timeoutInMinutes: ${{ variables.runTestsTimeout }}
@@ -380,6 +387,8 @@ jobs:
380387
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
381388
fi
382389
390+
extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
391+
383392
buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
384393
385394
# Only use Docker when a container is specified
@@ -455,6 +464,8 @@ jobs:
455464
customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
456465
fi
457466
467+
extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
468+
458469
if [[ -n "${{ parameters.extraProperties }}" ]]; then
459470
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
460471
fi
@@ -585,6 +596,19 @@ jobs:
585596
TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
586597
displayName: Copy artifacts to Artifact Staging Directory
587598

599+
- ${{ if eq(parameters.targetOS, 'windows') }}:
600+
- powershell: |
601+
$sourcePath = "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml"
602+
$targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
603+
New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null
604+
Copy-Item $sourcePath -Destination $targetPath -Force
605+
displayName: Copy vertical manifest to Artifact Staging Directory
606+
- ${{ else }}:
607+
- script: |
608+
mkdir -p "$(Build.ArtifactStagingDirectory)/manifests"
609+
cp "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
610+
displayName: Copy vertical manifest to Artifact Staging Directory
611+
588612
# When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
589613
- ${{ if ne(parameters.buildSourceOnly, 'true') }}:
590614
- task: CopyFiles@2
@@ -598,3 +622,11 @@ jobs:
598622
artifact: $(Agent.JobName)_Artifacts
599623
displayName: Publish Artifacts
600624
continueOnError: true
625+
626+
# Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
627+
- task: PublishBuildArtifacts@1
628+
inputs:
629+
PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
630+
ArtifactName: VerticalManifests
631+
displayName: Publish Vertical Manifest
632+
condition: succeededOrFailed()

‎eng/pipelines/templates/stages/vmr-final-join.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
parameters:
2+
# Branch of the VMR to use (to push to for internal builds)
3+
- name: vmrBranch
4+
type: string
5+
default: $(Build.SourceBranch)
6+
7+
- name: pool_Windows
8+
type: object
9+
default:
10+
name: $(defaultPoolName)
11+
image: $(poolImage_Windows)
12+
demands: ImageOverride -equals $(poolImage_Windows)
13+
os: windows
14+
15+
stages:
16+
- stage: VMR_Final_Join
17+
displayName: VMR Final Join
18+
dependsOn: VMR_Vertical_Build
19+
condition: succeededOrFailed()
20+
variables:
21+
- template: ../variables/vmr-build.yml
22+
parameters:
23+
vmrBranch: ${{ parameters.vmrBranch }}
24+
25+
jobs:
26+
- job: FinalJoin
27+
displayName: Final Build Pass
28+
pool: ${{ parameters.pool_Windows }}
29+
timeoutInMinutes: 240
30+
templateContext:
31+
outputs:
32+
- output: buildArtifacts
33+
PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/MergedManifest.xml
34+
ArtifactName: AssetManifests
35+
displayName: Publish Merged Manifest
36+
sbomEnabled: false
37+
- output: buildArtifacts
38+
PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/assets
39+
ArtifactName: BlobArtifacts
40+
displayName: Publish Blob Artifacts
41+
sbomEnabled: false
42+
- output: buildArtifacts
43+
PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/packages
44+
ArtifactName: PackageArtifacts
45+
displayName: Publish Package Artifacts
46+
sbomEnabled: false
47+
steps:
48+
- template: ../steps/vmr-join-verticals.yml
49+
parameters:
50+
dotNetBuildPass: final
51+
primaryDependentJob: Windows_x64
52+
outputFolder: $(Build.ArtifactStagingDirectory)/artifacts

‎eng/pipelines/templates/steps/vmr-join-verticals.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
parameters:
2+
- name: dotNetBuildPass
3+
type: string
4+
default: final
5+
6+
- name: primaryDependentJob
7+
type: string
8+
default: Windows_x64
9+
10+
- name: outputFolder
11+
type: string
12+
default: $(Build.ArtifactStagingDirectory)/artifacts
13+
14+
steps:
15+
- task: DownloadBuildArtifacts@1
16+
inputs:
17+
artifactName: 'VerticalManifests'
18+
downloadPath: $(Build.ArtifactStagingDirectory)
19+
checkDownloadedFiles: true
20+
21+
- task: DownloadPipelineArtifact@2
22+
inputs:
23+
artifactName: ${{ parameters.primaryDependentJob }}_Artifacts
24+
targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
25+
checkDownloadedFiles: true
26+
27+
- powershell: eng/join-verticals.ps1
28+
/p:VerticalManifestsPath=$(Build.ArtifactStagingDirectory)/VerticalManifests
29+
/p:MainVertical=${{ parameters.primaryDependentJob }}
30+
/p:DotNetBuildPass=${{ parameters.dotNetBuildPass }}
31+
/p:BuildId=$(Build.BuildId)
32+
/p:AzureDevOpsToken=$(System.AccessToken)
33+
/p:AzureDevOpsBaseUri=$(System.CollectionUri)
34+
/p:AzureDevOpsProject=$(System.TeamProject)
35+
/p:MainVerticalArtifactsFolder=$(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
36+
/p:OutputFolder=${{ parameters.outputFolder }}
37+
displayName: Join Verticals

‎src/SourceBuild/content/eng/join-verticals.proj‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<Project Sdk="Microsoft.Build.Traversal">
2+
3+
<PropertyGroup>
4+
<TargetFramework>$(NetCurrent)</TargetFramework>
5+
</PropertyGroup>
6+
7+
<ItemGroup>
8+
<ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" BuildInParallel="true" />
9+
</ItemGroup>
10+
11+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
12+
<Target Name="JoinVerticals" DependsOnTargets="ResolveProjectReferences" AfterTargets="Build">
13+
<Error Condition="'$(MainVertical)' == ''" Text="MainVertical is not set." />
14+
<Error Condition="'$(VerticalManifestsPath)' == ''" Text="VerticalManifestsPath is not set." />
15+
<Error Condition="'$(BuildId)' == ''" Text="BuildId is not set." />
16+
<Error Condition="'$(AzureDevOpsBaseUri)' == ''" Text="AzureDevOpsBaseUri is not set." />
17+
<Error Condition="'$(AzureDevOpsProject)' == ''" Text="AzureDevOpsProject is not set." />
18+
<Error Condition="'$(MainVerticalArtifactsFolder)' == ''" Text="MainVerticalArtifactsFolder is not set." />
19+
<Error Condition="'$(OutputFolder)' == ''" Text="OutputFolder is not set." />
20+
21+
<ItemGroup>
22+
<VerticalManifest
23+
Include="$(VerticalManifestsPath)\*.xml"
24+
Exclude="$(VerticalManifestsPath)\*Mono*.xml;$(VerticalManifestsPath)\*Pgo*.xml;$(VerticalManifestsPath)\*Shortstack*.xml;$(VerticalManifestsPath)\*DevVersions*.xml" />
25+
</ItemGroup>
26+
27+
<!-- AzureDevOpsToken shouldn't be set when running in dnceng-public -->
28+
<Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals
29+
VerticalManifest="@(VerticalManifest)"
30+
MainVertical="$(MainVertical)"
31+
BuildId="$(BuildId)"
32+
AzureDevOpsToken="$(AzureDevOpsToken)"
33+
AzureDevOpsBaseUri="$(AzureDevOpsBaseUri)"
34+
AzureDevOpsProject="$(AzureDevOpsProject)"
35+
MainVerticalArtifactsFolder="$(MainVerticalArtifactsFolder)"
36+
OutputFolder="$(OutputFolder)" />
37+
</Target>
38+
39+
</Project>

‎src/SourceBuild/content/eng/join-verticals.ps1‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
[CmdletBinding(PositionalBinding=$false)]
2+
Param(
3+
[string][Alias('v')]$verbosity = "minimal",
4+
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
5+
)
6+
7+
$useGlobalNuGetCache=$false
8+
$ci = $true
9+
10+
. $PSScriptRoot\common\tools.ps1
11+
12+
$project = Join-Path $EngRoot "join-verticals.proj"
13+
$arguments = @()
14+
$targets = "/t:JoinVerticals"
15+
16+
try {
17+
$bl = '/bl:' + (Join-Path $LogDir 'JoinVerticals.binlog')
18+
19+
MSBuild -restore `
20+
$project `
21+
$bl `
22+
$targets `
23+
/p:Configuration=Release `
24+
@properties `
25+
@arguments
26+
}
27+
catch {
28+
Write-Host $_.ScriptStackTrace
29+
Write-PipelineTelemetryError -Category 'Build' -Message $_
30+
ExitWithExitCode 1
31+
}
32+
33+
ExitWithExitCode 0

‎src/SourceBuild/content/eng/merge-asset-manifests.proj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
<Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests
2323
AssetManifest="@(RepoAssetManifest)"
2424
MergedAssetManifestOutputPath="$(MergedAssetManifestOutputPath)"
25-
VmrBuildNumber="$(BUILD_BUILDNUMBER)" />
25+
VmrBuildNumber="$(BUILD_BUILDNUMBER)"
26+
VerticalName="$(VerticalName)" />
2627
</Target>
2728

2829
</Project>

‎src/SourceBuild/content/eng/pipelines/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,6 @@ extends:
9595
scope: lite
9696
${{ else }}:
9797
scope: full
98+
99+
- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
100+
- template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self

‎src/SourceBuild/content/eng/pipelines/pr.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,6 @@ stages:
4848
scope: lite
4949
${{ else }}:
5050
scope: full
51+
52+
- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
53+
- template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self

0 commit comments

Comments
 (0)