diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index d204c827fe2d..24f93dfcf380 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -168,6 +168,13 @@ jobs: displayName: Publish Artifacts sbomEnabled: true + # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml + ArtifactName: VerticalManifests + displayName: Publish Vertical Manifest + sbomEnabled: false + - ${{ if not(parameters.isBuiltFromVmr) }}: - output: pipelineArtifact displayName: Upload failed patches @@ -271,7 +278,7 @@ jobs: - script: | set extraBuildProperties= if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }} - call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} %extraBuildProperties% ${{ parameters.extraProperties }} + call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }} displayName: Build workingDirectory: ${{ variables.sourcesPath }} env: @@ -282,7 +289,7 @@ jobs: - ${{ if eq(parameters.runTests, 'True') }}: - script: | - call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }} + call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }} displayName: Run Tests workingDirectory: ${{ variables.sourcesPath }} timeoutInMinutes: ${{ variables.runTestsTimeout }} @@ -380,6 +387,8 @@ jobs: extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}" fi + extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)" + buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties" # Only use Docker when a container is specified @@ -455,6 +464,8 @@ jobs: customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}" fi + extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)" + if [[ -n "${{ parameters.extraProperties }}" ]]; then extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}" fi @@ -585,6 +596,19 @@ jobs: TargetFolder: $(Build.ArtifactStagingDirectory)/publishing displayName: Copy artifacts to Artifact Staging Directory + - ${{ if eq(parameters.targetOS, 'windows') }}: + - powershell: | + $sourcePath = "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" + $targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" + New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null + Copy-Item $sourcePath -Destination $targetPath -Force + displayName: Copy vertical manifest to Artifact Staging Directory + - ${{ else }}: + - script: | + mkdir -p "$(Build.ArtifactStagingDirectory)/manifests" + cp "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" + displayName: Copy vertical manifest to Artifact Staging Directory + # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages - ${{ if ne(parameters.buildSourceOnly, 'true') }}: - task: CopyFiles@2 @@ -598,3 +622,11 @@ jobs: artifact: $(Agent.JobName)_Artifacts displayName: Publish Artifacts continueOnError: true + + # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml + ArtifactName: VerticalManifests + displayName: Publish Vertical Manifest + condition: succeededOrFailed() diff --git a/eng/pipelines/templates/stages/vmr-final-join.yml b/eng/pipelines/templates/stages/vmr-final-join.yml new file mode 100644 index 000000000000..947e045c91ea --- /dev/null +++ b/eng/pipelines/templates/stages/vmr-final-join.yml @@ -0,0 +1,52 @@ +parameters: +# Branch of the VMR to use (to push to for internal builds) +- name: vmrBranch + type: string + default: $(Build.SourceBranch) + +- name: pool_Windows + type: object + default: + name: $(defaultPoolName) + image: $(poolImage_Windows) + demands: ImageOverride -equals $(poolImage_Windows) + os: windows + +stages: +- stage: VMR_Final_Join + displayName: VMR Final Join + dependsOn: VMR_Vertical_Build + condition: succeededOrFailed() + variables: + - template: ../variables/vmr-build.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + + jobs: + - job: FinalJoin + displayName: Final Build Pass + pool: ${{ parameters.pool_Windows }} + timeoutInMinutes: 240 + templateContext: + outputs: + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/MergedManifest.xml + ArtifactName: AssetManifests + displayName: Publish Merged Manifest + sbomEnabled: false + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/assets + ArtifactName: BlobArtifacts + displayName: Publish Blob Artifacts + sbomEnabled: false + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/packages + ArtifactName: PackageArtifacts + displayName: Publish Package Artifacts + sbomEnabled: false + steps: + - template: ../steps/vmr-join-verticals.yml + parameters: + dotNetBuildPass: final + primaryDependentJob: Windows_x64 + outputFolder: $(Build.ArtifactStagingDirectory)/artifacts \ No newline at end of file diff --git a/eng/pipelines/templates/steps/vmr-join-verticals.yml b/eng/pipelines/templates/steps/vmr-join-verticals.yml new file mode 100644 index 000000000000..50b1866b098f --- /dev/null +++ b/eng/pipelines/templates/steps/vmr-join-verticals.yml @@ -0,0 +1,37 @@ +parameters: +- name: dotNetBuildPass + type: string + default: final + +- name: primaryDependentJob + type: string + default: Windows_x64 + +- name: outputFolder + type: string + default: $(Build.ArtifactStagingDirectory)/artifacts + +steps: +- task: DownloadBuildArtifacts@1 + inputs: + artifactName: 'VerticalManifests' + downloadPath: $(Build.ArtifactStagingDirectory) + checkDownloadedFiles: true + +- task: DownloadPipelineArtifact@2 + inputs: + artifactName: ${{ parameters.primaryDependentJob }}_Artifacts + targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts + checkDownloadedFiles: true + +- powershell: eng/join-verticals.ps1 + /p:VerticalManifestsPath=$(Build.ArtifactStagingDirectory)/VerticalManifests + /p:MainVertical=${{ parameters.primaryDependentJob }} + /p:DotNetBuildPass=${{ parameters.dotNetBuildPass }} + /p:BuildId=$(Build.BuildId) + /p:AzureDevOpsToken=$(System.AccessToken) + /p:AzureDevOpsBaseUri=$(System.CollectionUri) + /p:AzureDevOpsProject=$(System.TeamProject) + /p:MainVerticalArtifactsFolder=$(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts + /p:OutputFolder=${{ parameters.outputFolder }} + displayName: Join Verticals \ No newline at end of file diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj new file mode 100644 index 000000000000..561464a10822 --- /dev/null +++ b/src/SourceBuild/content/eng/join-verticals.proj @@ -0,0 +1,39 @@ + + + + $(NetCurrent) + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/SourceBuild/content/eng/join-verticals.ps1 b/src/SourceBuild/content/eng/join-verticals.ps1 new file mode 100644 index 000000000000..bd19e2617cbe --- /dev/null +++ b/src/SourceBuild/content/eng/join-verticals.ps1 @@ -0,0 +1,33 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string][Alias('v')]$verbosity = "minimal", + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties +) + +$useGlobalNuGetCache=$false +$ci = $true + +. $PSScriptRoot\common\tools.ps1 + +$project = Join-Path $EngRoot "join-verticals.proj" +$arguments = @() +$targets = "/t:JoinVerticals" + +try { + $bl = '/bl:' + (Join-Path $LogDir 'JoinVerticals.binlog') + + MSBuild -restore ` + $project ` + $bl ` + $targets ` + /p:Configuration=Release ` + @properties ` + @arguments +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Build' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/src/SourceBuild/content/eng/merge-asset-manifests.proj b/src/SourceBuild/content/eng/merge-asset-manifests.proj index e1b5b053b4ad..ae49332985b3 100644 --- a/src/SourceBuild/content/eng/merge-asset-manifests.proj +++ b/src/SourceBuild/content/eng/merge-asset-manifests.proj @@ -22,7 +22,8 @@ + VmrBuildNumber="$(BUILD_BUILDNUMBER)" + VerticalName="$(VerticalName)" /> diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index 33b41f2c9163..e00f5bc76fa5 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -95,3 +95,6 @@ extends: scope: lite ${{ else }}: scope: full + + - ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}: + - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self \ No newline at end of file diff --git a/src/SourceBuild/content/eng/pipelines/pr.yml b/src/SourceBuild/content/eng/pipelines/pr.yml index 471f6a92b751..d1b51c7f322f 100644 --- a/src/SourceBuild/content/eng/pipelines/pr.yml +++ b/src/SourceBuild/content/eng/pipelines/pr.yml @@ -48,3 +48,6 @@ stages: scope: lite ${{ else }}: scope: full + +- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}: + - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs new file mode 100644 index 000000000000..d28e7ca24e00 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs @@ -0,0 +1,199 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Threading.Tasks; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.UnifiedBuild.Tasks; + +public class AzureDevOpsClient : IDisposable +{ + private readonly HttpClient _httpClient; + private readonly TaskLoggingHelper _logger; + + private const string _azureDevOpsApiVersion = "7.1-preview.5"; + // download in 100 MB chunks + private const int _downloadBufferSize = 1024 * 1024 * 100; + private const int _httpTimeoutSeconds = 300; + + public AzureDevOpsClient( + string? azureDevOpsToken, + string azureDevOpsBaseUri, + string azureDevOpsProject, + TaskLoggingHelper logger) + { + + _logger = logger; + + _httpClient = new(new HttpClientHandler { CheckCertificateRevocationList = true }); + + _httpClient.BaseAddress = new Uri($"{azureDevOpsBaseUri}/{azureDevOpsProject}/_apis/"); + + _httpClient.Timeout = TimeSpan.FromSeconds(_httpTimeoutSeconds); + + if (!string.IsNullOrEmpty(azureDevOpsToken)) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes($":{azureDevOpsToken}"))); + } + } + + /// + /// Downloads a build artifact as zip file + /// + public async Task DownloadArtifactZip(string buildId, string artifactName, string downloadPath, int retryCount) + { + var artifactInformation = await GetArtifactInformation(buildId, artifactName, retryCount); + string downloadUrl = artifactInformation.Resource.DownloadUrl; + + _logger.LogMessage(MessageImportance.High, $"Downloading artifact zip from {downloadUrl}"); + + try + { + using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(downloadUrl, retryCount); + using Stream readStream = await httpResponse.Content.ReadAsStreamAsync(); + using FileStream writeStream = File.Create(downloadPath); + + await readStream.CopyToAsync(writeStream, _downloadBufferSize); + } + catch (Exception ex) + { + _logger.LogError($"Failed to download artifact zip: {ex.Message}"); + throw; + } + } + + public async Task DownloadSingleFileFromArtifact(string buildId, string artifactName, string itemId, string itemSubPath, string downloadPath, int retryCount) + { + try + { + var downloadFileUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&fileId={itemId}&fileName={itemSubPath}&api-version={_azureDevOpsApiVersion}"; + + _logger.LogMessage(MessageImportance.High, $"Downloading file {itemSubPath} from {downloadFileUrl}"); + + using HttpResponseMessage fileDownloadResponse = await ExecuteApiCallWithRetry(downloadFileUrl, retryCount); + using Stream readStream = await fileDownloadResponse.Content.ReadAsStreamAsync(); + using FileStream writeStream = File.Create(downloadPath); + + await readStream.CopyToAsync(writeStream, _downloadBufferSize); + } + catch (Exception ex) + { + _logger.LogError($"Failed to download file: {ex.Message}"); + throw; + } + } + + public async Task GetArtifactFilesInformation(string buildId, string artifactName, int retryCount) + { + var artifactInformation = await GetArtifactInformation(buildId, artifactName, retryCount); + string artifactId = artifactInformation.Resource.Data; + + var getManifestUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&fileId={artifactId}&fileName={artifactName}&api-version={_azureDevOpsApiVersion}"; + + _logger.LogMessage(MessageImportance.High, $"Getting {artifactName} artifact manifest"); + + try + { + using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(getManifestUrl, retryCount); + + ArtifactFiles filesInformation = await httpResponse.Content.ReadFromJsonAsync() + ?? throw new ArgumentException($"Couldn't parse AzDo response {httpResponse.Content} to {nameof(ArtifactFiles)}"); + + return filesInformation; + } + catch (Exception ex) + { + _logger.LogError($"Failed to download file: {ex.Message}"); + throw; + } + } + + public async Task GetArtifactInformation(string buildId, string artifactName, int retryCount) + { + string relativeUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&api-version={_azureDevOpsApiVersion}"; + + _logger.LogMessage(MessageImportance.High, $"Getting {artifactName} metadata from {relativeUrl}"); + + try + { + using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(relativeUrl, retryCount); + + AzureDevOpsArtifactInformation artifactInformation = await httpResponse.Content.ReadFromJsonAsync() + ?? throw new ArgumentException($"Couldn't parse AzDo response {httpResponse.Content} to {nameof(AzureDevOpsArtifactInformation)}"); + + return artifactInformation; + } + catch(Exception ex) + { + _logger.LogError($"Failed to get artifact download URL: {ex.Message}"); + throw; + } + } + + private async Task ExecuteApiCallWithRetry(string relativeUrl, int retryCount) + { + int retriesRemaining = retryCount; + + while (true) + { + try + { + HttpResponseMessage httpResponse = await _httpClient.GetAsync(relativeUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + + httpResponse.EnsureSuccessStatusCode(); + + return httpResponse; + } + catch (Exception ex) when (ex is HttpRequestException || ex is TaskCanceledException) + { + if (ex is HttpRequestException && ex.Message.Contains(((int)HttpStatusCode.NotFound).ToString())) + { + _logger.LogError($"Resource not found at {relativeUrl}: {ex.Message}"); + throw; + } + + if (ex is HttpRequestException && ex.Message.Contains(((int)HttpStatusCode.Unauthorized).ToString())) + { + _logger.LogError($"Failure to authenticate: {ex.Message}"); + throw; + } + + if (retriesRemaining <= 0) + { + _logger.LogError($"There was an error calling AzureDevOps API against URI '{relativeUrl}' " + + $"after {retryCount} attempts. Exception: {ex}"); + throw; + } + + _logger.LogWarning($"There was an error calling AzureDevOps API against URI against URI '{relativeUrl}'. " + + $"{retriesRemaining} attempts remaining. Exception: {ex.ToString()}"); + } + + --retriesRemaining; + await Task.Delay(5000); + } + } + + public void Dispose() + { + _httpClient.Dispose(); + } + + public record Blob(string Id, int Size); + public record ArtifactItem(string Path, Blob Blob); + public record ArtifactFiles(string ManifestFormat, ArtifactItem[] Items, string[] ManifestReferences); +} \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs new file mode 100644 index 000000000000..8e2faef5c0b5 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -0,0 +1,335 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Xml.Linq; +using static Microsoft.DotNet.UnifiedBuild.Tasks.AzureDevOpsClient; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.UnifiedBuild.Tasks; + +public class JoinVerticals : Microsoft.Build.Utilities.Task +{ + /// + /// Paths to Verticals Manifests + /// + [Required] + public required ITaskItem[] VerticalManifest { get; init; } + + /// + /// Name of the main vertical that we'll take all artifacts from if they exist in this vertical, and at least one other vertical. + /// + [Required] + public required string MainVertical { get; init; } + + /// + /// Azure DevOps build id + /// + [Required] + public required string BuildId { get; init; } + + /// + /// Azure DevOps token, required scopes: "Build (read)", allowed to be empty when running in a public project + /// + public string? AzureDevOpsToken { get; set; } + + /// + /// Azure DevOps organization + /// + [Required] + public required string AzureDevOpsBaseUri { get; init; } + + /// + /// Azure DevOps project + /// + [Required] + public required string AzureDevOpsProject { get; init; } + + /// + /// Location of sownloaded artifacts from the main vertical + /// + [Required] + public required string MainVerticalArtifactsFolder { get; init; } + + /// + /// Folder where packages and assets will be stored + /// + [Required] + public required string OutputFolder { get; init; } + + private const string _packageElementName = "Package"; + private const string _blobElementName = "Blob"; + private const string _idAttribute = "Id"; + private const string _verticalNameAttribute = "VerticalName"; + private const string _artifactNameSuffix = "_Artifacts"; + private const string _assetsFolderName = "assets"; + private const string _packagesFolderName = "packages"; + private const int _retryCount = 10; + + private Dictionary> duplicatedItems = new(); + + public override bool Execute() + { + ExecuteAsync().GetAwaiter().GetResult(); + return !Log.HasLoggedErrors; + } + + private async Task ExecuteAsync() + { + List verticalManifests = VerticalManifest.Select(xmlPath => XDocument.Load(xmlPath.ItemSpec)).ToList(); + + XDocument mainVerticalManifest = verticalManifests.FirstOrDefault(manifest => GetRequiredRootAttribute(manifest, _verticalNameAttribute) == MainVertical) + ?? throw new ArgumentException($"Couldn't find main vertical manifest {MainVertical} in vertical manifest list"); + + if (!Directory.Exists(MainVerticalArtifactsFolder)) + { + throw new ArgumentException($"Main vertical artifacts directory {MainVerticalArtifactsFolder} not found."); + } + + string mainVerticalName = GetRequiredRootAttribute(mainVerticalManifest, _verticalNameAttribute); + + Dictionary packageElements = []; + Dictionary blobElements = []; + + List addedPackageIds = AddMissingElements(packageElements, mainVerticalManifest, _packageElementName); + List addedBlobIds = AddMissingElements(blobElements, mainVerticalManifest, _blobElementName); + + string packagesOutputDirectory = Path.Combine(OutputFolder, _packagesFolderName); + string blobsOutputDirectory = Path.Combine(OutputFolder, _assetsFolderName); + + CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _packagesFolderName), packagesOutputDirectory); + CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _assetsFolderName), blobsOutputDirectory); + + using var clientThrottle = new SemaphoreSlim(16, 16); + List downloadTasks = new(); + + foreach (XDocument verticalManifest in verticalManifests) + { + string verticalName = GetRequiredRootAttribute(verticalManifest, _verticalNameAttribute); + + // We already processed the main vertical + if (verticalName == MainVertical) + { + continue; + } + + addedPackageIds = AddMissingElements(packageElements, verticalManifest, _packageElementName); + addedBlobIds = AddMissingElements(blobElements, verticalManifest, _blobElementName); + + if (addedPackageIds.Count > 0) + { + downloadTasks.Add( + DownloadArtifactFiles( + BuildId, + $"{verticalName}{_artifactNameSuffix}", + addedPackageIds, + packagesOutputDirectory, + clientThrottle)); + } + + if (addedBlobIds.Count > 0) + { + downloadTasks.Add( + DownloadArtifactFiles( + BuildId, + $"{verticalName}{_artifactNameSuffix}", + addedBlobIds, + blobsOutputDirectory, + clientThrottle)); + } + } + + await Task.WhenAll(downloadTasks); + + // Create MergedManifest.xml + // taking the attributes from the main manifest + XElement mainManifestRoot = verticalManifests.First().Root + ?? throw new ArgumentException("The root element of the vertical manifest is null."); + mainManifestRoot.Attribute(_verticalNameAttribute)!.Remove(); + + string manifestOutputPath = Path.Combine(OutputFolder, "MergedManifest.xml"); + XDocument mergedManifest = new(new XElement( + mainManifestRoot.Name, + mainManifestRoot.Attributes(), + packageElements.Values.Select(v => v.Element).OrderBy(elem => elem.Attribute(_idAttribute)?.Value), + blobElements.Values.Select(v => v.Element).OrderBy(elem => elem.Attribute(_idAttribute)?.Value))); + + File.WriteAllText(manifestOutputPath, mergedManifest.ToString()); + + Log.LogMessage(MessageImportance.High, $"### Duplicate items found in the following verticals: ###"); + + foreach (var item in duplicatedItems) + { + Log.LogMessage(MessageImportance.High, $"Item: {item.Key} -- Produced by: {string.Join(", ", item.Value)}"); + } + } + + /// + /// Downloads specified packages and symbols from a specific build artifact and stores them in an output folder + /// + private async Task DownloadArtifactFiles( + string buildId, + string artifactName, + List fileNamesToDownload, + string outputDirectory, + SemaphoreSlim clientThrottle) + { + using AzureDevOpsClient azureDevOpsClient = new(AzureDevOpsToken, AzureDevOpsBaseUri, AzureDevOpsProject, Log); + + ArtifactFiles filesInformation = await azureDevOpsClient.GetArtifactFilesInformation(buildId, artifactName, _retryCount); + + await Task.WhenAll(fileNamesToDownload.Select(async fileName => + await DownloadFileFromArtifact( + filesInformation, + artifactName, + azureDevOpsClient, + buildId, + fileName, + outputDirectory, + clientThrottle))); + } + + private async Task DownloadFileFromArtifact( + ArtifactFiles artifactFilesMetadata, + string azureDevOpsArtifact, + AzureDevOpsClient azureDevOpsClient, + string buildId, + string manifestFile, + string destinationDirectory, + SemaphoreSlim clientThrottle) + { + try + { + await clientThrottle.WaitAsync(); + + ArtifactItem fileItem; + + var matchingFilePaths = artifactFilesMetadata.Items.Where(f => Path.GetFileName(f.Path) == Path.GetFileName(manifestFile)); + + if (!matchingFilePaths.Any()) + { + throw new ArgumentException($"File {manifestFile} not found in source files."); + } + + if (matchingFilePaths.Count() > 1) + { + // Picking the first one until https://github.com/dotnet/source-build/issues/4596 is resolved + if (manifestFile.Contains("productVersion.txt")) + { + fileItem = matchingFilePaths.First(); + } + else + { + // For some files it's not enough to compare the filename because they have 2 copies in the artifact + // e.g. assets/Release/dotnet-sdk-*-win-x64.zip and assets/Release/Sdk/*/dotnet-sdk-*-win-x64.zip + // In this case take the one matching the full path from the manifest + fileItem = matchingFilePaths + .SingleOrDefault(f => f.Path.EndsWith(manifestFile) || f.Path.EndsWith(manifestFile.Replace("/", @"\"))) + ?? throw new ArgumentException($"File {manifestFile} not found in source files."); + } + } + else + { + fileItem = matchingFilePaths.Single(); + } + + string itemId = fileItem.Blob.Id; + string artifactSubPath = fileItem.Path; + + string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(manifestFile)); + + await azureDevOpsClient.DownloadSingleFileFromArtifact(buildId, azureDevOpsArtifact, itemId, artifactSubPath, destinationFilePath, _retryCount); + } + catch (Exception ex) + { + Log.LogError($"Failed to download file {manifestFile} from artifact {azureDevOpsArtifact}: {ex.Message}"); + throw; + } + finally + { + clientThrottle.Release(); + } + } + + /// + /// Copy all files from the source directory to the destination directory, + /// in a flat layout + /// + private void CopyMainVerticalAssets(string sourceDirectory, string destinationDirectory) + { + var sourceFiles = Directory.EnumerateFiles(sourceDirectory, "*", SearchOption.AllDirectories); + + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + + foreach (var sourceFile in sourceFiles) + { + string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(sourceFile)); + + Log.LogMessage(MessageImportance.High, $"Copying {sourceFile} to {destinationFilePath}"); + File.Copy(sourceFile, destinationFilePath, true); + } + } + + /// + /// Find the artifacts from the vertical manifest that are not already in the dictionary and add them + /// Return a list of the added artifact ids + /// + private List AddMissingElements(Dictionary addedArtifacts, XDocument verticalManifest, string elementName) + { + List addedFiles = []; + + string verticalName = verticalManifest.Root!.Attribute(_verticalNameAttribute)!.Value; + + foreach (XElement artifactElement in verticalManifest.Descendants(elementName)) + { + string elementId = artifactElement.Attribute(_idAttribute)?.Value + ?? throw new ArgumentException($"Required attribute '{_idAttribute}' not found in {elementName} element."); + + if (addedArtifacts.TryAdd(elementId, new AddedElement(verticalName, artifactElement))) + { + if (elementName == _packageElementName) + { + string version = artifactElement.Attribute("Version")?.Value + ?? throw new ArgumentException($"Required attribute 'Version' not found in {elementName} element."); + + elementId += $".{version}.nupkg"; + } + + addedFiles.Add(elementId); + Log.LogMessage(MessageImportance.High, $"Taking {elementName} '{elementId}' from '{verticalName}'"); + } + else + { + AddedElement previouslyAddedArtifact = addedArtifacts[elementId]; + if (previouslyAddedArtifact.VerticalName != MainVertical) + { + if (!duplicatedItems.TryAdd(elementId, new List { verticalName, previouslyAddedArtifact.VerticalName })) + { + duplicatedItems[elementId].Add(verticalName); + } + } + } + } + + return addedFiles; + } + + private static string GetRequiredRootAttribute(XDocument document, string attributeName) + { + return document.Root?.Attribute(attributeName)?.Value + ?? throw new ArgumentException($"Required attribute '{attributeName}' not found in root element."); + } + + private record AddedElement(string VerticalName, XElement Element); +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs index eff0413e88b8..c2b87171ad2c 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs @@ -33,7 +33,13 @@ public class MergeAssetManifests : Task /// public string VmrBuildNumber { get; set; } = string.Empty; + /// + /// Vmr Vertical Name, e.g. "Android_Shortstack_arm". Allowed to be empty for non official builds. + /// + public string VerticalName { get; set; } = string.Empty; + private static readonly string _buildIdAttribute = "BuildId"; + private static readonly string _verticalNameAttribute = "VerticalName"; private static readonly string _azureDevOpsBuildNumberAttribute = "AzureDevOpsBuildNumber"; private static readonly string[] _ignoredAttributes = [ _buildIdAttribute, @@ -54,6 +60,7 @@ public override bool Execute() // Set the BuildId and AzureDevOpsBuildNumber attributes to the value of VmrBuildNumber mergedManifestRoot.SetAttributeValue(_buildIdAttribute, VmrBuildNumber); mergedManifestRoot.SetAttributeValue(_azureDevOpsBuildNumberAttribute, VmrBuildNumber); + mergedManifestRoot.SetAttributeValue(_verticalNameAttribute, VerticalName); List packageElements = new(); List blobElements = new(); @@ -70,6 +77,7 @@ public override bool Execute() XDocument verticalManifest = new(new XElement(mergedManifestRoot.Name, mergedManifestRoot.Attributes(), packageElements, blobElements)); File.WriteAllText(MergedAssetManifestOutputPath, verticalManifest.ToString()); + Log.LogMessage(MessageImportance.High, $"Merged asset manifest written to {MergedAssetManifestOutputPath}"); return !Log.HasLoggedErrors; } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs new file mode 100644 index 000000000000..d7d2ba96bd78 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +#nullable enable + +namespace Microsoft.DotNet.UnifiedBuild.Tasks; + +public record AzureDevOpsArtifactInformation(int Id, string Name, string Source, AzdoArtifactResources Resource); +public record AzdoArtifactResources(string Type, string Data, AzdoArtifactProperties Properties, string Url, string DownloadUrl); +public record AzdoArtifactProperties(string RootId, string Artifactsize, string HashType, string DomainId); \ No newline at end of file