From af61cb8a473d75135760b8e9acb1be8e1fc90b45 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 11:12:41 -0500 Subject: [PATCH 01/26] auto-generated 1es conversion of arcade templates --- eng/common/templates/job/execute-sdl.yml | 112 ++--- eng/common/templates/job/job.yml | 125 ++---- .../templates/job/publish-build-assets.yml | 110 ++--- eng/common/templates/job/source-build.yml | 53 +-- .../templates/job/source-index-stage1.yml | 15 +- eng/common/templates/jobs/jobs.yml | 51 +-- eng/common/templates/jobs/source-build.yml | 27 +- .../templates/post-build/common-variables.yml | 42 +- .../templates/post-build/post-build.yml | 381 ++++++++---------- .../post-build/setup-maestro-vars.yml | 118 +++--- .../templates/steps/component-governance.yml | 3 +- eng/common/templates/steps/execute-sdl.yml | 41 +- eng/common/templates/steps/generate-sbom.yml | 38 +- eng/common/templates/steps/publish-logs.yml | 34 +- eng/common/templates/steps/send-to-helix.yml | 6 +- eng/common/templates/steps/source-build.yml | 44 +- .../templates/variables/pool-providers.yml | 62 +-- .../templates/variables/sdl-variables.yml | 4 +- 18 files changed, 421 insertions(+), 845 deletions(-) diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 7870f93bc1..69cf5747fd 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,69 +1,58 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# Outputs were added with YAML Conditional expressions generated using AI in a best attempt to preserve the boolean logic that would have lead to the tasks producing outputs executing before conversion. You may need to review the generated expressions for correctness of the execution logic, and you will need to manually translate any cross-template parameters. +# Output from "eng\common\templates\steps\execute-sdl.yml" added to job "Run SDL tool" with conditionals extracted using AI from the following files: "eng\common\templates\steps\execute-sdl.yml". +# Output from "eng\common\templates\steps\execute-sdl.yml" added to job "Run SDL tool" with conditionals extracted using AI from the following files: "eng\common\templates\steps\execute-sdl.yml". + parameters: - enable: 'false' # Whether the SDL validation job should execute or not - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. + enable: 'false' + overrideParameters: '' + additionalParameters: '' overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; - # optional: determines if build artifacts should be downloaded. + sdlContinueOnError: false downloadArtifacts: true - # optional: determines if this job should search the directory of downloaded artifacts for - # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks. extractArchiveArtifacts: false - dependsOn: '' # Optional: dependencies of the job - artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts - # Usage: - # artifactNames: - # - 'BlobArtifacts' - # - 'Artifacts_Windows_NT_Release' - # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, - # not pipeline artifacts, so doesn't affect the use of this parameter. + dependsOn: '' + artifactNames: '' pipelineArtifactNames: [] - jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) variables: - - group: DotNet-VSTS-Bot - - name: AzDOProjectName - value: ${{ parameters.AzDOProjectName }} - - name: AzDOPipelineId - value: ${{ parameters.AzDOPipelineId }} - - name: AzDOBuildId - value: ${{ parameters.AzDOBuildId }} - - template: /eng/common/templates/variables/sdl-variables.yml - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - template: /eng/common/templates/variables/pool-providers.yml - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 + - group: DotNet-VSTS-Bot + - name: AzDOProjectName + value: ${{ parameters.AzDOProjectName }} + - name: AzDOPipelineId + value: ${{ parameters.AzDOPipelineId }} + - name: AzDOBuildId + value: ${{ parameters.AzDOBuildId }} + - template: /eng/common/templates/variables/sdl-variables.yml@self + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + - template: /eng/common/templates/variables/pool-providers.yml@self + templateContext: + outputs: + - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + - output: pipelineArtifact + displayName: 'Publish GuardianConfiguration' + condition: succeededOrFailed() + targetPath: $(Agent.BuildDirectory)/.gdn + artifactName: GuardianConfiguration + - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + - output: pipelineArtifact + displayName: 'Publish SARIF files to CodeAnalysisLogs container' + condition: succeededOrFailed() + targetPath: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs steps: - checkout: self clean: true - - # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - + - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: - ${{ each artifactName in parameters.artifactNames }}: @@ -91,7 +80,6 @@ jobs: itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts checkDownloadedFiles: true - - ${{ each artifactName in parameters.pipelineArtifactNames }}: - task: DownloadPipelineArtifact@2 displayName: Download Pipeline Artifacts @@ -104,36 +92,24 @@ jobs: artifactName: ${{ artifactName }} downloadPath: $(Build.ArtifactStagingDirectory)\artifacts checkDownloadedFiles: true - - - powershell: eng/common/sdl/trim-assets-version.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts + - powershell: eng/common/sdl/trim-assets-version.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts displayName: Trim the version from the NuGet packages continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts + - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts + - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - - powershell: eng/common/sdl/extract-artifact-archives.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts + - powershell: eng/common/sdl/extract-artifact-archives.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts displayName: Extract Archive Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - - template: /eng/common/templates/steps/execute-sdl.yml + - template: /eng/common/templates/steps/execute-sdl.yml@self parameters: overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} overrideParameters: ${{ parameters.overrideParameters }} additionalParameters: ${{ parameters.additionalParameters }} publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 01c0dd995e..c3ac7e4d0b 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,8 +1,14 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish pipeline artifacts' in the templateContext section. +# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish logs' in the templateContext section. +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Logs' in the templateContext section. +# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish build retry configuration' in the templateContext section. +# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job cancelTimeoutInMinutes: '' condition: '' container: '' @@ -15,9 +21,6 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md artifacts: '' enableMicrobuild: false enablePublishBuildArtifacts: false @@ -33,41 +36,29 @@ parameters: name: '' preSteps: [] runAsPublic: false -# Sbom related params enableSbom: true PackageVersion: 7.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - jobs: - job: ${{ parameters.name }} - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} - ${{ if ne(parameters.container, '') }}: container: ${{ parameters.container }} - ${{ if ne(parameters.continueOnError, '') }}: continueOnError: ${{ parameters.continueOnError }} - ${{ if ne(parameters.dependsOn, '') }}: dependsOn: ${{ parameters.dependsOn }} - ${{ if ne(parameters.displayName, '') }}: displayName: ${{ parameters.displayName }} - ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} - ${{ if ne(parameters.strategy, '') }}: strategy: ${{ parameters.strategy }} - ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE @@ -75,53 +66,57 @@ jobs: - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - name: EnableRichCodeNavigation value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY value: 3,2000 - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - ${{ if ne(variable.name, '') }}: - name: ${{ variable.name }} value: ${{ variable.value }} - - # handle variable groups - ${{ if ne(variable.group, '') }}: - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - ${{ if ne(variable.template, '') }}: - template: ${{ variable.template }} ${{ if ne(variable.parameters, '') }}: parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - ${{ each pair in variable }}: - name: ${{ pair.key }} value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-HelixApi-Access - ${{ if ne(parameters.workspace, '') }}: workspace: ${{ parameters.workspace }} - + templateContext: + outputs: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - output: pipelineArtifact + displayName: 'Publish pipeline artifacts' + condition: always() + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + publishLocation: Container + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - output: pipelineArtifact + displayName: 'Publish logs' + condition: always() + targetPath: artifacts/log + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - output: pipelineArtifact + displayName: 'Publish Logs' + condition: always() + targetPath: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + publishLocation: Container + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - output: pipelineArtifact + displayName: 'Publish build retry configuration' + targetPath: $(Build.SourcesDirectory)\eng\common\BuildConfiguration + artifactName: BuildConfiguration steps: - ${{ if ne(parameters.preSteps, '') }}: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - task: MicroBuildSigningPlugin@3 @@ -134,10 +129,8 @@ jobs: TeamName: $(_TeamName) continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@1 - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 inputs: @@ -145,10 +138,8 @@ jobs: artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - ${{ each step in parameters.steps }}: - ${{ step }} - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - task: RichCodeNavIndexer@0 displayName: RichCodeNav Upload @@ -158,8 +149,7 @@ jobs: richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} continueOnError: true - - - template: /eng/common/templates/steps/component-governance.yml + - template: /eng/common/templates/steps/component-governance.yml@self parameters: ${{ if eq(parameters.disableComponentGovernance, '') }}: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: @@ -169,7 +159,6 @@ jobs: ${{ else }}: disableComponentGovernance: ${{ parameters.disableComponentGovernance }} componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 @@ -178,7 +167,6 @@ jobs: continueOnError: ${{ parameters.continueOnError }} env: TeamName: $(_TeamName) - - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - task: CopyFiles@2 @@ -193,31 +181,8 @@ jobs: SourceFolder: 'artifacts/packages' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: [] + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: [] - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - task: PublishTestResults@2 displayName: Publish XUnit Test Results @@ -240,16 +205,10 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml + - template: /eng/common/templates/steps/generate-sbom.yml@self parameters: PackageVersion: ${{ parameters.packageVersion}} BuildDropPath: ${{ parameters.buildDropPath }} IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish build retry configuration - continueOnError: true + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: [] \ No newline at end of file diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 3115990d51..a5e8767a1c 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -1,72 +1,63 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish ReleaseConfigs Artifact' in the templateContext section. +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish SymbolPublishingExclusionsFile Artifact' in the templateContext section. +# Output added to job "Asset_Registry_Publish" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +# Output added to job "Asset_Registry_Publish" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. parameters: configuration: 'Debug' - - # Optional: condition for the job to run condition: '' - - # Optional: 'true' if future jobs should run even if this job fails continueOnError: false - - # Optional: dependencies of the job dependsOn: '' - - # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing publishAssetsImmediately: false - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - jobs: - job: Asset_Registry_Publish - dependsOn: ${{ parameters.dependsOn }} timeoutInMinutes: 150 - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: displayName: Publish Assets ${{ else }}: displayName: Publish to Build Asset Registry - variables: - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates/variables/pool-providers.yml@self - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - # unconditional - needed for logs publishing (redactor tool version) - - template: /eng/common/templates/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - + - template: /eng/common/templates/post-build/common-variables.yml@self + templateContext: + outputs: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - output: pipelineArtifact + displayName: 'Publish ReleaseConfigs Artifact' + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + publishLocation: Container + artifactName: ReleaseConfigs + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - output: pipelineArtifact + displayName: 'Publish SymbolPublishingExclusionsFile Artifact' + condition: eq(variables['SymbolExclusionFile'], 'true') + targetPath: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + publishLocation: Container + artifactName: ReleaseConfigs + - output: pipelineArtifact + displayName: 'Publish Logs' + condition: always() + targetPath: '$(Build.SourcesDirectory)/PostBuildLogs' + publishLocation: Container + artifactName: PostBuildLogs steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - checkout: self fetchDepth: 3 clean: true - - task: DownloadBuildArtifacts@0 displayName: Download artifact inputs: @@ -75,22 +66,14 @@ jobs: checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - task: NuGetAuthenticate@1 - - task: PowerShell@2 displayName: Publish Build Assets inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -99,14 +82,6 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - - - task: PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - task: powershell@2 displayName: Check if SymbolPublishingExclusionsFile.txt exists inputs: @@ -122,34 +97,17 @@ jobs: Write-Host "Symbols Exclusion file does not exists" Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" } - - - task: PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml + - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self parameters: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' - + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates/steps/publish-logs.yml + - template: /eng/common/templates/steps/publish-logs.yml@self parameters: - JobLabel: 'Publish_Artifacts_Logs' + JobLabel: 'Publish_Artifacts_Logs' \ No newline at end of file diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 8a3deef2b7..0d87b31edf 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -1,66 +1,35 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. platform: {} - jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) - ${{ each property in parameters.platform.jobProperties }}: ${{ property.key }}: ${{ property.value }} - ${{ if ne(parameters.platform.container, '') }}: container: ${{ parameters.platform.container }} - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} - workspace: clean: all - + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) steps: - - template: /eng/common/templates/steps/source-build.yml + - template: /eng/common/templates/steps/source-build.yml@self parameters: - platform: ${{ parameters.platform }} + platform: ${{ parameters.platform }} \ No newline at end of file diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b5a3e5c4a6..1c86018a1b 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,3 +1,5 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: runAsPublic: false sourceIndexPackageVersion: 1.0.1-20240129.2 @@ -8,7 +10,6 @@ parameters: condition: '' dependsOn: '' pool: '' - jobs: - job: SourceIndexStage1 dependsOn: ${{ parameters.dependsOn }} @@ -22,8 +23,7 @@ jobs: value: ${{ parameters.binlogPath }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: source-dot-net stage1 variables - - template: /eng/common/templates/variables/pool-providers.yml - + - template: /eng/common/templates/variables/pool-providers.yml@self ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} ${{ if eq(parameters.pool, '') }}: @@ -34,11 +34,9 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2022.amd64 - steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - - task: UseDotNet@2 displayName: Use .NET 8 SDK inputs: @@ -46,22 +44,17 @@ jobs: version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - - script: | $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) - - script: ${{ parameters.sourceIndexBuildCommand }} displayName: Build Repository - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output displayName: Process Binlog into indexable sln - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) displayName: Upload stage1 artifacts to source index env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) \ No newline at end of file diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 289bb2396c..78668e65e0 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,82 +1,48 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md continueOnError: false - - # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates/jobs/source-build.yml for options sourceBuildParameters: [] - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build enabled: false - # Optional: Include toolset dependencies in the generated graph files includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false - enableSourceIndex: false sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: - ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters + - template: /eng/common/templates/job/job.yml@self + parameters: ${{ each parameter in parameters }}: ${{ if ne(parameter.key, 'jobs') }}: ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties ${{ each property in job }}: ${{ if ne(property.key, 'job') }}: ${{ property.key }}: ${{ property.value }} - name: ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates/jobs/source-build.yml + - template: /eng/common/templates/jobs/source-build.yml@self parameters: allCompletedJobId: Source_Build_Complete ${{ each parameter in parameters.sourceBuildParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml + - template: /eng/common/templates/job/source-index-stage1.yml@self parameters: runAsPublic: ${{ parameters.runAsPublic }} ${{ each parameter in parameters.sourceIndexParams }}: ${{ parameter.key }}: ${{ parameter.value }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml + - template: /eng/common/templates/job/publish-build-assets.yml@self parameters: continueOnError: ${{ parameters.continueOnError }} dependsOn: @@ -88,10 +54,9 @@ jobs: - ${{ job.job }} - ${{ if eq(parameters.enableSourceBuild, true) }}: - Source_Build_Complete - runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} \ No newline at end of file diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51..fdeaa59ff6 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -1,28 +1,13 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. allCompletedJobId: '' - - # See /eng/common/templates/job/source-build.yml jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. platforms: [] - jobs: - - ${{ if ne(parameters.allCompletedJobId, '') }}: - job: ${{ parameters.allCompletedJobId }} displayName: Source-Build Complete @@ -32,15 +17,13 @@ jobs: - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - ${{ if eq(length(parameters.platforms), 0) }}: - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - - ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates/job/source-build.yml + - template: /eng/common/templates/job/source-build.yml@self parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates/job/source-build.yml + - template: /eng/common/templates/job/source-build.yml@self parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} + platform: ${{ parameters.defaultManagedPlatform }} \ No newline at end of file diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index b9ede10bf0..e29be13b71 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,24 +1,20 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. variables: - - group: Publish-Build-Assets - - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - name: BinlogToolVersion - value: 1.0.11 - - - name: runCodesignValidationInjection - value: false +- group: Publish-Build-Assets +- name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} +- name: MaestroApiEndPoint + value: "https://maestro.dot.net" +- name: MaestroApiAccessToken + value: $(MaestroAccessToken) +- name: MaestroApiVersion + value: "2020-02-20" +- name: SourceLinkCLIVersion + value: 3.0.0 +- name: SymbolToolVersion + value: 1.0.1 +- name: BinlogToolVersion + value: 1.0.11 +- name: runCodesignValidationInjection + value: false \ No newline at end of file diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index bbc010fe73..3e0410a5d5 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,235 +1,191 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - +- name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 +- name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 +- name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' +- name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false +- name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true +- name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false +- name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true +- name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true +- name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true +- name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' +- name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' +- name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' +- name: validateDependsOn + type: object + default: + - build +- name: publishDependsOn + type: object + default: + - Validate +- name: publishAssetsImmediately + type: boolean + default: false stages: - ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate Build Assets variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates/post-build/common-variables.yml@self + - template: /eng/common/templates/variables/pool-providers.yml@self jobs: - - job: + - job: '' displayName: NuGet Validation condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd - # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: + - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + - job: '' displayName: Signing Validation condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd - # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Logs' + condition: always() + targetPath: '$(Build.SourcesDirectory)/PostBuildLogs' + publishLocation: Container + artifactName: PostBuildLogs steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: + - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' ${{ parameters.signingValidationAdditionalParameters }} + - template: /eng/common/templates/steps/publish-logs.yml@self + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + - job: '' displayName: SourceLink Validation condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd - # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml + - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Extract/ -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + - template: /eng/common/templates/job/execute-sdl.yml@self parameters: enable: ${{ parameters.SDLValidationParameters.enable }} publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }} @@ -237,7 +193,6 @@ stages: continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: @@ -246,37 +201,27 @@ stages: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates/post-build/common-variables.yml@self + - template: /eng/common/templates/variables/pool-providers.yml@self jobs: - - job: + - job: '' displayName: Publish Using Darc timeoutInMinutes: 120 pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd - # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 - - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: NuGetAuthenticate@1 + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' \ No newline at end of file diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4..ad39920d70 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,70 +1,62 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: BARBuildId: '' PromoteToChannelIds: '' - steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars +- ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" - - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true +- task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} \ No newline at end of file diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 0ecec47b0c..6a0431ec60 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -1,7 +1,8 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: disableComponentGovernance: false componentGovernanceIgnoreDirectories: '' - steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 07426fde05..723f8ae0a9 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -1,3 +1,7 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish GuardianConfiguration' in the templateContext section. +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish SARIF files to CodeAnalysisLogs container' in the templateContext section. parameters: overrideGuardianVersion: '' executeAllSdlToolsScript: '' @@ -6,15 +10,12 @@ parameters: publishGuardianDirectoryToPipeline: false sdlContinueOnError: false condition: '' - steps: - task: NuGetAuthenticate@1 inputs: nuGetServiceConnections: GuardianConnect - - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - - ${{ if ne(parameters.overrideGuardianVersion, '') }}: - pwsh: | Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl @@ -22,7 +23,6 @@ steps: $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian (Overridden) - - ${{ if eq(parameters.overrideGuardianVersion, '') }}: - pwsh: | Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl @@ -30,29 +30,17 @@ steps: $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian - - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} displayName: Execute SDL (Overridden) continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} - - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianCliLocation $(GuardianCliLocation) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} + - powershell: ${{ parameters.executeAllSdlToolsScript }} -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} - - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - task: DeleteFiles@1 displayName: Delete Guardian dependencies to avoid uploading inputs: @@ -61,28 +49,11 @@ steps: c i condition: succeededOrFailed() - - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() - - # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration - # with the "SARIF SAST Scans Tab" Azure DevOps extension - task: CopyFiles@2 displayName: Copy SARIF files inputs: flattenFolders: true - sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ contents: '**/*.sarif' targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs - condition: succeededOrFailed() - - # Use PublishBuildArtifacts because the SARIF extension only checks this case - # see microsoft/sarif-azuredevops-extension#4 - - task: PublishBuildArtifacts@1 - displayName: Publish SARIF files to CodeAnalysisLogs container - inputs: - pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs - artifactName: CodeAnalysisLogs condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38f..180a76d6e8 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -1,9 +1,7 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# The SBOM tasks have been removed because they are not required for the unofficial template. +# You can manually enable SBOM in the unofficial template if needed, othewise its automatically enabled when using official template. https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sbom parameters: PackageVersion: 7.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' @@ -11,38 +9,16 @@ parameters: ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom IgnoreDirectories: '' sbomContinueOnError: true - steps: -- task: PowerShell@2 +- task: PowerShell@2 displayName: Prep for SBOM generation in (Non-linux) condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: + inputs: filePath: ./eng/common/generate-sbom-prep.ps1 arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 - script: | chmod +x ./eng/common/generate-sbom-prep.sh ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} displayName: Prep for SBOM generation in (Linux) condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) - + continueOnError: ${{ parameters.sbomContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 80861297dd..437705b17a 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,10 +1,11 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Logs' in the templateContext section. parameters: StageLabel: '' JobLabel: '' CustomSensitiveDataList: '' - # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed BinlogToolVersion: '1.0.11' - steps: - task: Powershell@2 displayName: Prepare Binlogs to Upload @@ -15,35 +16,10 @@ steps: Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true condition: always() - - task: PowerShell@2 displayName: Redact Logs inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 - # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml - # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - # If the file exists - sensitive data for redaction will be sourced from it - # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} - -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' - '$(akams-client-secret)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' - ${{parameters.CustomSensitiveDataList}} + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' -BinlogToolVersion ${{parameters.BinlogToolVersion}} -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' '$(publishing-dnceng-devdiv-code-r-build-re)' '$(MaestroAccessToken)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' '$(akams-client-secret)' '$(microsoft-symbol-server-pat)' '$(symweb-symbol-server-pat)' '$(dn-bot-all-orgs-build-rw-code-rw)' ${{parameters.CustomSensitiveDataList}} continueOnError: true - condition: always() - -- task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() + condition: always() \ No newline at end of file diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 3eb7e2d5f8..68fa739c4a 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -5,6 +5,8 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY + HelixProjectArguments: '' # optional -- arguments passed to the build command HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution @@ -29,7 +31,7 @@ parameters: continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -59,7 +61,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 32738aa938..0bb85b7e06 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -1,23 +1,12 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish BuildLogs' in the templateContext section. parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. platform: {} - steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) - script: | set -x df -h - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) # In that case, call the feed setup script to add internal feeds corresponding to public ones. # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. @@ -29,7 +18,6 @@ steps: chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. # This only works if there is a username/email configured, which won't be the case in most CI runs. git config --get user.email @@ -38,7 +26,6 @@ steps: git config user.name dn-bot fi fi - # If building on the internal project, the internal storage variable may be available (usually only if needed) # In that case, add variables to allow the download of internal runtimes if the specified versions are not found # in the default public locations. @@ -46,43 +33,35 @@ steps: if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' fi - buildConfig=Release # Check if AzDO substitutes in a build config from a variable, and use it if so. if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then buildConfig='$(_BuildConfig)' fi - officialBuildArgs= if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi - targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi - runtimeOsArgs= if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' fi - baseOsArgs= if [ '${{ parameters.platform.baseOS }}' != '' ]; then baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' fi - publishArgs= if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then publishArgs='--publish' fi - assetManifestFileName=SourceBuild_RidSpecific.xml if [ '${{ parameters.platform.name }}' != '' ]; then assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml fi - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ @@ -98,8 +77,6 @@ steps: /p:DotNetBuildRepo=true \ /p:AssetManifestFileName=$assetManifestFileName displayName: Build - -# Upload build logs for diagnosis. - task: CopyFiles@2 displayName: Prepare BuildLogs staging directory inputs: @@ -112,20 +89,7 @@ steps: CleanTargetFolder: true continueOnError: true condition: succeededOrFailed() - -- task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() - -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets - task: ComponentGovernanceComponentDetection@0 displayName: Component Detection (Exclude upstream cache) inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' \ No newline at end of file diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 9cc5c550d3..986a36e3c1 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,57 +1,7 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, -# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. - -# Motivation: -# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS -# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template -# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. - -# How to use: -# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. -# -# First, import the template in an arcade-ified repo to pick up the variables, e.g.: -# -# variables: -# - template: /eng/common/templates/variables/pool-providers.yml -# -# ... then anywhere specifying the pool provider use the runtime variables, -# $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: -# -# pool: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 - +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. variables: - # Coalesce the target and source branches so we know when a PR targets a release branch - # If these variables are somehow missing, fall back to main (tends to have more capacity) - - # Any new -Svc alternative pools should have variables added here to allow for splitting work - - name: DncEngPublicBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore-Svc-Public' - ), - False, - 'NetCore-Public' - ) - ] - - - name: DncEngInternalBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore1ESPool-Svc-Internal' - ), - False, - 'NetCore1ESPool-Internal' - ) - ] \ No newline at end of file +- name: DncEngPublicBuildPool + value: $[ replace( replace( eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public' ) ] +- name: DncEngInternalBuildPool + value: $[ replace( replace( eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal' ), False, 'NetCore1ESPool-Internal' ) ] \ No newline at end of file diff --git a/eng/common/templates/variables/sdl-variables.yml b/eng/common/templates/variables/sdl-variables.yml index dbdd66d4a4..86af5b73b2 100644 --- a/eng/common/templates/variables/sdl-variables.yml +++ b/eng/common/templates/variables/sdl-variables.yml @@ -1,6 +1,6 @@ +# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. +# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. variables: -# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in -# sync with the packages.config file. - name: DefaultGuardianVersion value: 0.109.0 - name: GuardianPackagesConfigFile From 4d22c3280915b54bdfccf697735e1302dcabb852 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 11:35:43 -0500 Subject: [PATCH 02/26] Initial 1es conversion --- build/vsts-ci.yml | 614 ++++++++++++++++++++++++---------------------- 1 file changed, 320 insertions(+), 294 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 85d6cfc5ff..c5cc4b284a 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -2,17 +2,6 @@ # ML.NET's official, signed build ################################################################################ -resources: - containers: - - container: CentosContainer - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8-mlnet - - - container: UbuntuCrossArmContainer - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm - - - container: UbuntuCrossArm64Container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm64 - variables: BuildConfig: Release _BuildConfig: $(BuildConfig) @@ -22,289 +11,326 @@ variables: DOTNET_MULTILEVEL_LOOKUP: 0 Codeql.Enabled: true Codeql.SkipTaskAutoInjection: True #default to not inject CodeQL tasks, we'll enable it in a single job. +- template: /eng/common-variables.yml +- template: /eng/common/templates/variables/pool-providers.yml -stages: -- stage: build - displayName: Build - jobs: - ################################################################################ - - job: Linux_x64 - ################################################################################ - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - container: CentosContainer - steps: - # Only build native assets to avoid conflicts. - - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Linux package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - ################################################################################ - - job: Linux_arm - ################################################################################ - variables: - ROOTFS_DIR: '/crossrootfs/arm' - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - container: UbuntuCrossArmContainer - steps: - # Only build native assets to avoid conflicts. - - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Linux_arm package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - ################################################################################ - - job: Linux_arm64 - ################################################################################ - variables: - ROOTFS_DIR: '/crossrootfs/arm64' - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - container: UbuntuCrossArm64Container - steps: - # Only build native assets to avoid conflicts. - - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Linux_arm64 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - ################################################################################ - - job: MacOS - ################################################################################ - pool: - vmImage: macOS-12 - steps: - - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula - displayName: Install build dependencies - # Only build native assets to avoid conflicts. - - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish macOS package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - ################################################################################ - - job: MacOS_Apple_Silicon - ################################################################################ - pool: - vmImage: macOS-12 - steps: - # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/2322#issuecomment-749211076 - - script: | - rm -rf /usr/local/bin/2to3 - displayName: MacOS Homebrew bug Workaround - continueOnError: true - - script: brew update && brew install libomp && brew link libomp --force - displayName: Install build dependencies - # Only build native assets to avoid conflicts. - - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish macOS_M1 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - ################################################################################ - - job: Windows_arm64 - ################################################################################ - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - # Only build native assets to avoid conflicts. - - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_arm64 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown - - ################################################################################ - - job: Windows_x86 - ################################################################################ - variables: - _TeamName: DotNetCore - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - # Only build native assets to avoid conflicts. - - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_x86 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown - - ################################################################################ - - job: Windows_x64 - ################################################################################ - variables: - Codeql.SkipTaskAutoInjection: False # run CodeQL in this job - _TeamName: DotNetCore +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - # Build both native and managed assets. - - script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true + name: Azure-Pipelines-1ESPT-ExDShared + image: windows-2022 + os: windows + containers: + CentosContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8-mlnet + UbuntuCrossArmContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm + UbuntuCrossArm64Container: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm64 + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: build displayName: Build - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_x64 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown - - ################################################################################ - - template: /eng/common/templates/job/job.yml - parameters: - name: Package - ################################################################################ - artifacts: - publish: - artifacts: false - logs: true - manifests: true - enableMicrobuild: true - enablePublishUsingPipelines: true - enableSourceIndex: true - dependsOn: - - Linux_x64 - - Linux_arm - - Linux_arm64 - - MacOS - - Windows_x86 - - Windows_x64 - - MacOS_Apple_Silicon - variables: - - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. - - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages - - _SignType: real - - _TeamName: DotNetCore - - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) - /p:TeamName=$(_TeamName) - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:OfficialBuildId=$(OfficialBuildId) - /p:DotNetPublishUsingPipelines=$true - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: pkgassets - downloadPath: $(Build.SourcesDirectory)/artifacts - - # Depending on the value of DotNetFinalVersionKind, the name of the package will change. - # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" - # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable - # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version - - script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) - displayName: Build Packages - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown - - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - dependsOn: Package - enablePublishBuildArtifacts: true - publishAssetsImmediately: true - publishUsingPipelines: true - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - -- template: /eng/common/templates/post-build/post-build.yml - parameters: - publishingInfraVersion: 3 - # This is to enable SDL runs part of Post-Build Validation Stage - SDLValidationParameters: - enable: true - publishGdn: true - continueOnError: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL https://devdiv.visualstudio.com/ - -TsaProjectName DEVDIV - -TsaNotificationEmail mlnetcore@microsoft.com - -TsaCodebaseAdmin REDMOND\ericstj - -TsaBugAreaPath "DevDiv\Net Libraries" - -TsaIterationPath DevDiv - -TsaRepositoryName machinelearning - -TsaCodebaseName machinelearning - -TsaPublish $True' \ No newline at end of file + jobs: + ################################################################################ + - job: Linux_x64 + ################################################################################ + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204-pt + os: linux + container: CentosContainer + steps: + # Only build native assets to avoid conflicts. + - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Linux package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml@self + + ################################################################################ + - job: Linux_arm + ################################################################################ + variables: + ROOTFS_DIR: '/crossrootfs/arm' + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204-pt + os: linux + container: UbuntuCrossArmContainer + steps: + # Only build native assets to avoid conflicts. + - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Linux_arm package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml@self + + ################################################################################ + - job: Linux_arm64 + ################################################################################ + variables: + ROOTFS_DIR: '/crossrootfs/arm64' + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204-pt + os: linux + container: UbuntuCrossArm64Container + steps: + # Only build native assets to avoid conflicts. + - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Linux_arm64 package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml@self + + ################################################################################ + - job: MacOS + ################################################################################ + pool: + name: Azure Pipelines + vmImage: macOS-12 + os: macOS + steps: + - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula + displayName: Install build dependencies + # Only build native assets to avoid conflicts. + - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish macOS package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + ################################################################################ + - job: MacOS_Apple_Silicon + ################################################################################ + pool: + name: Azure Pipelines + vmImage: macOS-12 + os: macOS + steps: + # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/2322#issuecomment-749211076 + - script: | + rm -rf /usr/local/bin/2to3 + displayName: MacOS Homebrew bug Workaround + continueOnError: true + - script: brew update && brew install libomp && brew link libomp --force + displayName: Install build dependencies + # Only build native assets to avoid conflicts. + - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish macOS_M1 package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + ################################################################################ + - job: Windows_arm64 + ################################################################################ + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + # Only build native assets to avoid conflicts. + - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Windows_arm64 package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown + + ################################################################################ + - job: Windows_x86 + ################################################################################ + variables: + _TeamName: DotNetCore + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + # Only build native assets to avoid conflicts. + - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Windows_x86 package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown + + ################################################################################ + - job: Windows_x64 + ################################################################################ + variables: + Codeql.SkipTaskAutoInjection: False # run CodeQL in this job + _TeamName: DotNetCore + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + + # Build both native and managed assets. + - script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true + displayName: Build + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - task: PublishBuildArtifacts@1 + displayName: Publish Windows_x64 package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown + + ################################################################################ + - template: /eng/common/templates/job/job.yml + parameters: + name: Package + ################################################################################ + artifacts: + publish: + artifacts: false + logs: true + manifests: true + enableMicrobuild: true + enablePublishUsingPipelines: true + enableSourceIndex: true + dependsOn: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - MacOS + - Windows_x86 + - Windows_x64 + - MacOS_Apple_Silicon + variables: + - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. + - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages + - _SignType: real + - _TeamName: DotNetCore + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(OfficialBuildId) + /p:DotNetPublishUsingPipelines=$true + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + + # Download all agent packages from all previous phases + - task: DownloadBuildArtifacts@0 + displayName: Download package assets + inputs: + artifactName: pkgassets + downloadPath: $(Build.SourcesDirectory)/artifacts + + # Depending on the value of DotNetFinalVersionKind, the name of the package will change. + # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" + # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable + # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version + - script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) + displayName: Build Packages + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown + + - template: /eng/common/templates/job/publish-build-assets.yml + parameters: + dependsOn: Package + enablePublishBuildArtifacts: true + publishAssetsImmediately: true + publishUsingPipelines: true + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + - template: /eng/common/templates/post-build/post-build.yml + parameters: + publishingInfraVersion: 3 + # This is to enable SDL runs part of Post-Build Validation Stage + SDLValidationParameters: + enable: true + publishGdn: true + continueOnError: false + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL https://devdiv.visualstudio.com/ + -TsaProjectName DEVDIV + -TsaNotificationEmail mlnetcore@microsoft.com + -TsaCodebaseAdmin REDMOND\ericstj + -TsaBugAreaPath "DevDiv\Net Libraries" + -TsaIterationPath DevDiv + -TsaRepositoryName machinelearning + -TsaCodebaseName machinelearning + -TsaPublish $True' \ No newline at end of file From b296634f9b75bba291196ab33a667ae58f292f24 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 11:46:20 -0500 Subject: [PATCH 03/26] gotta use name/value syntax in variables mixed w/ templates --- build/vsts-ci.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index c5cc4b284a..901c838a09 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -3,14 +3,22 @@ ################################################################################ variables: - BuildConfig: Release - _BuildConfig: $(BuildConfig) - OfficialBuildId: $(BUILD.BUILDNUMBER) - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - DOTNET_MULTILEVEL_LOOKUP: 0 - Codeql.Enabled: true - Codeql.SkipTaskAutoInjection: True #default to not inject CodeQL tasks, we'll enable it in a single job. +- name: BuildConfig + value: Release +- name: _BuildConfig: + value: $(BuildConfig) +- name: OfficialBuildId: + value: $(BUILD.BUILDNUMBER) +- name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 +- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 +- name: DOTNET_MULTILEVEL_LOOKUP + value: 0 +- name: Codeql.Enabled + value: true +- name: Codeql.SkipTaskAutoInjection + value: True #default to not inject CodeQL tasks, we'll enable it in a single job. - template: /eng/common-variables.yml - template: /eng/common/templates/variables/pool-providers.yml From bf2d4caf58da04997202d8d8dcaff853c12f0a39 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 11:47:33 -0500 Subject: [PATCH 04/26] typos --- build/vsts-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 901c838a09..32bc1328f6 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -5,9 +5,9 @@ variables: - name: BuildConfig value: Release -- name: _BuildConfig: +- name: _BuildConfig value: $(BuildConfig) -- name: OfficialBuildId: +- name: OfficialBuildId value: $(BUILD.BUILDNUMBER) - name: DOTNET_CLI_TELEMETRY_OPTOUT value: 1 From 0d18029582b6e7756d9be88528e5c76c7e8ea2a0 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 11:48:49 -0500 Subject: [PATCH 05/26] copy-paste error --- build/vsts-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 32bc1328f6..ade9ac72db 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -19,8 +19,7 @@ variables: value: true - name: Codeql.SkipTaskAutoInjection value: True #default to not inject CodeQL tasks, we'll enable it in a single job. -- template: /eng/common-variables.yml -- template: /eng/common/templates/variables/pool-providers.yml +- template: /eng/common/templates/variables/pool-providers.yml@self resources: repositories: From 68214f4749d6cc5386eeb99929fc1e0cdb7f4c32 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 11:49:58 -0500 Subject: [PATCH 06/26] use more @self --- build/vsts-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index ade9ac72db..8bd8fefee5 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -139,7 +139,7 @@ extends: artifactName: pkgassets artifactType: container - - template: /eng/common/templates/steps/generate-sbom.yml + - template: /eng/common/templates/steps/generate-sbom.yml@self ################################################################################ - job: MacOS_Apple_Silicon @@ -167,7 +167,7 @@ extends: artifactName: pkgassets artifactType: container - - template: /eng/common/templates/steps/generate-sbom.yml + - template: /eng/common/templates/steps/generate-sbom.yml@self ################################################################################ - job: Windows_arm64 @@ -188,7 +188,7 @@ extends: artifactName: pkgassets artifactType: container - - template: /eng/common/templates/steps/generate-sbom.yml + - template: /eng/common/templates/steps/generate-sbom.yml@self # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown @@ -215,7 +215,7 @@ extends: artifactName: pkgassets artifactType: container - - template: /eng/common/templates/steps/generate-sbom.yml + - template: /eng/common/templates/steps/generate-sbom.yml@self # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown @@ -250,7 +250,7 @@ extends: artifactName: pkgassets artifactType: container - - template: /eng/common/templates/steps/generate-sbom.yml + - template: /eng/common/templates/steps/generate-sbom.yml@self # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown From 4e6269fb2e671d024a6448d14128854166a8de0d Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 11:52:48 -0500 Subject: [PATCH 07/26] use even more @self --- build/vsts-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 8bd8fefee5..e62547920f 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -257,7 +257,7 @@ extends: displayName: Dotnet Server Shutdown ################################################################################ - - template: /eng/common/templates/job/job.yml + - template: /eng/common/templates/job/job.yml@self parameters: name: Package ################################################################################ @@ -312,7 +312,7 @@ extends: - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown - - template: /eng/common/templates/job/publish-build-assets.yml + - template: /eng/common/templates/job/publish-build-assets.yml@self parameters: dependsOn: Package enablePublishBuildArtifacts: true @@ -323,7 +323,7 @@ extends: image: 1es-windows-2022-pt os: windows - - template: /eng/common/templates/post-build/post-build.yml + - template: /eng/common/templates/post-build/post-build.yml@self parameters: publishingInfraVersion: 3 # This is to enable SDL runs part of Post-Build Validation Stage From c9f0d7934ab6b38efcb55293220a5b12a902af2a Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 12:08:23 -0500 Subject: [PATCH 08/26] Try compliant replacement for PublishBuildArtifacts --- build/vsts-ci.yml | 120 +++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 72 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index e62547920f..79301651dd 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -56,20 +56,17 @@ extends: image: 1es-ubuntu-2204-pt os: linux container: CentosContainer + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true displayName: Build Native Assets - - task: PublishBuildArtifacts@1 - displayName: Publish Linux package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - ################################################################################ - job: Linux_arm ################################################################################ @@ -80,20 +77,17 @@ extends: image: 1es-ubuntu-2204-pt os: linux container: UbuntuCrossArmContainer + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true displayName: Build Native Assets - - task: PublishBuildArtifacts@1 - displayName: Publish Linux_arm package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - ################################################################################ - job: Linux_arm64 ################################################################################ @@ -104,20 +98,17 @@ extends: image: 1es-ubuntu-2204-pt os: linux container: UbuntuCrossArm64Container + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true displayName: Build Native Assets - - task: PublishBuildArtifacts@1 - displayName: Publish Linux_arm64 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - ################################################################################ - job: MacOS ################################################################################ @@ -125,6 +116,12 @@ extends: name: Azure Pipelines vmImage: macOS-12 os: macOS + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula displayName: Install build dependencies @@ -132,15 +129,6 @@ extends: - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true displayName: Build Native Assets - - task: PublishBuildArtifacts@1 - displayName: Publish macOS package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - ################################################################################ - job: MacOS_Apple_Silicon ################################################################################ @@ -148,6 +136,12 @@ extends: name: Azure Pipelines vmImage: macOS-12 os: macOS + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/2322#issuecomment-749211076 - script: | @@ -160,15 +154,6 @@ extends: - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true displayName: Build Native Assets - - task: PublishBuildArtifacts@1 - displayName: Publish macOS_M1 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - ################################################################################ - job: Windows_arm64 ################################################################################ @@ -176,20 +161,17 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt os: windows + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: # Only build native assets to avoid conflicts. - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true displayName: Build Native Assets - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_arm64 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown @@ -203,20 +185,17 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt os: windows + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: # Only build native assets to avoid conflicts. - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true displayName: Build Native Assets - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_x86 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown @@ -231,6 +210,12 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt os: windows + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + artifactType: container steps: # Build both native and managed assets. @@ -243,15 +228,6 @@ extends: verbosity: 'Verbose' alertWarningLevel: 'High' - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_x64 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml@self - # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown From 98428d3bb7062665dd9a4efa3d5a8763ebfa8836 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 12:13:15 -0500 Subject: [PATCH 09/26] fix pool --- build/vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 79301651dd..abbf407ab7 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -32,7 +32,7 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: pool: - name: Azure-Pipelines-1ESPT-ExDShared + name: $(DncEngInternalBuildPool) image: windows-2022 os: windows containers: From 33b623d32d7bdb2b5b7ecebeef48bc910a953e52 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 12:14:15 -0500 Subject: [PATCH 10/26] fix image --- build/vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index abbf407ab7..a99fad029e 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -33,7 +33,7 @@ extends: parameters: pool: name: $(DncEngInternalBuildPool) - image: windows-2022 + image: 1es-windows-2022-pt os: windows containers: CentosContainer: From 369c134e4d1fe88f11b68e3eb67b267906b7a613 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 13:28:26 -0500 Subject: [PATCH 11/26] pipelineArtifact->buildArtifact --- build/vsts-ci.yml | 56 ++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index a99fad029e..caffc29c03 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -58,10 +58,9 @@ extends: container: CentosContainer templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true @@ -79,10 +78,9 @@ extends: container: UbuntuCrossArmContainer templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true @@ -100,10 +98,9 @@ extends: container: UbuntuCrossArm64Container templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true @@ -118,10 +115,9 @@ extends: os: macOS templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: - script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && rm '/usr/local/bin/2to3-3.11' && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula displayName: Install build dependencies @@ -138,10 +134,9 @@ extends: os: macOS templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/2322#issuecomment-749211076 - script: | @@ -163,10 +158,9 @@ extends: os: windows templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: # Only build native assets to avoid conflicts. - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true @@ -187,10 +181,9 @@ extends: os: windows templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: # Only build native assets to avoid conflicts. - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true @@ -212,10 +205,9 @@ extends: os: windows templateContext: outputs: - - output: pipelineArtifact - targetPath: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: pkgassets - artifactType: container + - output: buildArtifact + PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + ArtifactName: pkgassets steps: # Build both native and managed assets. From 3e12489c622543e41eb28bef2e684eadd2d9fbbd Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 13:29:44 -0500 Subject: [PATCH 12/26] typo --- build/vsts-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index caffc29c03..302161139d 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -58,7 +58,7 @@ extends: container: CentosContainer templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: @@ -78,7 +78,7 @@ extends: container: UbuntuCrossArmContainer templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: @@ -98,7 +98,7 @@ extends: container: UbuntuCrossArm64Container templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: @@ -115,7 +115,7 @@ extends: os: macOS templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: @@ -134,7 +134,7 @@ extends: os: macOS templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: @@ -158,7 +158,7 @@ extends: os: windows templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: @@ -181,7 +181,7 @@ extends: os: windows templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: @@ -205,7 +205,7 @@ extends: os: windows templateContext: outputs: - - output: buildArtifact + - output: buildArtifacts PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets ArtifactName: pkgassets steps: From 0025d9dfca8d81f5f2037ede4811c48bbf710340 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 1 Mar 2024 16:53:53 -0500 Subject: [PATCH 13/26] use vs2019 as before, not 2022 --- build/vsts-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 302161139d..df61abc5f3 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -33,7 +33,7 @@ extends: parameters: pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2019-pt os: windows containers: CentosContainer: @@ -154,7 +154,7 @@ extends: ################################################################################ pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2019-pt os: windows templateContext: outputs: @@ -177,7 +177,7 @@ extends: _TeamName: DotNetCore pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2019-pt os: windows templateContext: outputs: @@ -201,7 +201,7 @@ extends: _TeamName: DotNetCore pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2019-pt os: windows templateContext: outputs: @@ -258,7 +258,7 @@ extends: /p:DotNetPublishUsingPipelines=$true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2019-pt os: windows steps: @@ -288,7 +288,7 @@ extends: publishUsingPipelines: true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2019-pt os: windows - template: /eng/common/templates/post-build/post-build.yml@self From 89b4f05f326538325473b0b82f366bc9e6da8405 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Mon, 4 Mar 2024 09:40:42 -0500 Subject: [PATCH 14/26] try enabling cfg and spectre mitigation on Windows --- src/Native/CpuMathNative/CMakeLists.txt | 5 +++++ src/Native/FastTreeNative/CMakeLists.txt | 5 +++++ src/Native/LdaNative/CMakeLists.txt | 5 +++++ src/Native/MatrixFactorizationNative/CMakeLists.txt | 5 +++++ src/Native/MklProxyNative/CMakeLists.txt | 5 +++++ src/Native/OneDalNative/CMakeLists.txt | 4 ++++ src/Native/SymSgdNative/CMakeLists.txt | 4 ++++ 7 files changed, 33 insertions(+) diff --git a/src/Native/CpuMathNative/CMakeLists.txt b/src/Native/CpuMathNative/CMakeLists.txt index 3141e6c1e0..3eb8a52384 100644 --- a/src/Native/CpuMathNative/CMakeLists.txt +++ b/src/Native/CpuMathNative/CMakeLists.txt @@ -13,4 +13,9 @@ endif() add_library(CpuMathNative SHARED ${SOURCES} ${RESOURCES}) +if (MSVC AND NOT MSVC_VERSION LESS 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /guard:cf") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") +endif() + install_library_and_symbols (CpuMathNative) diff --git a/src/Native/FastTreeNative/CMakeLists.txt b/src/Native/FastTreeNative/CMakeLists.txt index e03e2baaa2..a908d7c82a 100644 --- a/src/Native/FastTreeNative/CMakeLists.txt +++ b/src/Native/FastTreeNative/CMakeLists.txt @@ -17,4 +17,9 @@ endif() add_library(FastTreeNative SHARED ${SOURCES} ${RESOURCES}) +if (MSVC AND NOT MSVC_VERSION LESS 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /guard:cf") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") +endif() + install_library_and_symbols (FastTreeNative) diff --git a/src/Native/LdaNative/CMakeLists.txt b/src/Native/LdaNative/CMakeLists.txt index efc816c5cc..2159463a15 100644 --- a/src/Native/LdaNative/CMakeLists.txt +++ b/src/Native/LdaNative/CMakeLists.txt @@ -16,6 +16,11 @@ set(SOURCES add_library(LdaNative SHARED ${SOURCES} ${RESOURCES}) +if (MSVC AND NOT MSVC_VERSION LESS 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /guard:cf") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") +endif() + if (${ARCHITECTURE} STREQUAL "arm") if (WIN32) target_link_libraries(LdaNative shell32.lib) diff --git a/src/Native/MatrixFactorizationNative/CMakeLists.txt b/src/Native/MatrixFactorizationNative/CMakeLists.txt index 4f45ccacd8..092e2c3d75 100644 --- a/src/Native/MatrixFactorizationNative/CMakeLists.txt +++ b/src/Native/MatrixFactorizationNative/CMakeLists.txt @@ -43,6 +43,11 @@ endif() add_library(MatrixFactorizationNative SHARED ${SOURCES} ${RESOURCES}) +if (MSVC AND NOT MSVC_VERSION LESS 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /guard:cf") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") +endif() + if (${ARCHITECTURE} STREQUAL "arm") if (WIN32) target_link_libraries(MatrixFactorizationNative shell32.lib) diff --git a/src/Native/MklProxyNative/CMakeLists.txt b/src/Native/MklProxyNative/CMakeLists.txt index caea9450cb..7b939f3cf2 100644 --- a/src/Native/MklProxyNative/CMakeLists.txt +++ b/src/Native/MklProxyNative/CMakeLists.txt @@ -18,6 +18,11 @@ endif() add_library(MklProxyNative SHARED ${SOURCES} ${RESOURCES}) target_link_libraries(MklProxyNative PUBLIC ${MKL_LIBRARY}) +if (MSVC AND NOT MSVC_VERSION LESS 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /guard:cf") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") +endif() + if(APPLE) set_target_properties(MklProxyNative PROPERTIES INSTALL_RPATH "@loader_path") endif() diff --git a/src/Native/OneDalNative/CMakeLists.txt b/src/Native/OneDalNative/CMakeLists.txt index c40fa9abd7..7028f3324b 100644 --- a/src/Native/OneDalNative/CMakeLists.txt +++ b/src/Native/OneDalNative/CMakeLists.txt @@ -24,5 +24,9 @@ endif() set(SOURCES OneDalAlgorithms.cpp) add_library(OneDalNative SHARED ${SOURCES} ${RESOURCES}) +if (MSVC AND NOT MSVC_VERSION LESS 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /guard:cf") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") +endif() target_link_libraries(OneDalNative PUBLIC ${WRAPPER_LINK_OPTIONS}) install_library_and_symbols(OneDalNative) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index f40d1a46b4..01652d2aab 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -39,6 +39,10 @@ endif() add_definitions(-DUSE_OMP) add_library(SymSgdNative SHARED ${SOURCES} ${RESOURCES}) +if (MSVC AND NOT MSVC_VERSION LESS 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /guard:cf") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") +endif() target_link_libraries(SymSgdNative PUBLIC ${MKL_LIBRARY} PUBLIC ${OPENMP_LIBRARY}) if(APPLE) From c6ead2b37bcf73c6b7bcebe927c4623f0c9f61f0 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Mon, 4 Mar 2024 10:23:55 -0500 Subject: [PATCH 15/26] Update Arcade --- eng/common/native/init-distro-rid.sh | 80 ++-- eng/common/native/init-os-and-arch.sh | 7 +- eng/common/post-build/publish-using-darc.ps1 | 4 +- eng/common/templates-official/job/job.yml | 255 ++++++++++++ .../templates-official/job/onelocbuild.yml | 112 +++++ .../job/publish-build-assets.yml | 157 +++++++ .../templates-official/job/source-build.yml | 67 +++ .../job/source-index-stage1.yml | 67 +++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 97 +++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 24 ++ .../post-build/post-build.yml | 285 +++++++++++++ .../post-build/setup-maestro-vars.yml | 70 ++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../steps/component-governance.yml | 13 + .../steps/generate-sbom.yml | 48 +++ .../templates-official/steps/publish-logs.yml | 49 +++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 91 +++++ .../templates-official/steps/source-build.yml | 129 ++++++ .../variables/pool-providers.yml | 45 ++ .../variables/sdl-variables.yml | 7 + eng/common/templates/job/execute-sdl.yml | 112 +++-- eng/common/templates/job/job.yml | 125 ++++-- .../templates/job/publish-build-assets.yml | 110 +++-- eng/common/templates/job/source-build.yml | 53 ++- .../templates/job/source-index-stage1.yml | 15 +- eng/common/templates/jobs/jobs.yml | 51 ++- eng/common/templates/jobs/source-build.yml | 27 +- .../templates/post-build/common-variables.yml | 42 +- .../templates/post-build/post-build.yml | 383 ++++++++++-------- .../post-build/setup-maestro-vars.yml | 118 +++--- .../templates/steps/component-governance.yml | 3 +- eng/common/templates/steps/execute-sdl.yml | 41 +- eng/common/templates/steps/generate-sbom.yml | 38 +- eng/common/templates/steps/publish-logs.yml | 34 +- eng/common/templates/steps/source-build.yml | 44 +- .../templates/variables/pool-providers.yml | 62 ++- .../templates/variables/sdl-variables.yml | 4 +- eng/common/tools.ps1 | 14 +- eng/common/tools.sh | 8 +- 43 files changed, 2553 insertions(+), 469 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/pool-providers.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index de1687b2cc..5dcbfd700f 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # getNonPortableDistroRid # @@ -11,21 +11,20 @@ # non-portable rid getNonPortableDistroRid() { - local targetOs="$1" - local targetArch="$2" - local rootfsDir="$3" - local nonPortableRid="" + targetOs="$1" + targetArch="$2" + rootfsDir="$3" + nonPortableRid="" if [ "$targetOs" = "linux" ]; then + # shellcheck disable=SC1091 if [ -e "${rootfsDir}/etc/os-release" ]; then - source "${rootfsDir}/etc/os-release" - - if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then - # remove the last version digit - VERSION_ID="${VERSION_ID%.*}" + . "${rootfsDir}/etc/os-release" + if [ "${ID}" = "rhel" ] || [ "${ID}" = "rocky" ] || [ "${ID}" = "alpine" ]; then + VERSION_ID="${VERSION_ID%.*}" # Remove the last version digit for these distros fi - if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then + if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" else # Rolling release distros either do not set VERSION_ID, set it as blank or @@ -33,45 +32,45 @@ getNonPortableDistroRid() # so omit it here to be consistent with everything else. nonPortableRid="${ID}-${targetArch}" fi - elif [ -e "${rootfsDir}/android_platform" ]; then - source "$rootfsDir"/android_platform + # shellcheck disable=SC1091 + . "${rootfsDir}/android_platform" nonPortableRid="$RID" fi fi if [ "$targetOs" = "freebsd" ]; then - # $rootfsDir can be empty. freebsd-version is shell script and it should always work. - __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; }) + # $rootfsDir can be empty. freebsd-version is a shell script and should always work. + __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" - elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then + elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" elif [ "$targetOs" = "illumos" ]; then __uname_version=$(uname -v) case "$__uname_version" in omnios-*) - __omnios_major_version=$(echo "${__uname_version:8:2}") - nonPortableRid=omnios."$__omnios_major_version"-"$targetArch" - ;; + __omnios_major_version=$(echo "$__uname_version" | cut -c9-10) + nonPortableRid="omnios.$__omnios_major_version-${targetArch}" + ;; joyent_*) - __smartos_major_version=$(echo "${__uname_version:7:4}") - nonPortableRid=smartos."$__smartos_major_version"-"$targetArch" - ;; - illumos_*) - nonPortableRid=openindiana-"$targetArch" - ;; + __smartos_major_version=$(echo "$__uname_version" | cut -c9-10) + nonPortableRid="smartos.$__smartos_major_version-${targetArch}" + ;; + *) + nonPortableRid="illumos-${targetArch}" + ;; esac elif [ "$targetOs" = "solaris" ]; then __uname_version=$(uname -v) - __solaris_major_version=$(echo "${__uname_version%.*}") - nonPortableRid=solaris."$__solaris_major_version"-"$targetArch" + __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) + nonPortableRid="solaris.$__solaris_major_version-${targetArch}" elif [ "$targetOs" = "haiku" ]; then - __uname_release=$(uname -r) + __uname_release="$(uname -r)" nonPortableRid=haiku.r"$__uname_release"-"$targetArch" fi - echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')" + echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' } # initDistroRidGlobal @@ -85,26 +84,23 @@ getNonPortableDistroRid() # None # # Notes: -# -# It is important to note that the function does not return anything, but it -# exports the following variables on success: -# -# __DistroRid : Non-portable rid of the target platform. -# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. -# +# It is important to note that the function does not return anything, but it +# exports the following variables on success: +# __DistroRid : Non-portable rid of the target platform. +# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. initDistroRidGlobal() { - local targetOs="$1" - local targetArch="$2" - local rootfsDir="" - if [ "$#" -ge 3 ]; then + targetOs="$1" + targetArch="$2" + rootfsDir="" + if [ $# -ge 3 ]; then rootfsDir="$3" fi if [ -n "${rootfsDir}" ]; then # We may have a cross build. Check for the existence of the rootfsDir if [ ! -e "${rootfsDir}" ]; then - echo "Error rootfsDir has been passed, but the location is not valid." + echo "Error: rootfsDir has been passed, but the location is not valid." exit 1 fi fi @@ -119,7 +115,7 @@ initDistroRidGlobal() STRINGS="$(command -v llvm-strings || true)" fi - # Check for musl-based distros (e.g Alpine Linux, Void Linux). + # Check for musl-based distros (e.g. Alpine Linux, Void Linux). if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then __PortableTargetOS="linux-musl" diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh index e693617a6c..38921d4338 100644 --- a/eng/common/native/init-os-and-arch.sh +++ b/eng/common/native/init-os-and-arch.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # Use uname to determine what the OS is. OSName=$(uname -s | tr '[:upper:]' '[:lower:]') @@ -35,6 +35,10 @@ fi case "$CPUName" in arm64|aarch64) arch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + arch=arm + fi ;; loongarch64) @@ -50,6 +54,7 @@ case "$CPUName" in ;; armv7l|armv8l) + # shellcheck disable=SC1091 if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then arch=armel else diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 1e779fec4d..5a3a32ea8d 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -12,7 +12,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() @@ -46,7 +46,7 @@ try { } Write-Host 'done.' -} +} catch { Write-Host $_ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 0000000000..352607308f --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,255 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - publish: artifacts/log + artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: Publish logs + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration + artifact: BuildConfiguration + displayName: Publish build retry configuration + continueOnError: true diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 0000000000..ba9ba49303 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 0000000000..5f54135569 --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,157 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - checkout: self + fetchDepth: 3 + clean: true + + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 0000000000..50f04e642a --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals build.ubuntu.1804.amd64 + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2-pt + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 0000000000..757af7c7c4 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,67 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20231213.4 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: windows.vs2022.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022.amd64 + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET 8 SDK + inputs: + packageType: sdk + version: 8.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 0000000000..b68d3c2f31 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 0000000000..857a0f8ba4 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 0000000000..08e5db9bb1 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 0000000000..b9ede10bf0 --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,24 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.11 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 0000000000..5c98fe1c0f --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 0000000000..0c87f149a4 --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 0000000000..da669030da --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 0000000000..f67a210d62 --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 0000000000..0ecec47b0c --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 0000000000..488b560e8b --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 0000000000..84b2f559c5 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,49 @@ +parameters: + StageLabel: '' + JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/templates-official/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.11' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 0000000000..83d97a26a0 --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 0000000000..3eb7e2d5f8 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 0000000000..b1db70842f --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/sb/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 0000000000..beab7d1bfb --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022-pt + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 0000000000..dbdd66d4a4 --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 69cf5747fd..7870f93bc1 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,58 +1,69 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. -# Outputs were added with YAML Conditional expressions generated using AI in a best attempt to preserve the boolean logic that would have lead to the tasks producing outputs executing before conversion. You may need to review the generated expressions for correctness of the execution logic, and you will need to manually translate any cross-template parameters. -# Output from "eng\common\templates\steps\execute-sdl.yml" added to job "Run SDL tool" with conditionals extracted using AI from the following files: "eng\common\templates\steps\execute-sdl.yml". -# Output from "eng\common\templates\steps\execute-sdl.yml" added to job "Run SDL tool" with conditionals extracted using AI from the following files: "eng\common\templates\steps\execute-sdl.yml". - parameters: - enable: 'false' - overrideParameters: '' - additionalParameters: '' + enable: 'false' # Whether the SDL validation job should execute or not + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - sdlContinueOnError: false + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; + # optional: determines if build artifacts should be downloaded. downloadArtifacts: true + # optional: determines if this job should search the directory of downloaded artifacts for + # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks. extractArchiveArtifacts: false - dependsOn: '' - artifactNames: '' + dependsOn: '' # Optional: dependencies of the job + artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts + # Usage: + # artifactNames: + # - 'BlobArtifacts' + # - 'Artifacts_Windows_NT_Release' + # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, + # not pipeline artifacts, so doesn't affect the use of this parameter. pipelineArtifactNames: [] + jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) variables: - - group: DotNet-VSTS-Bot - - name: AzDOProjectName - value: ${{ parameters.AzDOProjectName }} - - name: AzDOPipelineId - value: ${{ parameters.AzDOPipelineId }} - - name: AzDOBuildId - value: ${{ parameters.AzDOBuildId }} - - template: /eng/common/templates/variables/sdl-variables.yml@self - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - template: /eng/common/templates/variables/pool-providers.yml@self - templateContext: - outputs: - - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - - output: pipelineArtifact - displayName: 'Publish GuardianConfiguration' - condition: succeededOrFailed() - targetPath: $(Agent.BuildDirectory)/.gdn - artifactName: GuardianConfiguration - - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - - output: pipelineArtifact - displayName: 'Publish SARIF files to CodeAnalysisLogs container' - condition: succeededOrFailed() - targetPath: $(Build.SourcesDirectory)/CodeAnalysisLogs - artifactName: CodeAnalysisLogs + - group: DotNet-VSTS-Bot + - name: AzDOProjectName + value: ${{ parameters.AzDOProjectName }} + - name: AzDOPipelineId + value: ${{ parameters.AzDOPipelineId }} + - name: AzDOBuildId + value: ${{ parameters.AzDOBuildId }} + - template: /eng/common/templates/variables/sdl-variables.yml + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + - template: /eng/common/templates/variables/pool-providers.yml + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64 steps: - checkout: self clean: true + + # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self + - template: /eng/common/templates/post-build/setup-maestro-vars.yml + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: - ${{ each artifactName in parameters.artifactNames }}: @@ -80,6 +91,7 @@ jobs: itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts checkDownloadedFiles: true + - ${{ each artifactName in parameters.pipelineArtifactNames }}: - task: DownloadPipelineArtifact@2 displayName: Download Pipeline Artifacts @@ -92,24 +104,36 @@ jobs: artifactName: ${{ artifactName }} downloadPath: $(Build.ArtifactStagingDirectory)\artifacts checkDownloadedFiles: true - - powershell: eng/common/sdl/trim-assets-version.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts + + - powershell: eng/common/sdl/trim-assets-version.ps1 + -InputPath $(Build.ArtifactStagingDirectory)\artifacts displayName: Trim the version from the NuGet packages continueOnError: ${{ parameters.sdlContinueOnError }} - - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts + + - powershell: eng/common/sdl/extract-artifact-packages.ps1 + -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts + -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts + + - powershell: eng/common/sdl/extract-artifact-packages.ps1 + -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts + -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} + - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - - powershell: eng/common/sdl/extract-artifact-archives.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts + - powershell: eng/common/sdl/extract-artifact-archives.ps1 + -InputPath $(Build.ArtifactStagingDirectory)\artifacts + -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts displayName: Extract Archive Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - template: /eng/common/templates/steps/execute-sdl.yml@self + + - template: /eng/common/templates/steps/execute-sdl.yml parameters: overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} overrideParameters: ${{ parameters.overrideParameters }} additionalParameters: ${{ parameters.additionalParameters }} publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index c3ac7e4d0b..01c0dd995e 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,14 +1,8 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. -# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish pipeline artifacts' in the templateContext section. -# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish logs' in the templateContext section. -# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Logs' in the templateContext section. -# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish build retry configuration' in the templateContext section. -# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. -# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. -# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. -# Output added to job "${{ parameters.name }}" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job cancelTimeoutInMinutes: '' condition: '' container: '' @@ -21,6 +15,9 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md artifacts: '' enableMicrobuild: false enablePublishBuildArtifacts: false @@ -36,29 +33,41 @@ parameters: name: '' preSteps: [] runAsPublic: false +# Sbom related params enableSbom: true PackageVersion: 7.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + jobs: - job: ${{ parameters.name }} + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} + ${{ if ne(parameters.container, '') }}: container: ${{ parameters.container }} + ${{ if ne(parameters.continueOnError, '') }}: continueOnError: ${{ parameters.continueOnError }} + ${{ if ne(parameters.dependsOn, '') }}: dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.displayName, '') }}: displayName: ${{ parameters.displayName }} + ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} + ${{ if ne(parameters.strategy, '') }}: strategy: ${{ parameters.strategy }} + ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE @@ -66,57 +75,53 @@ jobs: - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - name: EnableRichCodeNavigation value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY value: 3,2000 - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] - ${{ if ne(variable.name, '') }}: - name: ${{ variable.name }} value: ${{ variable.value }} + + # handle variable groups - ${{ if ne(variable.group, '') }}: - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] - ${{ if ne(variable.template, '') }}: - template: ${{ variable.template }} ${{ if ne(variable.parameters, '') }}: parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - ${{ each pair in variable }}: - name: ${{ pair.key }} value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-HelixApi-Access + ${{ if ne(parameters.workspace, '') }}: workspace: ${{ parameters.workspace }} - templateContext: - outputs: - - ${{ if ne(parameters.artifacts.publish, '') }}: - - output: pipelineArtifact - displayName: 'Publish pipeline artifacts' - condition: always() - targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' - publishLocation: Container - artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - output: pipelineArtifact - displayName: 'Publish logs' - condition: always() - targetPath: artifacts/log - artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - output: pipelineArtifact - displayName: 'Publish Logs' - condition: always() - targetPath: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - publishLocation: Container - artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - output: pipelineArtifact - displayName: 'Publish build retry configuration' - targetPath: $(Build.SourcesDirectory)\eng\common\BuildConfiguration - artifactName: BuildConfiguration + steps: - ${{ if ne(parameters.preSteps, '') }}: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - task: MicroBuildSigningPlugin@3 @@ -129,8 +134,10 @@ jobs: TeamName: $(_TeamName) continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@1 + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 inputs: @@ -138,8 +145,10 @@ jobs: artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + - ${{ each step in parameters.steps }}: - ${{ step }} + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - task: RichCodeNavIndexer@0 displayName: RichCodeNav Upload @@ -149,7 +158,8 @@ jobs: richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} continueOnError: true - - template: /eng/common/templates/steps/component-governance.yml@self + + - template: /eng/common/templates/steps/component-governance.yml parameters: ${{ if eq(parameters.disableComponentGovernance, '') }}: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: @@ -159,6 +169,7 @@ jobs: ${{ else }}: disableComponentGovernance: ${{ parameters.disableComponentGovernance }} componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 @@ -167,6 +178,7 @@ jobs: continueOnError: ${{ parameters.continueOnError }} env: TeamName: $(_TeamName) + - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - task: CopyFiles@2 @@ -181,8 +193,31 @@ jobs: SourceFolder: 'artifacts/packages' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: [] - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: [] + - task: PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - publish: artifacts/log + artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: Publish logs + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - task: PublishTestResults@2 displayName: Publish XUnit Test Results @@ -205,10 +240,16 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml@self + - template: /eng/common/templates/steps/generate-sbom.yml parameters: PackageVersion: ${{ parameters.packageVersion}} BuildDropPath: ${{ parameters.buildDropPath }} IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: [] \ No newline at end of file + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration + artifact: BuildConfiguration + displayName: Publish build retry configuration + continueOnError: true diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index a5e8767a1c..bb42240f86 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -1,63 +1,72 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. -# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish ReleaseConfigs Artifact' in the templateContext section. -# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish SymbolPublishingExclusionsFile Artifact' in the templateContext section. -# Output added to job "Asset_Registry_Publish" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. -# Output added to job "Asset_Registry_Publish" with YAML conditionals extracted using AI. Review this expression against the originating file for correctness. parameters: configuration: 'Debug' + + # Optional: condition for the job to run condition: '' + + # Optional: 'true' if future jobs should run even if this job fails continueOnError: false + + # Optional: dependencies of the job dependsOn: '' + + # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing publishAssetsImmediately: false + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + jobs: - job: Asset_Registry_Publish + dependsOn: ${{ parameters.dependsOn }} timeoutInMinutes: 150 + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: displayName: Publish Assets ${{ else }}: displayName: Publish to Build Asset Registry + variables: - - template: /eng/common/templates/variables/pool-providers.yml@self + - template: /eng/common/templates/variables/pool-providers.yml - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - - template: /eng/common/templates/post-build/common-variables.yml@self - templateContext: - outputs: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - output: pipelineArtifact - displayName: 'Publish ReleaseConfigs Artifact' - targetPath: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - publishLocation: Container - artifactName: ReleaseConfigs - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - output: pipelineArtifact - displayName: 'Publish SymbolPublishingExclusionsFile Artifact' - condition: eq(variables['SymbolExclusionFile'], 'true') - targetPath: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - publishLocation: Container - artifactName: ReleaseConfigs - - output: pipelineArtifact - displayName: 'Publish Logs' - condition: always() - targetPath: '$(Build.SourcesDirectory)/PostBuildLogs' - publishLocation: Container - artifactName: PostBuildLogs + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/templates/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Publishing-Internal + demands: ImageOverride -equals windows.vs2019.amd64 + steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - checkout: self fetchDepth: 3 clean: true + - task: DownloadBuildArtifacts@0 displayName: Download artifact inputs: @@ -66,14 +75,22 @@ jobs: checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: NuGetAuthenticate@1 + - task: PowerShell@2 displayName: Publish Build Assets inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -82,6 +99,14 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + + - task: PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + - task: powershell@2 displayName: Check if SymbolPublishingExclusionsFile.txt exists inputs: @@ -97,17 +122,34 @@ jobs: Write-Host "Symbols Exclusion file does not exists" Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" } + + - task: PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self + - template: /eng/common/templates/post-build/setup-maestro-vars.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: PowerShell@2 displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates/steps/publish-logs.yml@self + - template: /eng/common/templates/steps/publish-logs.yml parameters: - JobLabel: 'Publish_Artifacts_Logs' \ No newline at end of file + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 0d87b31edf..8a3deef2b7 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -1,35 +1,66 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. platform: {} + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) + ${{ each property in parameters.platform.jobProperties }}: ${{ property.key }}: ${{ property.value }} + ${{ if ne(parameters.platform.container, '') }}: container: ${{ parameters.platform.container }} + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} + workspace: clean: all - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish BuildLogs' - condition: succeededOrFailed() - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + steps: - - template: /eng/common/templates/steps/source-build.yml@self + - template: /eng/common/templates/steps/source-build.yml parameters: - platform: ${{ parameters.platform }} \ No newline at end of file + platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 1c86018a1b..b5a3e5c4a6 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,5 +1,3 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: runAsPublic: false sourceIndexPackageVersion: 1.0.1-20240129.2 @@ -10,6 +8,7 @@ parameters: condition: '' dependsOn: '' pool: '' + jobs: - job: SourceIndexStage1 dependsOn: ${{ parameters.dependsOn }} @@ -23,7 +22,8 @@ jobs: value: ${{ parameters.binlogPath }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: source-dot-net stage1 variables - - template: /eng/common/templates/variables/pool-providers.yml@self + - template: /eng/common/templates/variables/pool-providers.yml + ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} ${{ if eq(parameters.pool, '') }}: @@ -34,9 +34,11 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2022.amd64 + steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} + - task: UseDotNet@2 displayName: Use .NET 8 SDK inputs: @@ -44,17 +46,22 @@ jobs: version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) + - script: | $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) + - script: ${{ parameters.sourceIndexBuildCommand }} displayName: Build Repository + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output displayName: Process Binlog into indexable sln + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) displayName: Upload stage1 artifacts to source index env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) \ No newline at end of file + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 78668e65e0..289bb2396c 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,48 +1,82 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md continueOnError: false + + # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates/jobs/source-build.yml for options sourceBuildParameters: [] + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build enabled: false + # Optional: Include toolset dependencies in the generated graph files includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false + enableSourceIndex: false sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + jobs: - ${{ each job in parameters.jobs }}: - - template: /eng/common/templates/job/job.yml@self - parameters: + - template: ../job/job.yml + parameters: + # pass along parameters ${{ each parameter in parameters }}: ${{ if ne(parameter.key, 'jobs') }}: ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties ${{ each property in job }}: ${{ if ne(property.key, 'job') }}: ${{ property.key }}: ${{ property.value }} + name: ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates/jobs/source-build.yml@self + - template: /eng/common/templates/jobs/source-build.yml parameters: allCompletedJobId: Source_Build_Complete ${{ each parameter in parameters.sourceBuildParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: /eng/common/templates/job/source-index-stage1.yml@self + - template: ../job/source-index-stage1.yml parameters: runAsPublic: ${{ parameters.runAsPublic }} ${{ each parameter in parameters.sourceIndexParams }}: ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: /eng/common/templates/job/publish-build-assets.yml@self + - template: ../job/publish-build-assets.yml parameters: continueOnError: ${{ parameters.continueOnError }} dependsOn: @@ -54,9 +88,10 @@ jobs: - ${{ job.job }} - ${{ if eq(parameters.enableSourceBuild, true) }}: - Source_Build_Complete + runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} \ No newline at end of file + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index fdeaa59ff6..a15b07eb51 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -1,13 +1,28 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. allCompletedJobId: '' + + # See /eng/common/templates/job/source-build.yml jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. platforms: [] + jobs: + - ${{ if ne(parameters.allCompletedJobId, '') }}: - job: ${{ parameters.allCompletedJobId }} displayName: Source-Build Complete @@ -17,13 +32,15 @@ jobs: - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - ${{ if eq(length(parameters.platforms), 0) }}: - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + - ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates/job/source-build.yml@self + - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + - ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates/job/source-build.yml@self + - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} \ No newline at end of file + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index e29be13b71..b9ede10bf0 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,20 +1,24 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. variables: -- group: Publish-Build-Assets -- name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} -- name: MaestroApiEndPoint - value: "https://maestro.dot.net" -- name: MaestroApiAccessToken - value: $(MaestroAccessToken) -- name: MaestroApiVersion - value: "2020-02-20" -- name: SourceLinkCLIVersion - value: 3.0.0 -- name: SymbolToolVersion - value: 1.0.1 -- name: BinlogToolVersion - value: 1.0.11 -- name: runCodesignValidationInjection - value: false \ No newline at end of file + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.11 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 3e0410a5d5..ee70e2b399 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,191 +1,235 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: -- name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 -- name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 -- name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' -- name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false -- name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true -- name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false -- name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true -- name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true -- name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true -- name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' -- name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' -- name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' -- name: validateDependsOn - type: object - default: - - build -- name: publishDependsOn - type: object - default: - - Validate -- name: publishAssetsImmediately - type: boolean - default: false + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + stages: - ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate Build Assets variables: - - template: /eng/common/templates/post-build/common-variables.yml@self - - template: /eng/common/templates/variables/pool-providers.yml@self + - template: common-variables.yml + - template: /eng/common/templates/variables/pool-providers.yml jobs: - - job: '' + - job: displayName: NuGet Validation condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd + # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 + steps: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - job: '' + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: displayName: Signing Validation condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd + # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Logs' - condition: always() - targetPath: '$(Build.SourcesDirectory)/PostBuildLogs' - publishLocation: Container - artifactName: PostBuildLogs steps: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' ${{ parameters.signingValidationAdditionalParameters }} - - template: /eng/common/templates/steps/publish-logs.yml@self - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - job: '' + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: displayName: SourceLink Validation condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd + # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Extract/ -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - template: /eng/common/templates/job/execute-sdl.yml@self + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml parameters: enable: ${{ parameters.SDLValidationParameters.enable }} publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }} @@ -193,6 +237,7 @@ stages: continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: @@ -201,27 +246,37 @@ stages: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - - template: /eng/common/templates/post-build/common-variables.yml@self - - template: /eng/common/templates/variables/pool-providers.yml@self + - template: common-variables.yml + - template: /eng/common/templates/variables/pool-providers.yml jobs: - - job: '' + - job: displayName: Publish Using Darc timeoutInMinutes: 120 pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: VSEngSS-MicroBuild2022-1ES demands: Cmd + # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml@self - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: NuGetAuthenticate@1 - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' \ No newline at end of file + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index ad39920d70..0c87f149a4 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,62 +1,70 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: BARBuildId: '' PromoteToChannelIds: '' + steps: -- ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true -- task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 } - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} \ No newline at end of file + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 6a0431ec60..0ecec47b0c 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -1,8 +1,7 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. parameters: disableComponentGovernance: false componentGovernanceIgnoreDirectories: '' + steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 723f8ae0a9..07426fde05 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -1,7 +1,3 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. -# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish GuardianConfiguration' in the templateContext section. -# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish SARIF files to CodeAnalysisLogs container' in the templateContext section. parameters: overrideGuardianVersion: '' executeAllSdlToolsScript: '' @@ -10,12 +6,15 @@ parameters: publishGuardianDirectoryToPipeline: false sdlContinueOnError: false condition: '' + steps: - task: NuGetAuthenticate@1 inputs: nuGetServiceConnections: GuardianConnect + - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' + - ${{ if ne(parameters.overrideGuardianVersion, '') }}: - pwsh: | Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl @@ -23,6 +22,7 @@ steps: $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian (Overridden) + - ${{ if eq(parameters.overrideGuardianVersion, '') }}: - pwsh: | Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl @@ -30,17 +30,29 @@ steps: $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" displayName: Install Guardian + - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} displayName: Execute SDL (Overridden) continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) - task: DeleteFiles@1 displayName: Delete Guardian dependencies to avoid uploading inputs: @@ -49,11 +61,28 @@ steps: c i condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension - task: CopyFiles@2 displayName: Copy SARIF files inputs: flattenFolders: true - sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ contents: '**/*.sarif' targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index 180a76d6e8..a06373f38f 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -1,7 +1,9 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. -# The SBOM tasks have been removed because they are not required for the unofficial template. -# You can manually enable SBOM in the unofficial template if needed, othewise its automatically enabled when using official template. https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sbom +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + parameters: PackageVersion: 7.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' @@ -9,16 +11,38 @@ parameters: ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom IgnoreDirectories: '' sbomContinueOnError: true + steps: -- task: PowerShell@2 +- task: PowerShell@2 displayName: Prep for SBOM generation in (Non-linux) condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: + inputs: filePath: ./eng/common/generate-sbom-prep.ps1 arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 - script: | chmod +x ./eng/common/generate-sbom-prep.sh ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} displayName: Prep for SBOM generation in (Linux) condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} \ No newline at end of file + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 437705b17a..80861297dd 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,11 +1,10 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. -# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Logs' in the templateContext section. parameters: StageLabel: '' JobLabel: '' CustomSensitiveDataList: '' + # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed BinlogToolVersion: '1.0.11' + steps: - task: Powershell@2 displayName: Prepare Binlogs to Upload @@ -16,10 +15,35 @@ steps: Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true condition: always() + - task: PowerShell@2 displayName: Redact Logs inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' -BinlogToolVersion ${{parameters.BinlogToolVersion}} -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' '$(publishing-dnceng-devdiv-code-r-build-re)' '$(MaestroAccessToken)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' '$(akams-client-secret)' '$(microsoft-symbol-server-pat)' '$(symweb-symbol-server-pat)' '$(dn-bot-all-orgs-build-rw-code-rw)' ${{parameters.CustomSensitiveDataList}} + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} continueOnError: true - condition: always() \ No newline at end of file + condition: always() + +- task: PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 0bb85b7e06..32738aa938 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -1,12 +1,23 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. -# The Task 'PublishPipelineArtifact@1' has been converted to an output named 'Publish BuildLogs' in the templateContext section. parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. platform: {} + steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) - script: | set -x df -h + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) # In that case, call the feed setup script to add internal feeds corresponding to public ones. # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. @@ -18,6 +29,7 @@ steps: chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. # This only works if there is a username/email configured, which won't be the case in most CI runs. git config --get user.email @@ -26,6 +38,7 @@ steps: git config user.name dn-bot fi fi + # If building on the internal project, the internal storage variable may be available (usually only if needed) # In that case, add variables to allow the download of internal runtimes if the specified versions are not found # in the default public locations. @@ -33,35 +46,43 @@ steps: if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' fi + buildConfig=Release # Check if AzDO substitutes in a build config from a variable, and use it if so. if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then buildConfig='$(_BuildConfig)' fi + officialBuildArgs= if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi + targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi + runtimeOsArgs= if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' fi + baseOsArgs= if [ '${{ parameters.platform.baseOS }}' != '' ]; then baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' fi + publishArgs= if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then publishArgs='--publish' fi + assetManifestFileName=SourceBuild_RidSpecific.xml if [ '${{ parameters.platform.name }}' != '' ]; then assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml fi + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ @@ -77,6 +98,8 @@ steps: /p:DotNetBuildRepo=true \ /p:AssetManifestFileName=$assetManifestFileName displayName: Build + +# Upload build logs for diagnosis. - task: CopyFiles@2 displayName: Prepare BuildLogs staging directory inputs: @@ -89,7 +112,20 @@ steps: CleanTargetFolder: true continueOnError: true condition: succeededOrFailed() + +- task: PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets - task: ComponentGovernanceComponentDetection@0 displayName: Component Detection (Exclude upstream cache) inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' \ No newline at end of file + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 986a36e3c1..d236f9fdbb 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,7 +1,57 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: +# +# pool: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals windows.vs2019.amd64 + variables: -- name: DncEngPublicBuildPool - value: $[ replace( replace( eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public' ) ] -- name: DncEngInternalBuildPool - value: $[ replace( replace( eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal' ), False, 'NetCore1ESPool-Internal' ) ] \ No newline at end of file + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + - name: DncEngPublicBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore-Svc-Public' + ), + False, + 'NetCore-Public' + ) + ] + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] diff --git a/eng/common/templates/variables/sdl-variables.yml b/eng/common/templates/variables/sdl-variables.yml index 86af5b73b2..dbdd66d4a4 100644 --- a/eng/common/templates/variables/sdl-variables.yml +++ b/eng/common/templates/variables/sdl-variables.yml @@ -1,6 +1,6 @@ -# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool. -# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected. variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. - name: DefaultGuardianVersion value: 0.109.0 - name: GuardianPackagesConfigFile diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a8f8edb329..9bf873e3c2 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,6 +65,11 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } +# True if the build is a product build +[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } + +[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() } + function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -379,7 +384,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.5 + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.8.5 $defaultXCopyMSBuildVersion = '17.8.5' if (!$vsRequirements) { @@ -445,7 +450,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." + throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." } } if ($vsInstallDir -eq $null) { @@ -482,7 +487,7 @@ function InstallXCopyMSBuild([string]$packageVersion) { } function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'RoslynTools.MSBuild' + $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" @@ -850,7 +855,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) { + # Skip this when the build is a child of the VMR orchestrator build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and $properties -notlike "*DotNetBuildRepo=true*") { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index ece4b73079..db64e298ff 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,6 +68,9 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} +# True if the build is a product build +product_build=${product_build:-false} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -141,7 +144,7 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="$repo_root/.dotnet" + dotnet_root="${repo_root}.dotnet" export DOTNET_INSTALL_DIR="$dotnet_root" @@ -503,7 +506,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if [[ "$ci" == "true" && -n ${SYSTEM_TEAMPROJECT:-} ]]; then + # Skip this when the build is a child of the VMR orchestrator build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error From f678121b440cf6c96009e19de66e0fd0e88e6500 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Mon, 4 Mar 2024 10:29:51 -0500 Subject: [PATCH 16/26] oops --- build/vsts-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index df61abc5f3..5cf511914f 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -19,7 +19,7 @@ variables: value: true - name: Codeql.SkipTaskAutoInjection value: True #default to not inject CodeQL tasks, we'll enable it in a single job. -- template: /eng/common/templates/variables/pool-providers.yml@self +- template: /eng/common/templates-official/variables/pool-providers.yml@self resources: repositories: @@ -225,7 +225,7 @@ extends: displayName: Dotnet Server Shutdown ################################################################################ - - template: /eng/common/templates/job/job.yml@self + - template: /eng/common/templates-official/job/job.yml@self parameters: name: Package ################################################################################ @@ -280,7 +280,7 @@ extends: - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown - - template: /eng/common/templates/job/publish-build-assets.yml@self + - template: /eng/common/templates-official/job/publish-build-assets.yml@self parameters: dependsOn: Package enablePublishBuildArtifacts: true @@ -291,7 +291,7 @@ extends: image: 1es-windows-2019-pt os: windows - - template: /eng/common/templates/post-build/post-build.yml@self + - template: /eng/common/templates-official/post-build/post-build.yml@self parameters: publishingInfraVersion: 3 # This is to enable SDL runs part of Post-Build Validation Stage From fd9fb10e3602e3fee5830ec5d4ac25fa70c61969 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Mon, 4 Mar 2024 13:09:38 -0500 Subject: [PATCH 17/26] See if it's publish: to blame --- build/vsts-ci.yml | 110 +++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 5cf511914f..5416addf87 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -224,61 +224,61 @@ extends: - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown - ################################################################################ - - template: /eng/common/templates-official/job/job.yml@self - parameters: - name: Package - ################################################################################ - artifacts: - publish: - artifacts: false - logs: true - manifests: true - enableMicrobuild: true - enablePublishUsingPipelines: true - enableSourceIndex: true - dependsOn: - - Linux_x64 - - Linux_arm - - Linux_arm64 - - MacOS - - Windows_x86 - - Windows_x64 - - MacOS_Apple_Silicon - variables: - - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. - - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages - - _SignType: real - - _TeamName: DotNetCore - - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) - /p:TeamName=$(_TeamName) - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:OfficialBuildId=$(OfficialBuildId) - /p:DotNetPublishUsingPipelines=$true - pool: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2019-pt - os: windows - steps: - - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: pkgassets - downloadPath: $(Build.SourcesDirectory)/artifacts - - # Depending on the value of DotNetFinalVersionKind, the name of the package will change. - # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" - # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable - # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version - - script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) - displayName: Build Packages - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown +# ################################################################################ +# - template: /eng/common/templates-official/job/job.yml@self +# parameters: +# name: Package +# ################################################################################ +# artifacts: +# publish: +# artifacts: false +# logs: true +# manifests: true +# enableMicrobuild: true +# enablePublishUsingPipelines: true +# enableSourceIndex: true +# dependsOn: +# - Linux_x64 +# - Linux_arm +# - Linux_arm64 +# - MacOS +# - Windows_x86 +# - Windows_x64 +# - MacOS_Apple_Silicon +# variables: +# - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. +# - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages +# - _SignType: real +# - _TeamName: DotNetCore +# - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) +# /p:TeamName=$(_TeamName) +# /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) +# /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) +# /p:OfficialBuildId=$(OfficialBuildId) +# /p:DotNetPublishUsingPipelines=$true +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2019-pt +# os: windows +# steps: +# +# # Download all agent packages from all previous phases +# - task: DownloadBuildArtifacts@0 +# displayName: Download package assets +# inputs: +# artifactName: pkgassets +# downloadPath: $(Build.SourcesDirectory)/artifacts +# +# # Depending on the value of DotNetFinalVersionKind, the name of the package will change. +# # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" +# # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable +# # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version +# - script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) +# displayName: Build Packages +# +# # Terminate all dotnet build processes. +# - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown +# displayName: Dotnet Server Shutdown - template: /eng/common/templates-official/job/publish-build-assets.yml@self parameters: From f03cbf35986bf39c28af439184071c5293af831c Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Mon, 4 Mar 2024 13:10:15 -0500 Subject: [PATCH 18/26] Revert "See if it's publish: to blame" This reverts commit fd9fb10e3602e3fee5830ec5d4ac25fa70c61969. --- build/vsts-ci.yml | 110 +++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 5416addf87..5cf511914f 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -224,61 +224,61 @@ extends: - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown -# ################################################################################ -# - template: /eng/common/templates-official/job/job.yml@self -# parameters: -# name: Package -# ################################################################################ -# artifacts: -# publish: -# artifacts: false -# logs: true -# manifests: true -# enableMicrobuild: true -# enablePublishUsingPipelines: true -# enableSourceIndex: true -# dependsOn: -# - Linux_x64 -# - Linux_arm -# - Linux_arm64 -# - MacOS -# - Windows_x86 -# - Windows_x64 -# - MacOS_Apple_Silicon -# variables: -# - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. -# - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages -# - _SignType: real -# - _TeamName: DotNetCore -# - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) -# /p:TeamName=$(_TeamName) -# /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) -# /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) -# /p:OfficialBuildId=$(OfficialBuildId) -# /p:DotNetPublishUsingPipelines=$true -# pool: -# name: $(DncEngInternalBuildPool) -# image: 1es-windows-2019-pt -# os: windows -# steps: -# -# # Download all agent packages from all previous phases -# - task: DownloadBuildArtifacts@0 -# displayName: Download package assets -# inputs: -# artifactName: pkgassets -# downloadPath: $(Build.SourcesDirectory)/artifacts -# -# # Depending on the value of DotNetFinalVersionKind, the name of the package will change. -# # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" -# # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable -# # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version -# - script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) -# displayName: Build Packages -# -# # Terminate all dotnet build processes. -# - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown -# displayName: Dotnet Server Shutdown + ################################################################################ + - template: /eng/common/templates-official/job/job.yml@self + parameters: + name: Package + ################################################################################ + artifacts: + publish: + artifacts: false + logs: true + manifests: true + enableMicrobuild: true + enablePublishUsingPipelines: true + enableSourceIndex: true + dependsOn: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - MacOS + - Windows_x86 + - Windows_x64 + - MacOS_Apple_Silicon + variables: + - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. + - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages + - _SignType: real + - _TeamName: DotNetCore + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(OfficialBuildId) + /p:DotNetPublishUsingPipelines=$true + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2019-pt + os: windows + steps: + + # Download all agent packages from all previous phases + - task: DownloadBuildArtifacts@0 + displayName: Download package assets + inputs: + artifactName: pkgassets + downloadPath: $(Build.SourcesDirectory)/artifacts + + # Depending on the value of DotNetFinalVersionKind, the name of the package will change. + # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" + # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable + # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version + - script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) + displayName: Build Packages + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown - template: /eng/common/templates-official/job/publish-build-assets.yml@self parameters: From d6a6e1151e6ab6fab3dd0805d9a381dee26d0b7f Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Mon, 4 Mar 2024 13:27:59 -0500 Subject: [PATCH 19/26] Avoid artifacts: stanza --- build/vsts-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 5cf511914f..7544d7d556 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -229,11 +229,11 @@ extends: parameters: name: Package ################################################################################ - artifacts: - publish: - artifacts: false - logs: true - manifests: true + #artifacts: + # publish: + # artifacts: false + # logs: true + # manifests: true enableMicrobuild: true enablePublishUsingPipelines: true enableSourceIndex: true From c4abc447d3d203eb2dd8fff96bbf97da4e25c8ab Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Mon, 4 Mar 2024 13:30:03 -0500 Subject: [PATCH 20/26] don't binskim source --- build/vsts-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 7544d7d556..ba4341d6d3 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -31,6 +31,10 @@ resources: extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: + sdl: + binskim: + enabled: true + scanOutputDirectoryOnly: true pool: name: $(DncEngInternalBuildPool) image: 1es-windows-2019-pt From e7e1e8d7a105f8f413f5f179d3103371be9a285e Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 5 Mar 2024 09:40:09 -0500 Subject: [PATCH 21/26] try autobaselining --- build/vsts-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index ba4341d6d3..57bdfcfcc7 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -31,6 +31,8 @@ resources: extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: + featureFlags: + autoBaseline: true sdl: binskim: enabled: true From 06a30fef05e32c3e57764a35eb61ef598b73649c Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 5 Mar 2024 13:13:03 -0500 Subject: [PATCH 22/26] Annotate disabled log upload --- build/vsts-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 57bdfcfcc7..0065876c67 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -235,6 +235,7 @@ extends: parameters: name: Package ################################################################################ + #disable log upload for now, https://github.com/dotnet/dnceng/issues/2133 #artifacts: # publish: # artifacts: false From f557199975d138b0e746665006afdad40939f163 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 5 Mar 2024 17:44:12 -0500 Subject: [PATCH 23/26] Remove AI tooling tag --- build/vsts-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 0065876c67..45d2b8a28d 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -48,8 +48,6 @@ extends: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm UbuntuCrossArm64Container: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm64 - customBuildTags: - - ES365AIMigrationTooling stages: - stage: build displayName: Build From ce3126b2d267439e69355699164a7fd47fc08797 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 5 Mar 2024 17:54:19 -0500 Subject: [PATCH 24/26] Move image names into variables & dedup --- build/vsts-ci.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 45d2b8a28d..802acba24d 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -19,6 +19,12 @@ variables: value: true - name: Codeql.SkipTaskAutoInjection value: True #default to not inject CodeQL tasks, we'll enable it in a single job. +- name: LinuxImage + value: 1es-ubuntu-2204-pt +- name: WindowsImage + value: 1es-windows-2019-pt +- name: MacImage + value: macOS-12 - template: /eng/common/templates-official/variables/pool-providers.yml@self resources: @@ -39,7 +45,7 @@ extends: scanOutputDirectoryOnly: true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2019-pt + image: $(WindowsImage) os: windows containers: CentosContainer: @@ -57,7 +63,7 @@ extends: ################################################################################ pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204-pt + image: $(LinuxImage) os: linux container: CentosContainer templateContext: @@ -77,7 +83,7 @@ extends: ROOTFS_DIR: '/crossrootfs/arm' pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204-pt + image: $(LinuxImage) os: linux container: UbuntuCrossArmContainer templateContext: @@ -97,7 +103,7 @@ extends: ROOTFS_DIR: '/crossrootfs/arm64' pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204-pt + image: $(LinuxImage) os: linux container: UbuntuCrossArm64Container templateContext: @@ -115,7 +121,7 @@ extends: ################################################################################ pool: name: Azure Pipelines - vmImage: macOS-12 + vmImage: $(MacImage) os: macOS templateContext: outputs: @@ -134,7 +140,7 @@ extends: ################################################################################ pool: name: Azure Pipelines - vmImage: macOS-12 + vmImage: $(MacImage) os: macOS templateContext: outputs: @@ -158,7 +164,7 @@ extends: ################################################################################ pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2019-pt + image: $(WindowsImage) os: windows templateContext: outputs: @@ -181,7 +187,7 @@ extends: _TeamName: DotNetCore pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2019-pt + image: $(WindowsImage) os: windows templateContext: outputs: @@ -205,7 +211,7 @@ extends: _TeamName: DotNetCore pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2019-pt + image: $(WindowsImage) os: windows templateContext: outputs: @@ -263,7 +269,7 @@ extends: /p:DotNetPublishUsingPipelines=$true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2019-pt + image: $(WindowsImage) os: windows steps: @@ -293,7 +299,7 @@ extends: publishUsingPipelines: true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2019-pt + image: $(WindowsImage) os: windows - template: /eng/common/templates-official/post-build/post-build.yml@self From f46372033c2d4ffb9a5085ddf9699b70c59ba9c6 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 6 Mar 2024 13:55:05 -0500 Subject: [PATCH 25/26] Useless bad commit to force CI re-run --- build/vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 802acba24d..3143752b5f 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -24,7 +24,7 @@ variables: - name: WindowsImage value: 1es-windows-2019-pt - name: MacImage - value: macOS-12 + value: macOS-1 - template: /eng/common/templates-official/variables/pool-providers.yml@self resources: From d9cfbbcb4c86dea0e0b78be0a3178cd2b32ebb5d Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 6 Mar 2024 13:55:26 -0500 Subject: [PATCH 26/26] Revert "Useless bad commit to force CI re-run" This reverts commit f46372033c2d4ffb9a5085ddf9699b70c59ba9c6. --- build/vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 3143752b5f..802acba24d 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -24,7 +24,7 @@ variables: - name: WindowsImage value: 1es-windows-2019-pt - name: MacImage - value: macOS-1 + value: macOS-12 - template: /eng/common/templates-official/variables/pool-providers.yml@self resources: