From d08e39468e9d13b8f79f069ef33f3674e86a7430 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 24 Oct 2022 14:02:07 -0700 Subject: [PATCH 01/25] Call arcade post-build template --- build/vsts-ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 73a0cc1dbd..5ddea810aa 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -362,6 +362,25 @@ jobs: msbuildVersion: 15.0 continueOnError: true + - template: eng\common\templates\post-build\post-build.yml + parameters: + publishingInfraVersion: 3 + # This is to enable SDL runs part of Post-Build Validation Stage + SDLValidationParameters: + enable: true + publishGdn: true + continueOnError: false + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "machinelearning" + -TsaCodebaseName "machinelearning" + -TsaPublish $True' + # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown From 3f038899ae68e57fc1c8cb9575253cf652fc54ad Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 24 Oct 2022 14:04:37 -0700 Subject: [PATCH 02/25] Fix post-build template location --- build/vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 5ddea810aa..27301e205f 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -362,7 +362,7 @@ jobs: msbuildVersion: 15.0 continueOnError: true - - template: eng\common\templates\post-build\post-build.yml + - template: /eng/common/templates/post-build/post-build.yml parameters: publishingInfraVersion: 3 # This is to enable SDL runs part of Post-Build Validation Stage From 8e658461885f43b678bbb790f153ed8122932ce0 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 24 Oct 2022 14:14:59 -0700 Subject: [PATCH 03/25] Add "Build" stage to pipeline --- build/vsts-ci.yml | 687 +++++++++++++++++++++++----------------------- 1 file changed, 345 insertions(+), 342 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 27301e205f..6c061888ec 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -22,345 +22,352 @@ variables: Codeql.Enabled: true Codeql.SkipTaskAutoInjection: True #default to not inject CodeQL tasks, we'll enable it in a single job. -jobs: -################################################################################ -- job: Linux_x64 -################################################################################ - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - container: CentosContainer - steps: - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist - # Only build native assets to avoid conflicts. - - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Linux package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - -################################################################################ -- job: Linux_arm -################################################################################ - variables: - ROOTFS_DIR: '/crossrootfs/arm' - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - container: UbuntuCrossArmContainer - steps: - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist - # Only build native assets to avoid conflicts. - - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Linux package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - -################################################################################ -- job: Linux_arm64 -################################################################################ - variables: - ROOTFS_DIR: '/crossrootfs/arm64' - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - container: UbuntuCrossArm64Container - steps: - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist - # Only build native assets to avoid conflicts. - - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish Linux package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - -################################################################################ -- job: MacOS -################################################################################ - pool: - vmImage: macOS-12 - steps: - - script: brew update && brew unlink python@3.8 && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula - displayName: Install build dependencies - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist - # Only build native assets to avoid conflicts. - - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish macOS package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml +stages: +- stage: build + displayName: Build + jobs: + ################################################################################ + - job: Linux_x64 + ################################################################################ + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + container: CentosContainer + steps: + - script: ./restore.sh + displayName: restore all projects + - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj + displayName: build redist + # Only build native assets to avoid conflicts. + - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Linux package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: PackageAssets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + ################################################################################ + - job: Linux_arm + ################################################################################ + variables: + ROOTFS_DIR: '/crossrootfs/arm' + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + container: UbuntuCrossArmContainer + steps: + - script: ./restore.sh + displayName: restore all projects + - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj + displayName: build redist + # Only build native assets to avoid conflicts. + - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Linux package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: PackageAssets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + ################################################################################ + - job: Linux_arm64 + ################################################################################ + variables: + ROOTFS_DIR: '/crossrootfs/arm64' + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + container: UbuntuCrossArm64Container + steps: + - script: ./restore.sh + displayName: restore all projects + - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj + displayName: build redist + # Only build native assets to avoid conflicts. + - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish Linux package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: PackageAssets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + ################################################################################ + - job: MacOS + ################################################################################ + pool: + vmImage: macOS-12 + steps: + - script: brew update && brew unlink python@3.8 && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula + displayName: Install build dependencies + - script: ./restore.sh + displayName: restore all projects + - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj + displayName: build redist + # Only build native assets to avoid conflicts. + - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish macOS package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: PackageAssets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + ################################################################################ + - job: MacOS_Apple_Silicon + ################################################################################ + pool: + vmImage: macOS-12 + steps: + # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/2322#issuecomment-749211076 + - script: | + rm -rf /usr/local/bin/2to3 + displayName: MacOS Homebrew bug Workaround + continueOnError: true + - script: brew update && brew unlink python@3.8 && brew install libomp && brew link libomp --force + displayName: Install build dependencies + - script: ./restore.sh + displayName: restore all projects + - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj + displayName: build redist + # Only build native assets to avoid conflicts. + - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - task: PublishBuildArtifacts@1 + displayName: Publish macOS package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: PackageAssets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + ################################################################################ + - job: Windows_x86 + ################################################################################ + variables: + _SignType: real + _UseEsrpSigning: true + _TeamName: DotNetCore + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: + + - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 + displayName: Install MicroBuild Signing Plugin + inputs: + signType: '$(_SignType)' + zipSources: false + esrpSigning: '$(_UseEsrpSigning)' + feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' + env: + TeamName: $(_TeamName) + continueOnError: false + condition: and(succeeded(), in(variables._SignType, 'real', 'test')) + + - script: ./restore.cmd + displayName: restore all projects + - script: ./build.cmd -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj + displayName: build redist + # Only build native assets to avoid conflicts. + - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true + displayName: Build Native Assets + + - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:SignBinaries=true + displayName: sign binaries + + - task: PublishBuildArtifacts@1 + displayName: Publish Windows_x86 package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: PackageAssets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown + + ################################################################################ + - job: Windows_x64 + ################################################################################ + variables: + Codeql.SkipTaskAutoInjection: False # run CodeQL in this job + _SignType: real + _UseEsrpSigning: true + _TeamName: DotNetCore + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: + + - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 + displayName: Install MicroBuild Signing Plugin + inputs: + signType: '$(_SignType)' + zipSources: false + esrpSigning: '$(_UseEsrpSigning)' + feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' + env: + TeamName: $(_TeamName) + continueOnError: false + condition: and(succeeded(), in(variables._SignType, 'real', 'test')) + + # Build both native and managed assets. + - script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true + displayName: Build + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + + - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:SignBinaries=true + displayName: sign binaries + + - task: PublishBuildArtifacts@1 + displayName: Publish Windows_x64 package assets + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: PackageAssets + artifactType: container + + - template: /eng/common/templates/steps/generate-sbom.yml + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown ################################################################################ -- job: MacOS_Apple_Silicon -################################################################################ - pool: - vmImage: macOS-12 - steps: - # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/2322#issuecomment-749211076 - - script: | - rm -rf /usr/local/bin/2to3 - displayName: MacOS Homebrew bug Workaround - continueOnError: true - - script: brew update && brew unlink python@3.8 && brew install libomp && brew link libomp --force - displayName: Install build dependencies - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist - # Only build native assets to avoid conflicts. - - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - task: PublishBuildArtifacts@1 - displayName: Publish macOS package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - -################################################################################ -- job: Windows_x86 -################################################################################ - variables: - _SignType: real - _UseEsrpSigning: true - _TeamName: DotNetCore - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 - steps: - - - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 - displayName: Install MicroBuild Signing Plugin - inputs: - signType: '$(_SignType)' - zipSources: false - esrpSigning: '$(_UseEsrpSigning)' - feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables._SignType, 'real', 'test')) - - - script: ./restore.cmd - displayName: restore all projects - - script: ./build.cmd -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist - # Only build native assets to avoid conflicts. - - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true - displayName: Build Native Assets - - - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:SignBinaries=true - displayName: sign binaries - - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_x86 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown - -################################################################################ -- job: Windows_x64 -################################################################################ - variables: - Codeql.SkipTaskAutoInjection: False # run CodeQL in this job - _SignType: real - _UseEsrpSigning: true - _TeamName: DotNetCore - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 - steps: - - - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 - displayName: Install MicroBuild Signing Plugin - inputs: - signType: '$(_SignType)' - zipSources: false - esrpSigning: '$(_UseEsrpSigning)' - feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables._SignType, 'real', 'test')) - - # Build both native and managed assets. - - script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true - displayName: Build - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:SignBinaries=true - displayName: sign binaries - - - task: PublishBuildArtifacts@1 - displayName: Publish Windows_x64 package assets - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets - artifactType: container - - - template: /eng/common/templates/steps/generate-sbom.yml - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown - -################################################################################ -- job: Package -################################################################################ - dependsOn: - - Linux_x64 - - Linux_arm - - Linux_arm64 - - MacOS - - Windows_x86 - - Windows_x64 - - MacOS_Apple_Silicon - variables: - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages - _SignType: real - _UseEsrpSigning: true - _TeamName: DotNetCore - _AzureDevopsFeedUrl: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json - _SymwebSymbolServerPath: https://microsoft.artifacts.visualstudio.com/DefaultCollection - _MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection - - # Only enable publishing in non-public, non PR scenarios. - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat symweb-symbol-server-pat - group: DotNet-Symbol-Server-Pats - - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 - steps: - - # Install MicroBuild plugin - - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 - displayName: Install MicroBuild Signing Plugin - inputs: - signType: '$(_SignType)' - zipSources: false - esrpSigning: '$(_UseEsrpSigning)' - feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables._SignType, 'real', 'test')) - - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: PackageAssets - downloadPath: $(Build.SourcesDirectory)/artifacts/pkgassets - - # Workaround https://github.com/Microsoft/vsts-tasks/issues/6739 - - task: CopyFiles@2 - displayName: Copy package assets to correct folder - inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/pkgassets/PackageAssets - targetFolder: $(Build.SourcesDirectory)/artifacts/pkgassets - - # Depending on the value of DotNetFinalVersionKind, the name of the package will change. - # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" - # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable - # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version - - script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:OfficialBuildId=$(OfficialBuildId) /p:DotNetFinalVersionKind=$(DotnetVersionKind) - displayName: Build Packages - - - script: ./sign.cmd /p:SignNugetPackages=true - displayName: sign packages - continueOnError: false - - - template: /eng/common/templates/steps/generate-sbom.yml - - # The generated .nupkgs and .snupkgs packages are published to Azure artifacts, - # in case we need to debug them. They're found under Release/Shipping - - task: PublishBuildArtifacts@1 - displayName: Push packages and symbol packages to Azure Artifacts - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/packages - artifactName: PackageAssets - artifactType: container - continueOnError: true - - - task: NuGetCommand@2 - displayName: Push packages to AzureDevOps feed - inputs: - command: push - nuGetFeedType: 'internal' - publishVstsFeed: 'public/MachineLearning' - packagesToPush: $(Build.SourcesDirectory)/artifacts/**/*.nupkg;!$(Build.SourcesDirectory)/artifacts/**/*.snupkg - - - task: MSBuild@1 - displayName: Publish Symbols to SymWeb Symbol Server - inputs: - solution: build/publish.proj - msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(symweb-symbol-server-pat) - msbuildVersion: 15.0 - continueOnError: true - - - task: MSBuild@1 - displayName: Publish Symbols to Msdl Symbol Server - inputs: - solution: build/publish.proj - msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(microsoft-symbol-server-pat) - msbuildVersion: 15.0 - continueOnError: true + - job: Package + ################################################################################ + dependsOn: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - MacOS + - Windows_x86 + - Windows_x64 + - MacOS_Apple_Silicon + variables: + DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. + NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages + _SignType: real + _UseEsrpSigning: true + _TeamName: DotNetCore + _AzureDevopsFeedUrl: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json + _SymwebSymbolServerPath: https://microsoft.artifacts.visualstudio.com/DefaultCollection + _MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection + + # Only enable publishing in non-public, non PR scenarios. + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat symweb-symbol-server-pat + group: DotNet-Symbol-Server-Pats + + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: + + # Install MicroBuild plugin + - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 + displayName: Install MicroBuild Signing Plugin + inputs: + signType: '$(_SignType)' + zipSources: false + esrpSigning: '$(_UseEsrpSigning)' + feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' + env: + TeamName: $(_TeamName) + continueOnError: false + condition: and(succeeded(), in(variables._SignType, 'real', 'test')) + + # Download all agent packages from all previous phases + - task: DownloadBuildArtifacts@0 + displayName: Download package assets + inputs: + artifactName: PackageAssets + downloadPath: $(Build.SourcesDirectory)/artifacts/pkgassets + + # Workaround https://github.com/Microsoft/vsts-tasks/issues/6739 + - task: CopyFiles@2 + displayName: Copy package assets to correct folder + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/pkgassets/PackageAssets + targetFolder: $(Build.SourcesDirectory)/artifacts/pkgassets + + # Depending on the value of DotNetFinalVersionKind, the name of the package will change. + # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" + # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable + # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version + - script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:OfficialBuildId=$(OfficialBuildId) /p:DotNetFinalVersionKind=$(DotnetVersionKind) + displayName: Build Packages + + - script: ./sign.cmd /p:SignNugetPackages=true + displayName: sign packages + continueOnError: false + + - template: /eng/common/templates/steps/generate-sbom.yml + + # The generated .nupkgs and .snupkgs packages are published to Azure artifacts, + # in case we need to debug them. They're found under Release/Shipping + - task: PublishBuildArtifacts@1 + displayName: Push packages and symbol packages to Azure Artifacts + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/packages + artifactName: PackageAssets + artifactType: container + continueOnError: true + + - task: NuGetCommand@2 + displayName: Push packages to AzureDevOps feed + inputs: + command: push + nuGetFeedType: 'internal' + publishVstsFeed: 'public/MachineLearning' + packagesToPush: $(Build.SourcesDirectory)/artifacts/**/*.nupkg;!$(Build.SourcesDirectory)/artifacts/**/*.snupkg + + - task: MSBuild@1 + displayName: Publish Symbols to SymWeb Symbol Server + inputs: + solution: build/publish.proj + msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(symweb-symbol-server-pat) + msbuildVersion: 15.0 + continueOnError: true + + - task: MSBuild@1 + displayName: Publish Symbols to Msdl Symbol Server + inputs: + solution: build/publish.proj + msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(microsoft-symbol-server-pat) + msbuildVersion: 15.0 + continueOnError: true + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown - template: /eng/common/templates/post-build/post-build.yml parameters: @@ -379,8 +386,4 @@ jobs: -TsaIterationPath $(_TsaIterationPath) -TsaRepositoryName "machinelearning" -TsaCodebaseName "machinelearning" - -TsaPublish $True' - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown + -TsaPublish $True' \ No newline at end of file From 0731af0c0c9d598b4fbb213ab39de9e6b4a80850 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 24 Oct 2022 14:17:31 -0700 Subject: [PATCH 04/25] Decrease indent of post-build import --- build/vsts-ci.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 6c061888ec..b1589f40a1 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -369,21 +369,21 @@ stages: - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown - - template: /eng/common/templates/post-build/post-build.yml - parameters: - publishingInfraVersion: 3 - # This is to enable SDL runs part of Post-Build Validation Stage - SDLValidationParameters: - enable: true - publishGdn: true - continueOnError: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "machinelearning" - -TsaCodebaseName "machinelearning" - -TsaPublish $True' \ No newline at end of file +template: /eng/common/templates/post-build/post-build.yml + parameters: + publishingInfraVersion: 3 + # This is to enable SDL runs part of Post-Build Validation Stage + SDLValidationParameters: + enable: true + publishGdn: true + continueOnError: false + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "machinelearning" + -TsaCodebaseName "machinelearning" + -TsaPublish $True' \ No newline at end of file From 467fbeb76e5d314821d821e4d7c83b62f804c55c Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 24 Oct 2022 14:21:11 -0700 Subject: [PATCH 05/25] Is this yml better? --- build/vsts-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index b1589f40a1..49871f2437 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -260,7 +260,7 @@ stages: - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown -################################################################################ + ################################################################################ - job: Package ################################################################################ dependsOn: @@ -369,7 +369,7 @@ stages: - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown -template: /eng/common/templates/post-build/post-build.yml +- template: /eng/common/templates/post-build/post-build.yml parameters: publishingInfraVersion: 3 # This is to enable SDL runs part of Post-Build Validation Stage From c458dd56d087082d6beb914ca3fb5ca32722f530 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 11:59:03 -0700 Subject: [PATCH 06/25] Use Arcade for publishing and signing --- build/vsts-ci.yml | 100 ++++--------------------------------------- eng/Publishing.props | 5 +++ eng/Signing.props | 25 +++-------- sign.cmd | 3 -- 4 files changed, 20 insertions(+), 113 deletions(-) create mode 100644 eng/Publishing.props delete mode 100644 sign.cmd diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 49871f2437..1c2ee94fc0 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -164,26 +164,12 @@ stages: - job: Windows_x86 ################################################################################ variables: - _SignType: real - _UseEsrpSigning: true _TeamName: DotNetCore pool: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 - displayName: Install MicroBuild Signing Plugin - inputs: - signType: '$(_SignType)' - zipSources: false - esrpSigning: '$(_UseEsrpSigning)' - feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables._SignType, 'real', 'test')) - - script: ./restore.cmd displayName: restore all projects - script: ./build.cmd -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj @@ -192,9 +178,6 @@ stages: - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true displayName: Build Native Assets - - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:SignBinaries=true - displayName: sign binaries - - task: PublishBuildArtifacts@1 displayName: Publish Windows_x86 package assets inputs: @@ -213,26 +196,12 @@ stages: ################################################################################ variables: Codeql.SkipTaskAutoInjection: False # run CodeQL in this job - _SignType: real - _UseEsrpSigning: true _TeamName: DotNetCore pool: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 - displayName: Install MicroBuild Signing Plugin - inputs: - signType: '$(_SignType)' - zipSources: false - esrpSigning: '$(_UseEsrpSigning)' - feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables._SignType, 'real', 'test')) - # Build both native and managed assets. - script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true displayName: Build @@ -243,10 +212,6 @@ stages: verbosity: 'Verbose' alertWarningLevel: 'High' - - - script: ./sign.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:SignBinaries=true - displayName: sign binaries - - task: PublishBuildArtifacts@1 displayName: Publish Windows_x64 package assets inputs: @@ -275,12 +240,14 @@ stages: DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages _SignType: real - _UseEsrpSigning: true _TeamName: DotNetCore - _AzureDevopsFeedUrl: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json - _SymwebSymbolServerPath: https://microsoft.artifacts.visualstudio.com/DefaultCollection - _MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection - + _InternalBuildArgs: /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(OfficialBuildId) + /p:DotNetPublishUsingPipelines=$true + # Only enable publishing in non-public, non PR scenarios. ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat symweb-symbol-server-pat @@ -291,19 +258,6 @@ stages: demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - # Install MicroBuild plugin - - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4 - displayName: Install MicroBuild Signing Plugin - inputs: - signType: '$(_SignType)' - zipSources: false - esrpSigning: '$(_UseEsrpSigning)' - feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables._SignType, 'real', 'test')) - # Download all agent packages from all previous phases - task: DownloadBuildArtifacts@0 displayName: Download package assets @@ -322,49 +276,11 @@ stages: # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version - - script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:OfficialBuildId=$(OfficialBuildId) /p:DotNetFinalVersionKind=$(DotnetVersionKind) + - script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) displayName: Build Packages - - script: ./sign.cmd /p:SignNugetPackages=true - displayName: sign packages - continueOnError: false - - template: /eng/common/templates/steps/generate-sbom.yml - # The generated .nupkgs and .snupkgs packages are published to Azure artifacts, - # in case we need to debug them. They're found under Release/Shipping - - task: PublishBuildArtifacts@1 - displayName: Push packages and symbol packages to Azure Artifacts - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/packages - artifactName: PackageAssets - artifactType: container - continueOnError: true - - - task: NuGetCommand@2 - displayName: Push packages to AzureDevOps feed - inputs: - command: push - nuGetFeedType: 'internal' - publishVstsFeed: 'public/MachineLearning' - packagesToPush: $(Build.SourcesDirectory)/artifacts/**/*.nupkg;!$(Build.SourcesDirectory)/artifacts/**/*.snupkg - - - task: MSBuild@1 - displayName: Publish Symbols to SymWeb Symbol Server - inputs: - solution: build/publish.proj - msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(symweb-symbol-server-pat) - msbuildVersion: 15.0 - continueOnError: true - - - task: MSBuild@1 - displayName: Publish Symbols to Msdl Symbol Server - inputs: - solution: build/publish.proj - msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(microsoft-symbol-server-pat) - msbuildVersion: 15.0 - continueOnError: true - # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown diff --git a/eng/Publishing.props b/eng/Publishing.props new file mode 100644 index 0000000000..e727fcf4bb --- /dev/null +++ b/eng/Publishing.props @@ -0,0 +1,5 @@ + + + 3 + + \ No newline at end of file diff --git a/eng/Signing.props b/eng/Signing.props index 7d99d0d9aa..91b6ee12dd 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -1,21 +1,10 @@ - - - - - - - - - - - - + + + + - - - - - - + + true + \ No newline at end of file diff --git a/sign.cmd b/sign.cmd deleted file mode 100644 index 6e75c6d287..0000000000 --- a/sign.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -sign -warnAsError 0 %*" -exit /b %ErrorLevel% \ No newline at end of file From dc4cd334ede3f0212e5be3c09191ee9b0006ed96 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 14:02:22 -0700 Subject: [PATCH 07/25] Add explicit publish step --- build/vsts-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 1c2ee94fc0..d54f63260b 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -276,9 +276,15 @@ stages: # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version - - script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) + - script: ./build.cmd -configuration $(BuildConfig) -pack -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) displayName: Build Packages + - template: /eng/common/templates/job/publish-build-assets.yml + parameters: + enablePublishBuildArtifacts: true + publishAssetsImmediately: true + publishUsingPipelines: true + - template: /eng/common/templates/steps/generate-sbom.yml # Terminate all dotnet build processes. From 4a29b4e19747a02b76e5679b5869157666f98f3e Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 14:52:00 -0700 Subject: [PATCH 08/25] Fix publish-build-assets template import --- build/vsts-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index d54f63260b..9aed902ebb 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -279,18 +279,18 @@ stages: - script: ./build.cmd -configuration $(BuildConfig) -pack -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) displayName: Build Packages - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - enablePublishBuildArtifacts: true - publishAssetsImmediately: true - publishUsingPipelines: true - - template: /eng/common/templates/steps/generate-sbom.yml # Terminate all dotnet build processes. - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown displayName: Dotnet Server Shutdown + - template: /eng/common/templates/job/publish-build-assets.yml + parameters: + enablePublishBuildArtifacts: true + publishAssetsImmediately: true + publishUsingPipelines: true + - template: /eng/common/templates/post-build/post-build.yml parameters: publishingInfraVersion: 3 From 20513b2b84284f4fc4557224b7b7fc5d00e54980 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 14:52:23 -0700 Subject: [PATCH 09/25] Remove a bunch of unnecessary build steps --- build/vsts-ci.yml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 9aed902ebb..48ca8e98e3 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -34,10 +34,6 @@ stages: demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: CentosContainer steps: - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true displayName: Build Native Assets @@ -61,10 +57,6 @@ stages: demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: UbuntuCrossArmContainer steps: - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true displayName: Build Native Assets @@ -88,10 +80,6 @@ stages: demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: UbuntuCrossArm64Container steps: - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist # Only build native assets to avoid conflicts. - script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true displayName: Build Native Assets @@ -113,10 +101,6 @@ stages: steps: - script: brew update && brew unlink python@3.8 && brew unlink libomp && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula displayName: Install build dependencies - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist # Only build native assets to avoid conflicts. - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true displayName: Build Native Assets @@ -143,10 +127,6 @@ stages: continueOnError: true - script: brew update && brew unlink python@3.8 && brew install libomp && brew link libomp --force displayName: Install build dependencies - - script: ./restore.sh - displayName: restore all projects - - script: ./build.sh -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist # Only build native assets to avoid conflicts. - script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true displayName: Build Native Assets @@ -169,11 +149,6 @@ stages: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - - - script: ./restore.cmd - displayName: restore all projects - - script: ./build.cmd -configuration $(BuildConfig) /p:CopyPackageAssets=true /p:SkipRIDAgnosticAssets=true -projects $(Build.SourcesDirectory)/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj - displayName: build redist # Only build native assets to avoid conflicts. - script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true displayName: Build Native Assets From d0dba0c484aa4a18ef93895f9492ceceb4529ac0 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 15:05:17 -0700 Subject: [PATCH 10/25] Rename intermediate build artifact PackageAssets to pkgassets This lets us remove an unnecessary copy step --- build/vsts-ci.yml | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 48ca8e98e3..038f549036 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -42,7 +42,7 @@ stages: displayName: Publish Linux package assets inputs: pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets + artifactName: pkgassets artifactType: container - template: /eng/common/templates/steps/generate-sbom.yml @@ -65,7 +65,7 @@ stages: displayName: Publish Linux package assets inputs: pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets + artifactName: pkgassets artifactType: container - template: /eng/common/templates/steps/generate-sbom.yml @@ -88,7 +88,7 @@ stages: displayName: Publish Linux package assets inputs: pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets + artifactName: pkgassets artifactType: container - template: /eng/common/templates/steps/generate-sbom.yml @@ -109,7 +109,7 @@ stages: displayName: Publish macOS package assets inputs: pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets + artifactName: pkgassets artifactType: container - template: /eng/common/templates/steps/generate-sbom.yml @@ -135,7 +135,7 @@ stages: displayName: Publish macOS package assets inputs: pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets + artifactName: pkgassets artifactType: container - template: /eng/common/templates/steps/generate-sbom.yml @@ -157,7 +157,7 @@ stages: displayName: Publish Windows_x86 package assets inputs: pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets + artifactName: pkgassets artifactType: container - template: /eng/common/templates/steps/generate-sbom.yml @@ -191,7 +191,7 @@ stages: displayName: Publish Windows_x64 package assets inputs: pathToPublish: $(Build.SourcesDirectory)/artifacts/pkgassets - artifactName: PackageAssets + artifactName: pkgassets artifactType: container - template: /eng/common/templates/steps/generate-sbom.yml @@ -237,15 +237,8 @@ stages: - task: DownloadBuildArtifacts@0 displayName: Download package assets inputs: - artifactName: PackageAssets - downloadPath: $(Build.SourcesDirectory)/artifacts/pkgassets - - # Workaround https://github.com/Microsoft/vsts-tasks/issues/6739 - - task: CopyFiles@2 - displayName: Copy package assets to correct folder - inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/pkgassets/PackageAssets - targetFolder: $(Build.SourcesDirectory)/artifacts/pkgassets + artifactName: pkgassets + downloadPath: $(Build.SourcesDirectory)/artifacts # Depending on the value of DotNetFinalVersionKind, the name of the package will change. # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" From 0bedc6577c0741fa87ec4484b9056208c449d7d9 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 15:09:14 -0700 Subject: [PATCH 11/25] Use job.yml template for Package job --- build/vsts-ci.yml | 94 ++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 038f549036..811a0d789c 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -201,57 +201,51 @@ stages: displayName: Dotnet Server Shutdown ################################################################################ - - job: Package + - template: /eng/common/templates/job/job.yml + parameters: + name: Package ################################################################################ - dependsOn: - - Linux_x64 - - Linux_arm - - Linux_arm64 - - MacOS - - Windows_x86 - - Windows_x64 - - MacOS_Apple_Silicon - variables: - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages - _SignType: real - _TeamName: DotNetCore - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) - /p:TeamName=$(_TeamName) - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:OfficialBuildId=$(OfficialBuildId) - /p:DotNetPublishUsingPipelines=$true - - # Only enable publishing in non-public, non PR scenarios. - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat symweb-symbol-server-pat - group: DotNet-Symbol-Server-Pats - - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 - steps: - - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: pkgassets - downloadPath: $(Build.SourcesDirectory)/artifacts - - # Depending on the value of DotNetFinalVersionKind, the name of the package will change. - # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" - # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable - # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version - - script: ./build.cmd -configuration $(BuildConfig) -pack -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) - displayName: Build Packages - - - template: /eng/common/templates/steps/generate-sbom.yml - - # Terminate all dotnet build processes. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown - displayName: Dotnet Server Shutdown + dependsOn: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - MacOS + - Windows_x86 + - Windows_x64 + - MacOS_Apple_Silicon + variables: + DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. + NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages + _SignType: real + _TeamName: DotNetCore + _InternalBuildArgs: /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(OfficialBuildId) + /p:DotNetPublishUsingPipelines=$true + pool: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: + + # Download all agent packages from all previous phases + - task: DownloadBuildArtifacts@0 + displayName: Download package assets + inputs: + artifactName: pkgassets + downloadPath: $(Build.SourcesDirectory)/artifacts + + # Depending on the value of DotNetFinalVersionKind, the name of the package will change. + # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" + # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable + # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version + - script: ./build.cmd -configuration $(BuildConfig) -pack -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) + displayName: Build Packages + + # Terminate all dotnet build processes. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown + displayName: Dotnet Server Shutdown - template: /eng/common/templates/job/publish-build-assets.yml parameters: From d86931b39ba1e2763f837c17bd7cde6eee62a601 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 15:30:21 -0700 Subject: [PATCH 12/25] Attempt to fix yml error by reformatting variables --- build/vsts-ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 811a0d789c..9407f92a7f 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -214,16 +214,16 @@ stages: - Windows_x64 - MacOS_Apple_Silicon variables: - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages - _SignType: real - _TeamName: DotNetCore - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) - /p:TeamName=$(_TeamName) - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:OfficialBuildId=$(OfficialBuildId) - /p:DotNetPublishUsingPipelines=$true + - DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string. + - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages + - _SignType: real + - _TeamName: DotNetCore + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(OfficialBuildId) + /p:DotNetPublishUsingPipelines=$true pool: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 From 31b7eb857a8cd682478fac6471c88f5657943031 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 17:10:36 -0700 Subject: [PATCH 13/25] Make Native.proj download MlNetMklDeps package --- global.json | 2 +- src/Native/Native.proj | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index a721e55d80..31c4897e23 100644 --- a/global.json +++ b/global.json @@ -13,7 +13,7 @@ "msbuild-sdks": { "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22514.3", "Microsoft.DotNet.Helix.Sdk": "7.0.0-beta.22514.3", - "Microsoft.Build.Traversal": "2.1.1", + "Microsoft.Build.Traversal": "3.2.0", "Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02", "Microsoft.SourceLink.Common": "1.1.0-beta-20206-02" } diff --git a/src/Native/Native.proj b/src/Native/Native.proj index 92578bb066..8f38769072 100644 --- a/src/Native/Native.proj +++ b/src/Native/Native.proj @@ -143,8 +143,10 @@ - + + + From ee9708d3338502520f3ad6d6213013c07d36894b Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 17:18:16 -0700 Subject: [PATCH 14/25] Fix publish build assets to depend on Package --- build/vsts-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 9407f92a7f..bc6626d11c 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -249,6 +249,7 @@ stages: - template: /eng/common/templates/job/publish-build-assets.yml parameters: + dependsOn: Package enablePublishBuildArtifacts: true publishAssetsImmediately: true publishUsingPipelines: true From eb18c8be51908e5e55c0de1888602ebe46d7509d Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 18:28:59 -0700 Subject: [PATCH 15/25] Enable publish during Package job --- build/vsts-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index bc6626d11c..b5a3c828ed 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -205,6 +205,7 @@ stages: parameters: name: Package ################################################################################ + enablePublishUsingPipelines: true dependsOn: - Linux_x64 - Linux_arm From 6fc1325cf5f9dfba788b3732b8ba007aa52e2786 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 25 Oct 2022 19:42:40 -0700 Subject: [PATCH 16/25] Set the pool for publish job --- build/vsts-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index b5a3c828ed..ac9b6363c4 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -254,6 +254,9 @@ stages: enablePublishBuildArtifacts: true publishAssetsImmediately: true publishUsingPipelines: true + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 - template: /eng/common/templates/post-build/post-build.yml parameters: From 18825e983a6b9fc5ddcda4e77fee2a279a4bac01 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 05:59:43 -0700 Subject: [PATCH 17/25] Enable PostBuildSign --- eng/Signing.props | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/Signing.props b/eng/Signing.props index 91b6ee12dd..b75c2d5454 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -6,5 +6,6 @@ true + true \ No newline at end of file From 66cb2ecc6dd2e9b47b510443b0091f81f7933f15 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 05:59:58 -0700 Subject: [PATCH 18/25] Fix author of Microsoft.ML.Mkl.Redist --- src/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.csproj b/src/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.csproj index 81e2f16c39..fb0e3f5809 100644 --- a/src/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.csproj +++ b/src/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.csproj @@ -1,7 +1,6 @@ - Intel netstandard2.0 false LICENSE.txt From e092c8c0e9d0fc427fc4983c844126a5d24adaf7 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 08:30:14 -0700 Subject: [PATCH 19/25] Make signing happen during Package job --- build/vsts-ci.yml | 2 +- eng/Signing.props | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index ac9b6363c4..1d61d1d043 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -241,7 +241,7 @@ stages: # For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release" # the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable # See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version - - script: ./build.cmd -configuration $(BuildConfig) -pack -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) + - script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs) displayName: Build Packages # Terminate all dotnet build processes. diff --git a/eng/Signing.props b/eng/Signing.props index b75c2d5454..91b6ee12dd 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -6,6 +6,5 @@ true - true \ No newline at end of file From ae05a620a36d6bbea59eccd522d4dbaa62b765db Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 11:35:05 -0700 Subject: [PATCH 20/25] Set SDK variable group --- build/vsts-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 1d61d1d043..edcefffb8e 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -259,6 +259,8 @@ stages: demands: ImageOverride -equals windows.vs2019.amd64 - template: /eng/common/templates/post-build/post-build.yml + variables: + - group: SDL_Settings parameters: publishingInfraVersion: 3 # This is to enable SDL runs part of Post-Build Validation Stage From 09837e77293c65b4b6c09446b889560373b8eed5 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 11:35:22 -0700 Subject: [PATCH 21/25] Publish logs and artifacts from Package job --- build/vsts-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index edcefffb8e..2cd1ff86fc 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -228,6 +228,10 @@ stages: pool: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 + artifacts: + publish: + artifacts: true + logs: true steps: # Download all agent packages from all previous phases From 396ebb60f389e75d3d8537e5ec2720e997a2f756 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 11:53:31 -0700 Subject: [PATCH 22/25] Enable more job template options --- build/vsts-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 2cd1ff86fc..dbda76a699 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -205,7 +205,14 @@ stages: parameters: name: Package ################################################################################ + artifacts: + publish: + artifacts: true + logs: true + manifests: true + enableMicrobuild: true enablePublishUsingPipelines: true + enableSourceIndex: true dependsOn: - Linux_x64 - Linux_arm @@ -228,10 +235,6 @@ stages: pool: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 - artifacts: - publish: - artifacts: true - logs: true steps: # Download all agent packages from all previous phases From a45a09d4b1a69d6daf18dedf35bb036eb9050a53 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 12:00:27 -0700 Subject: [PATCH 23/25] Attempt to fix import of SDL_Settings group --- build/vsts-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index dbda76a699..75482cfccd 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -21,7 +21,7 @@ variables: DOTNET_MULTILEVEL_LOOKUP: 0 Codeql.Enabled: true Codeql.SkipTaskAutoInjection: True #default to not inject CodeQL tasks, we'll enable it in a single job. - + - group: SDL_Settings stages: - stage: build displayName: Build @@ -266,8 +266,6 @@ stages: demands: ImageOverride -equals windows.vs2019.amd64 - template: /eng/common/templates/post-build/post-build.yml - variables: - - group: SDL_Settings parameters: publishingInfraVersion: 3 # This is to enable SDL runs part of Post-Build Validation Stage From 83c42ab79bc61f1f9a372e59328c09b3b97782b3 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 13:05:29 -0700 Subject: [PATCH 24/25] Attempt to fix syntax for variable group usage --- build/vsts-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 75482cfccd..e0439e8f33 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -12,8 +12,8 @@ resources: - container: UbuntuCrossArm64Container image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm64 - variables: +- group: SDL_Settings BuildConfig: Release OfficialBuildId: $(BUILD.BUILDNUMBER) DOTNET_CLI_TELEMETRY_OPTOUT: 1 @@ -21,7 +21,7 @@ variables: DOTNET_MULTILEVEL_LOOKUP: 0 Codeql.Enabled: true Codeql.SkipTaskAutoInjection: True #default to not inject CodeQL tasks, we'll enable it in a single job. - - group: SDL_Settings + stages: - stage: build displayName: Build From ec1f4203a5a030121660f8d0386ca6b04e776c5b Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Wed, 26 Oct 2022 14:16:37 -0700 Subject: [PATCH 25/25] Move pipeline variables into template We need to include top-level variable groups but can only do so if we use -name/value syntax. Hide this in a template so that it's less ugly. --- build/vsts-ci.yml | 9 +-------- eng/common-variables.yml | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 eng/common-variables.yml diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index e0439e8f33..f768a11bf7 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -13,14 +13,7 @@ resources: - container: UbuntuCrossArm64Container image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm64 variables: -- group: SDL_Settings - BuildConfig: Release - OfficialBuildId: $(BUILD.BUILDNUMBER) - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - DOTNET_MULTILEVEL_LOOKUP: 0 - Codeql.Enabled: true - Codeql.SkipTaskAutoInjection: True #default to not inject CodeQL tasks, we'll enable it in a single job. +- template: /eng/common-variables.yml stages: - stage: build diff --git a/eng/common-variables.yml b/eng/common-variables.yml new file mode 100644 index 0000000000..230de17298 --- /dev/null +++ b/eng/common-variables.yml @@ -0,0 +1,17 @@ +variables: +- group: SDL_Settings +- name: BuildConfig + value: Release +- name: OfficialBuildId + value: $(BUILD.BUILDNUMBER) +- name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 +- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 +- name: DOTNET_MULTILEVEL_LOOKUP + value: 0 +- name: Codeql.Enabled + value: true +- name: Codeql.SkipTaskAutoInjection + value: True #default to not inject CodeQL tasks, we'll enable it in a single job. +