From e02dfd43b2c38713424af35e91f68686d2e79f46 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Fri, 15 Nov 2019 15:29:12 -0800 Subject: [PATCH 01/14] Initial work to make corefx templates re-usable for live live --- eng/pipelines/libraries/build.yml | 78 +++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 eng/pipelines/libraries/build.yml diff --git a/eng/pipelines/libraries/build.yml b/eng/pipelines/libraries/build.yml new file mode 100644 index 00000000000000..3ee51dbc2f3aa6 --- /dev/null +++ b/eng/pipelines/libraries/build.yml @@ -0,0 +1,78 @@ +parameters: + buildConfig: '' + osGroup: '' + archType: '' + osSubgroup: '' + framework: netcoreapp + isOfficialBuild: false + timeoutInMinutes: 150 + preBuildSteps: [] + +jobs: + - template: /eng/common/templates/job/job.yml + parameters: + variables: + - _msbuildCommonParameters: '' + - _archiveTestsParameter: '' + - _finalFrameworkArg: -framework ${{ parameters.framework }} + + - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: + - _finalFrameworkArg: -allConfigurations + + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) + + - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) + - _commonArguments: $(_args) + + # Windows variables + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - _buildScript: $(buildScriptFileName).cmd + - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci + + # Non-Windows variables + - ${{ if ne(parameters.targetOS, 'Windows_NT') }}: + - _buildScript: ./$(buildScriptFileName).sh + - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - _commonArguments: $(_args) -stripSymbols + + enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed + helixRepo: dotnet/runtime + + $ {{ if eq(parameters.framework, 'netcoreapp') }}: + name: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.framework, 'netcoreapp') }}: + name: ${{ format('{0} {1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + + workspace: + clean: all + + enablePublishBuildArtifacts: true + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + steps: + - ${{ if ne(parameters.preBuildSteps[0], '') }}: + - ${{ parameters.preBuildSteps }} + + - script: $(_buildScript) -restore $(_commonArguments) + displayName: Restore + + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - task: DotNetCoreCLI@2 + displayName: Restore internal tools + condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) + inputs: + command: restore + feedsToUse: config + projects: 'eng/common/internal/Tools.csproj' + nugetConfigPath: 'eng/internal/NuGet.config' + restoreDirectory: '$(Build.SourcesDirectory)\.packages' + verbosityRestore: 'normal' + externalFeedCredentials: 'dotnet-core-internal-tooling' + + - script: $(_buildScript) + -build + $(_commonArguments) + $(_msbuildCommonParameters) + displayName: Build Sources \ No newline at end of file From 6733081f132c26a777b98b17595b014da692555a Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Fri, 15 Nov 2019 16:48:02 -0800 Subject: [PATCH 02/14] Support building and running tests --- eng/pipelines/libraries/build.yml | 68 +++++++++++++++++++++++++------ eng/pipelines/libraries/helix.yml | 4 -- src/libraries/build.proj | 2 +- 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/libraries/build.yml b/eng/pipelines/libraries/build.yml index 3ee51dbc2f3aa6..b790a245046a28 100644 --- a/eng/pipelines/libraries/build.yml +++ b/eng/pipelines/libraries/build.yml @@ -7,14 +7,24 @@ parameters: isOfficialBuild: false timeoutInMinutes: 150 preBuildSteps: [] + testScope: '' + runTests: true + publishTestArtifacts: false jobs: - template: /eng/common/templates/job/job.yml parameters: variables: - _msbuildCommonParameters: '' - - _archiveTestsParameter: '' - _finalFrameworkArg: -framework ${{ parameters.framework }} + - _skipTestRestoreArg: '' + - _testScopeArg: '' + - _actionParameters: -build + + - ${{ if ne(parameters.testScope, '') }}: + - _testScopeArg: -testscope ${{ parameters.testScope }} + - _skipTestRestoreArg: /p:SkipTestRestore=true + - _actionParameters: -buildtests - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: - _finalFrameworkArg: -allConfigurations @@ -22,7 +32,7 @@ jobs: - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) + - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testScopeArg) $(_skipTestRestoreArg) - _commonArguments: $(_args) # Windows variables @@ -36,18 +46,37 @@ jobs: - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _commonArguments: $(_args) -stripSymbols - + + - ${{ if or(eq(parameters.runTests, 'true'), eq(parameters.publishTestArtifacts, 'true')) }}: + - ${{ if ne(parameters.framework, 'allConfigurations') }}: + - _commonArguments: $(_args) /p:ArchiveTests=Tests + - ${{ if eq(parameters.framework, 'allConfigurations') }}: + - _commonArguments: $(_args) /p:ArchiveTests=Packages + enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed helixRepo: dotnet/runtime - $ {{ if eq(parameters.framework, 'netcoreapp') }}: - name: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.framework, 'netcoreapp') }}: - name: ${{ format('{0} {1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if eq(parameters.testScope, '') }}: + name: ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + $ {{ if eq(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0}{1} Build {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0} Build {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} + + ${{ if ne(parameters.testScope, '') }}: + name: ${{ format('libraries_test_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + $ {{ if eq(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0}{1} Test {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0} Test {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} + + ${{ if ne(parameters.testScope, '') }}: + dependencies: + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} workspace: clean: all - + enablePublishBuildArtifacts: true timeoutInMinutes: ${{ parameters.timeoutInMinutes }} @@ -57,7 +86,7 @@ jobs: - script: $(_buildScript) -restore $(_commonArguments) displayName: Restore - + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - task: DotNetCoreCLI@2 displayName: Restore internal tools @@ -70,9 +99,24 @@ jobs: restoreDirectory: '$(Build.SourcesDirectory)\.packages' verbosityRestore: 'normal' externalFeedCredentials: 'dotnet-core-internal-tooling' - + - script: $(_buildScript) - -build + $(_actionParameters) $(_commonArguments) $(_msbuildCommonParameters) - displayName: Build Sources \ No newline at end of file + displayName: Build + + - ${{ if and(eq(parameters.runTests, 'true'), ne(parameters.isOfficialBuild, 'false')) }}: + - template: /eng/pipelines/libraries/helix.yml + parameters: + targetOS: ${{ parameters.targetOS }} + archGroup: ${{ parameters.archType }} + configuration: ${{ parameters.buildConfig }} + helixQueues: $(_helixQueues) + msbuildScript: $(_msbuildCommand) + framework: ${{ parameters.framework }} + testScope: ${{ parameters.testScope }} + creator: dotnet-bot + helixToken: '' + + # TODO: publish test artifacts when publishTestArtifacts == true. diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index 8b20f44624de5b..4f5f2d28af3c45 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -4,10 +4,8 @@ parameters: creator: '' framework: '' helixQueues: '' - helixToken: '' msbuildScript: '' targetOS: '' - officialBuildId: '' testScope: 'innerloop' # innerloop | outerloop | all condition: always() @@ -22,9 +20,7 @@ steps: /p:TargetGroup=${{ parameters.framework }} /p:HelixTargetQueues=${{ parameters.helixQueues }} /p:HelixBuild=$(Build.BuildNumber) - /p:HelixAccessToken=${{ parameters.helixToken }} /p:Creator=${{ parameters.creator }} - /p:OfficialBuildId=${{ parameters.officialBuildId }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: Send to Helix condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/src/libraries/build.proj b/src/libraries/build.proj index 2d14f266047e6a..f132086061a01c 100644 --- a/src/libraries/build.proj +++ b/src/libraries/build.proj @@ -23,7 +23,7 @@ <_RestoreProjects Include="$(MSBuildThisFileDirectory)restore\dirs.proj" Build="true" /> - + <_RestoreProjects Include="$(MSBuildThisFileDirectory)tests.proj" Build="false" /> From 2e79ad420fdc76c9e0744274a323a84d7e4e1189 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Mon, 18 Nov 2019 14:19:01 -0800 Subject: [PATCH 03/14] First try to use platform-matrix --- eng/pipelines/libraries/.azure-ci.yml | 52 +++++++++++++-------------- eng/pipelines/libraries/build.yml | 40 +++++++++++++-------- eng/pipelines/libraries/variables.yml | 2 ++ 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/eng/pipelines/libraries/.azure-ci.yml b/eng/pipelines/libraries/.azure-ci.yml index af7dc48acb068b..1d7e4b033b5644 100644 --- a/eng/pipelines/libraries/.azure-ci.yml +++ b/eng/pipelines/libraries/.azure-ci.yml @@ -84,39 +84,35 @@ variables: value: .NETCore stages: - # Windows legs - - template: ${{ variables['pipelinesPath'] }}/windows.yml - parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - # Linux legs - - template: ${{ variables['pipelinesPath'] }}/linux.yml - parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - - # MacOS legs - - template: ${{ variables['pipelinesPath'] }}/macos.yml - parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - - # FreeBSD leg is only for official builds - # - template: ${{ variables['pipelinesPath'] }}/freebsd.yml - # parameters: - # isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - # fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} + - stage: Build + jobs: + - template: /eng/pipelines/coreclr/templates/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build.yml + buildConfig: Release + platforms: + - Windows_NT_x64 + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + + - template: /eng/pipelines/coreclr/templates/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build.yml + buildConfig: Release + platforms: + - Windows_NT_x64 + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + runTests: true + testScope: innerloop # Publish and validation steps. Only run in official builds - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}: - - template: ${{ variables['pipelinesPath'] }}/pre-publish.yml + - ${{ if eq(variables['isOfficialBuild'], 'true') }}: + - template: /eng/pipelines/libraries/pre-publish.yml parameters: dependsOn: - - WindowsStage - - LinuxStage - - MacOSStage - # - FreeBSD + - Build - template: eng\common\templates\post-build\post-build.yml parameters: diff --git a/eng/pipelines/libraries/build.yml b/eng/pipelines/libraries/build.yml index b790a245046a28..ec77f8bbcad6ce 100644 --- a/eng/pipelines/libraries/build.yml +++ b/eng/pipelines/libraries/build.yml @@ -8,8 +8,11 @@ parameters: timeoutInMinutes: 150 preBuildSteps: [] testScope: '' - runTests: true + runTests: false publishTestArtifacts: false + helixQueues: [] + container: '' + crossrootfsDir: '' jobs: - template: /eng/common/templates/job/job.yml @@ -17,13 +20,16 @@ jobs: variables: - _msbuildCommonParameters: '' - _finalFrameworkArg: -framework ${{ parameters.framework }} - - _skipTestRestoreArg: '' + - _skipTestRestoreArg: /p:SkipTestRestore=true - _testScopeArg: '' - _actionParameters: -build + - ${{ if ne(parameters.crossrootfsDir, '') }}: + - ROOTFS_DIR: ${{ parameters.crossrootfsDir}} + - ${{ if ne(parameters.testScope, '') }}: - _testScopeArg: -testscope ${{ parameters.testScope }} - - _skipTestRestoreArg: /p:SkipTestRestore=true + - _skipTestRestoreArg: '' - _actionParameters: -buildtests - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: @@ -56,6 +62,12 @@ jobs: enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed helixRepo: dotnet/runtime + ${{ if ne(parameters.container, '') }}: + ${{ if eq(parameters.container.registry, 'mcr') }}: + container: ${{ format('{0}:{1}', 'mcr.microsoft.com/dotnet-buildtools/prereqs', parameters.container.image) }} + ${{ if ne(parameters.container.registry, 'mcr') }}: + container: ${{ format('{0}:{1}', parameters.container.registry, parameters.container.image) }} + ${{ if eq(parameters.testScope, '') }}: name: ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} $ {{ if eq(parameters.framework, 'netcoreapp') }}: @@ -106,17 +118,17 @@ jobs: $(_msbuildCommonParameters) displayName: Build - - ${{ if and(eq(parameters.runTests, 'true'), ne(parameters.isOfficialBuild, 'false')) }}: + - ${{ if eq(parameters.runTests, 'true') }}: - template: /eng/pipelines/libraries/helix.yml - parameters: - targetOS: ${{ parameters.targetOS }} - archGroup: ${{ parameters.archType }} - configuration: ${{ parameters.buildConfig }} - helixQueues: $(_helixQueues) - msbuildScript: $(_msbuildCommand) - framework: ${{ parameters.framework }} - testScope: ${{ parameters.testScope }} - creator: dotnet-bot - helixToken: '' + parameters: + targetOS: ${{ parameters.targetOS }} + archGroup: ${{ parameters.archType }} + configuration: ${{ parameters.buildConfig }} + helixQueues: ${{ join(';', parameters.helixQueues) }} + msbuildScript: $(_msbuildCommand) + framework: ${{ parameters.framework }} + testScope: ${{ parameters.testScope }} + creator: dotnet-bot + helixToken: '' # TODO: publish test artifacts when publishTestArtifacts == true. diff --git a/eng/pipelines/libraries/variables.yml b/eng/pipelines/libraries/variables.yml index 2e0cf9f6da83f0..f0d229fc74f102 100644 --- a/eng/pipelines/libraries/variables.yml +++ b/eng/pipelines/libraries/variables.yml @@ -5,3 +5,5 @@ variables: value: $(Build.SourcesDirectory)/src/libraries - name: pipelinesPath value: /eng/pipelines/libraries + - name: isOfficialBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} From bdcd4573bbd7243921e29941a317a9b3b223ff04 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 19 Nov 2019 22:30:51 -0800 Subject: [PATCH 04/14] Add a common layer in libraries build and a job/test template --- eng/pipelines/common/platform-matrix.yml | 16 ++ eng/pipelines/libraries/.azure-ci.yml | 180 ++++++++++++------ eng/pipelines/libraries/base-job.yml | 100 ++++++++++ eng/pipelines/libraries/build-job.yml | 120 ++++++++++++ eng/pipelines/libraries/build-test-job.yml | 65 +++++++ eng/pipelines/libraries/build.yml | 134 ------------- .../libraries/helix-queues-setup.yml | 120 ++++++++++++ eng/pipelines/libraries/outerloop.yml | 103 ++++++---- eng/pipelines/libraries/run-test-job.yml | 63 ++++++ eng/pipelines/libraries/variables.yml | 8 + eng/referenceAssemblies.props | 6 +- src/libraries/build.proj | 2 +- src/libraries/tests.proj | 3 +- 13 files changed, 678 insertions(+), 242 deletions(-) create mode 100644 eng/pipelines/libraries/base-job.yml create mode 100644 eng/pipelines/libraries/build-job.yml create mode 100644 eng/pipelines/libraries/build-test-job.yml delete mode 100644 eng/pipelines/libraries/build.yml create mode 100644 eng/pipelines/libraries/helix-queues-setup.yml create mode 100644 eng/pipelines/libraries/run-test-job.yml diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 66f4ddf9820080..fa674a5998aa21 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -159,6 +159,22 @@ jobs: managedTestBuildOsGroup: OSX ${{ insert }}: ${{ parameters.jobParameters }} +- ${{ if containsValue(parameters.platforms, 'WebAssembly_wasm') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + osGroup: WebAssembly + archType: wasm + platform: WebAssembly_wasm + container: + image: ubuntu-16.04-a50a721-20191023143845 + registry: mcr + jobParameters: + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ insert }}: ${{ parameters.jobParameters }} + # FreeBSD # FreeBSD machines are currently offline. Re-enable in the official build when diff --git a/eng/pipelines/libraries/.azure-ci.yml b/eng/pipelines/libraries/.azure-ci.yml index 1d7e4b033b5644..33cda03d60c598 100644 --- a/eng/pipelines/libraries/.azure-ci.yml +++ b/eng/pipelines/libraries/.azure-ci.yml @@ -45,29 +45,6 @@ pr: - eng/pipelines/installer/* - eng/pipelines/common/* -resources: - containers: - - container: centos7_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df28-20191023143754 - - - container: centos6_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-f39df28-20191023143802 - - - container: alpine_39_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-WithNode-0fc54a3-20190918214015 - - - container: alpine_37_arm64_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-406629a-20191023143847 - - - container: ubuntu_1604_arm64_cross_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-cfdd435-20191023143847 - - - container: ubuntu_1604_arm_cross_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-14.04-23cacb0-20191023143847 - - - container: ubuntu_1604 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-a50a721-20191023143845 - variables: - template: variables.yml @@ -83,51 +60,130 @@ variables: - name: _DotNetValidationArtifactsCategory value: .NETCore -stages: +jobs: + - template: /eng/pipelines/common/checkout-job.yml - - stage: Build - jobs: - - template: /eng/pipelines/coreclr/templates/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build.yml + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Release + platforms: + - Linux_rhel6_x64 + - Windows_NT_x86 + - WebAssembly_wasm + - ${{ if eq(variables['isFullMatrix'], true) }}: + - OSX_x64 + - Windows_NT_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + framework: netcoreapp + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Release + platforms: + - Windows_NT_x86 + - ${{ if eq(variables['isFullMatrix'], true) }}: + - Windows_NT_x64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + framework: netfx + ${{ if eq(variables['isOfficialBuild'], false) }}: + runTests: true + testScope: innerloop + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + ${{ if eq(variables['isFullMatrix'], false) }}: + buildConfig: Debug + ${{ if eq(variables['isFullMatrix'], true) }}: + buildConfig: Release + platforms: + - Windows_NT_x64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + framework: allConfigurations + ${{ if eq(variables['isOfficialBuild'], false) }}: + runTests: true + + - ${{ if eq(variables['isFullMatrix'], false) }}: + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Debug + platforms: + - Windows_NT_x64 + - OSX_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + framework: netcoreapp + + - ${{ if eq(variables['isOfficialBuild'], false) }}: + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-test-job.yml + ${{ if eq(variables['isFullMatrix'], false) }}: + buildConfig: Debug + ${{ if eq(variables['isFullMatrix'], true) }}: buildConfig: Release - platforms: - - Windows_NT_x64 - jobParameters: - isOfficialBuild: ${{ variables['isOfficialBuild'] }} + platforms: + - Windows_NT_x64 + - OSX_x64 + - Linux_x64 + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + testScope: innerloop + framework: netcoreapp - - template: /eng/pipelines/coreclr/templates/platform-matrix.yml + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - Linux_rhel6_x64 + - Windows_NT_x86 + - ${{ if eq(variables['isFullMatrix'], true) }}: + - OSX_x64 + - Windows_NT_x64 + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + isFullMatrix: ${{ variables['isFullMatrix'] }} + testScope: innerloop + framework: netcoreapp + + - ${{ if eq(variables['isFullMatrix'], false) }}: + - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build.yml - buildConfig: Release + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Debug platforms: - Windows_NT_x64 + - OSX_x64 + - Linux_x64 + - Linux_musl_x64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: isOfficialBuild: ${{ variables['isOfficialBuild'] }} - runTests: true + isFullMatrix: ${{ variables['isFullMatrix'] }} testScope: innerloop - - # Publish and validation steps. Only run in official builds - - ${{ if eq(variables['isOfficialBuild'], 'true') }}: - - template: /eng/pipelines/libraries/pre-publish.yml - parameters: - dependsOn: - - Build - - - template: eng\common\templates\post-build\post-build.yml - parameters: - validateDependsOn: - - PrePublish - enableSymbolValidation: false # https://github.com/dotnet/arcade/issues/2871 - SDLValidationParameters: - enable: true - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL "$(TsaInstanceURL)" - -TsaProjectName "$(TsaProjectName)" - -TsaNotificationEmail "$(TsaNotificationEmail)" - -TsaCodebaseAdmin "$(TsaCodebaseAdmin)" - -TsaBugAreaPath "$(TsaBugAreaPath)" - -TsaIterationPath "$(TsaIterationPath)" - -TsaRepositoryName "CoreFX" - -TsaCodebaseName "CoreFX" - -TsaPublish $True' + framework: netcoreapp diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml new file mode 100644 index 00000000000000..b3cb574fa6adff --- /dev/null +++ b/eng/pipelines/libraries/base-job.yml @@ -0,0 +1,100 @@ +parameters: + buildConfig: '' + osGroup: '' + archType: '' + osSubgroup: '' + framework: '' + isOfficialBuild: false + timeoutInMinutes: 150 + container: '' + steps: [] + dependsOn: [] + variables: {} + name: '' + displayName: '' + pool: '' + stepName: '' + +jobs: + - template: /eng/common/templates/job/job.yml + parameters: + name: ${{ format('libraries_{0}_{1}_{2}{3}_{4}_{5}', parameters.stepName, parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if eq(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0} {1}{2} {3} {4}', parameters.stepName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0} {1} {2} {3} {4}', parameters.stepName, parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} + + enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed + container: ${{ parameters.container }} + helixRepo: dotnet/runtime + pool: ${{ parameters.pool }} + variables: + - _BuildConfig: ${{ parameters.buildConfig }} + - _msbuildCommonParameters: '' + - _stripSymbolsArg: '' + - _runtimeOSArg: '' + - _portableArg: '' + - _finalFrameworkArg: -framework ${{ parameters.framework }} + - _buildScript: $(buildScriptFileName)$(scriptExt) + - _warnAsErrorArg: '' + + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_rhel6')) }}: + - _runtimeOSArg: /p:RuntimeOS=rhel.6 + - _portableArg: /p:PortableBuild=false + + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_musl')) }}: + - _runtimeOSArg: /p:RuntimeOS=linux-musl + + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, ''), eq(parameters.archType, 'arm')) }}: + - _runtimeOSArg: /p:RuntimeOS=ubuntu.16.04 + + - ${{ if and(or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'WebAssembly')), ne(parameters.container, '')) }}: + - _warnAsErrorArg: '-warnAsError false' + + - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: + - _runtimeOSArg: -os ${{ parameters.osGroup }} + + - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: + - _finalFrameworkArg: -allConfigurations + + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) + + # Windows variables + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci + - _runtimeOSArg: /p:RuntimeOS=win10 + + # Non-Windows variables + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci + - _buildScript: ./$(buildScriptFileName)$(scriptExt) + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - _stripSymbolsArg: -stripSymbols + + - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) + - ${{ parameters.variables }} + + dependsOn: + - checkout + - ${{ parameters.dependsOn }} + workspace: + clean: all + + enablePublishBuildArtifacts: true + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + steps: + - checkout: none + clean: true + + - download: current + artifact: Checkout_bundle + displayName: Download Checkout.bundle + + - script: | + git clone $(Pipeline.Workspace)/Checkout_bundle/Checkout.bundle . + git remote set-url origin $(Build.Repository.Uri) + displayName: Clone the repository from Checkout.bundle + + - ${{ parameters.steps }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml new file mode 100644 index 00000000000000..d2aca88a577f95 --- /dev/null +++ b/eng/pipelines/libraries/build-job.yml @@ -0,0 +1,120 @@ +parameters: + buildConfig: '' + osGroup: '' + osSubgroup: '' + archType: '' + framework: netcoreapp + isOfficialBuild: false + timeoutInMinutes: 150 + preBuildSteps: [] + container: '' + variables: {} + pool: '' + runTests: false + +jobs: + - template: /eng/pipelines/libraries/base-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + archType: ${{ parameters.archType }} + framework: ${{ parameters.framework }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + preBuildSteps: ${{ parameters.preBuildSteps }} + container: ${{ parameters.container }} + pool: ${{ parameters.pool }} + stepName: Build + + variables: + - _skipTestRestoreArg: /p:SkipTestRestore=true + - _buildAction: -build + - ${{ parameters.variables }} + - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: + - EMSDK_PATH: $(Build.BinariesDirectory)/emsdk + - ${{ if eq(parameters.runTests, true) }}: + - _buildAction: -build -buildtests /p:ArchiveTests=true + - ${{ if ne(parameters.framework, 'allConfigurations') }}: + - _skipTestRestoreArg: /p:SkipTestRestore=false + + steps: + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) + displayName: Restore + + - ${{ if eq(parameters.isOfficialBuild, true) }}: + - task: DotNetCoreCLI@2 + displayName: Restore internal tools + condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) + inputs: + command: restore + feedsToUse: config + projects: 'eng/common/internal/Tools.csproj' + nugetConfigPath: 'eng/internal/NuGet.config' + restoreDirectory: '$(Build.SourcesDirectory)\.packages' + verbosityRestore: 'normal' + externalFeedCredentials: 'dotnet-core-internal-tooling' + + - ${{ if eq(parameters.osGroup, 'OSX') }}: + - script: | + brew install pkgconfig openssl + ln -s /usr/local/opt/openssl/lib/pkgconfig/libcrypto.pc /usr/local/lib/pkgconfig/ + ln -s /usr/local/opt/openssl/lib/pkgconfig/libssl.pc /usr/local/lib/pkgconfig/ + ln -s /usr/local/opt/openssl/lib/pkgconfig/openssl.pc /usr/local/lib/pkgconfig/ + displayName: Install Build Dependencies + + - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: + - script: | + EMSCRIPTEN_VERSION=1.38.47 + git clone https://github.com/emscripten-core/emsdk.git $(EMSDK_PATH) + cd $(EMSDK_PATH) + ./emsdk install ${EMSCRIPTEN_VERSION}-upstream + ./emsdk activate --embedded ${EMSCRIPTEN_VERSION}-upstream + displayName: Install Emscripten + + - script: $(_buildScript) + $(_buildAction) + $(_buildArguments) + displayName: Build + + - ${{ if eq(parameters.runTests, false) }}: + - task: CopyFiles@2 + displayName: Prepare ref folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/ref + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/ref + + - task: CopyFiles@2 + displayName: Prepare runtime folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/runtime + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/runtime + + - task: CopyFiles@2 + displayName: Prepare testhost folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/testhost + + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin + includeRootFolder: true + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: Build Assets + + - ${{ if eq(parameters.runTests, true) }}: + - template: /eng/pipelines/libraries/helix.yml + parameters: + targetOS: ${{ parameters.osGroup }} + archGroup: ${{ parameters.archType }} + configuration: ${{ parameters.buildConfig }} + helixQueues: ${{ join(';', parameters.helixQueues) }} + msbuildScript: $(_msbuildCommand) + framework: ${{ parameters.framework }} + testScope: ${{ parameters.testScope }} + creator: dotnet-bot + helixToken: '' diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml new file mode 100644 index 00000000000000..4a401c7531ce48 --- /dev/null +++ b/eng/pipelines/libraries/build-test-job.yml @@ -0,0 +1,65 @@ +parameters: + buildConfig: '' + osGroup: '' + osSubgroup: '' + archType: '' + framework: netcoreapp + isOfficialBuild: false + timeoutInMinutes: 150 + container: '' + publishTestArtifacs: true + pool: '' + testScope: '' + variables: {} + runTests: false + +jobs: + - template: /eng/pipelines/libraries/base-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + archType: ${{ parameters.archType }} + framework: ${{ parameters.framework }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + container: ${{ parameters.container }} + pool: ${{ parameters.pool }} + stepName: Test + + dependsOn: + - ${{ format('libraries_Build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + + variables: + - _archiveTestsParameter: /p:ArchiveTests=true + - _skipTestRestoreArg: /p:SkipTestRestore=false + + - ${{ parameters.variables }} + + steps: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Build Assets + artifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + artifactFileName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}$(archiveExtension) + unpackFolder: $(Build.SourcesDirectory)/artifacts + + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) + displayName: Restore + + - script: $(_buildScript) + -buildtests + -testScope ${{ parameters.testScope }} + $(_buildArguments) + $(_archiveTestsParameter) + displayName: Build + + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: true + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}_{3}', parameters.framework, parameters.osGroup, parameters.archType, parameters.buildConfig) }} + displayName: Test Assets diff --git a/eng/pipelines/libraries/build.yml b/eng/pipelines/libraries/build.yml deleted file mode 100644 index ec77f8bbcad6ce..00000000000000 --- a/eng/pipelines/libraries/build.yml +++ /dev/null @@ -1,134 +0,0 @@ -parameters: - buildConfig: '' - osGroup: '' - archType: '' - osSubgroup: '' - framework: netcoreapp - isOfficialBuild: false - timeoutInMinutes: 150 - preBuildSteps: [] - testScope: '' - runTests: false - publishTestArtifacts: false - helixQueues: [] - container: '' - crossrootfsDir: '' - -jobs: - - template: /eng/common/templates/job/job.yml - parameters: - variables: - - _msbuildCommonParameters: '' - - _finalFrameworkArg: -framework ${{ parameters.framework }} - - _skipTestRestoreArg: /p:SkipTestRestore=true - - _testScopeArg: '' - - _actionParameters: -build - - - ${{ if ne(parameters.crossrootfsDir, '') }}: - - ROOTFS_DIR: ${{ parameters.crossrootfsDir}} - - - ${{ if ne(parameters.testScope, '') }}: - - _testScopeArg: -testscope ${{ parameters.testScope }} - - _skipTestRestoreArg: '' - - _actionParameters: -buildtests - - - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: - - _finalFrameworkArg: -allConfigurations - - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - - - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testScopeArg) $(_skipTestRestoreArg) - - _commonArguments: $(_args) - - # Windows variables - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(buildScriptFileName).cmd - - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci - - # Non-Windows variables - - ${{ if ne(parameters.targetOS, 'Windows_NT') }}: - - _buildScript: ./$(buildScriptFileName).sh - - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - _commonArguments: $(_args) -stripSymbols - - - ${{ if or(eq(parameters.runTests, 'true'), eq(parameters.publishTestArtifacts, 'true')) }}: - - ${{ if ne(parameters.framework, 'allConfigurations') }}: - - _commonArguments: $(_args) /p:ArchiveTests=Tests - - ${{ if eq(parameters.framework, 'allConfigurations') }}: - - _commonArguments: $(_args) /p:ArchiveTests=Packages - - enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed - helixRepo: dotnet/runtime - - ${{ if ne(parameters.container, '') }}: - ${{ if eq(parameters.container.registry, 'mcr') }}: - container: ${{ format('{0}:{1}', 'mcr.microsoft.com/dotnet-buildtools/prereqs', parameters.container.image) }} - ${{ if ne(parameters.container.registry, 'mcr') }}: - container: ${{ format('{0}:{1}', parameters.container.registry, parameters.container.image) }} - - ${{ if eq(parameters.testScope, '') }}: - name: ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - $ {{ if eq(parameters.framework, 'netcoreapp') }}: - displayName: ${{ format('{0}{1} Build {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.framework, 'netcoreapp') }}: - displayName: ${{ format('{0} Build {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.testScope, '') }}: - name: ${{ format('libraries_test_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - $ {{ if eq(parameters.framework, 'netcoreapp') }}: - displayName: ${{ format('{0}{1} Test {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.framework, 'netcoreapp') }}: - displayName: ${{ format('{0} Test {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.testScope, '') }}: - dependencies: - - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - workspace: - clean: all - - enablePublishBuildArtifacts: true - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - steps: - - ${{ if ne(parameters.preBuildSteps[0], '') }}: - - ${{ parameters.preBuildSteps }} - - - script: $(_buildScript) -restore $(_commonArguments) - displayName: Restore - - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - task: DotNetCoreCLI@2 - displayName: Restore internal tools - condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) - inputs: - command: restore - feedsToUse: config - projects: 'eng/common/internal/Tools.csproj' - nugetConfigPath: 'eng/internal/NuGet.config' - restoreDirectory: '$(Build.SourcesDirectory)\.packages' - verbosityRestore: 'normal' - externalFeedCredentials: 'dotnet-core-internal-tooling' - - - script: $(_buildScript) - $(_actionParameters) - $(_commonArguments) - $(_msbuildCommonParameters) - displayName: Build - - - ${{ if eq(parameters.runTests, 'true') }}: - - template: /eng/pipelines/libraries/helix.yml - parameters: - targetOS: ${{ parameters.targetOS }} - archGroup: ${{ parameters.archType }} - configuration: ${{ parameters.buildConfig }} - helixQueues: ${{ join(';', parameters.helixQueues) }} - msbuildScript: $(_msbuildCommand) - framework: ${{ parameters.framework }} - testScope: ${{ parameters.testScope }} - creator: dotnet-bot - helixToken: '' - - # TODO: publish test artifacts when publishTestArtifacts == true. diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml new file mode 100644 index 00000000000000..12716b9e4c56c6 --- /dev/null +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -0,0 +1,120 @@ +parameters: + jobTemplate: '' + variables: {} + osGroup: '' + osSubgroup: '' + archType: '' + container: '' + pool: '' + platform: '' + jobParameters: {} + +jobs: +- template: ${{ parameters.jobTemplate }} + parameters: + variables: ${{ parameters.variables }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + archType: ${{ parameters.archType }} + container: ${{ parameters.container }} + pool: ${{ parameters.pool }} + helixQueues: + + # Linux arm + - ${{ if eq(parameters.platform, 'Linux_arm') }}: + - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: + - \(Debian.9.Arm32.Open\)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-9-helix-arm32v7-74c9941-20190620155841 + + # Linux arm64 + - ${{ if eq(parameters.platform, 'Linux_arm64') }}: + - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: + - \(Ubuntu.1804.ArmArch.Open\)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm64v8-a45aeeb-20190620160300 + + # Linux musl x64 + - ${{ if eq(parameters.platform, 'Linux_musl_x64') }}: + - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: + - \(Alpine.310.Amd64.Open\)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.10-helix-3043688-20190918214010 + - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}: + - \(Alpine.38.Amd64.Open\)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.8-helix-09ca40b-20190620184125 + - \(Alpine.39.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-helix-09ca40b-20190620184719 + - \(Alpine.310.Amd64.Open\)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.10-helix-3043688-20190918214010 + + # Linux musl arm64 + - ${{ if eq(parameters.platform, 'Linux_musl_arm64') }}: + - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: + - \(Alpine.38.Arm64.Open\)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.8-helix-arm64v8-a45aeeb-20190620184035 + + # Linux rhel6 x64 + - ${{ if eq(parameters.platform, 'Linux_rhel6_x64') }}: + - RedHat.6.Amd64.Open + + # Linux x64 + - ${{ if eq(parameters.platform, 'Linux_x64') }}: + - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: + - Centos.7.Amd64.Open + - RedHat.7.Amd64.Open + - Debian.9.Amd64.Open + - Ubuntu.1604.Amd64.Open + - Ubuntu.1804.Amd64.Open + - SLES.12.Amd64.Open + - SLES.15.Amd64.Open + - \(Fedora.29.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-29-helix-09ca40b-20190508143249 + - \(Ubuntu.1910.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64-cfcfd50-20191030180623 + - \(Debian.10.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-7c6abd3-20190620155928 + - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}: + - Centos.7.Amd64.Open + - RedHat.7.Amd64.Open + - Debian.9.Amd64.Open + - Ubuntu.1604.Amd64.Open + - Ubuntu.1804.Amd64.Open + - SLES.15.Amd64.Open + - \(Fedora.29.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-29-helix-09ca40b-20190508143249 + + # OSX x64 + - ${{ if eq(parameters.platform, 'OSX_x64') }}: + - OSX.1013.Amd64.Open + - OSX.1014.Amd64.Open + + # Windows_NT x64 + - ${{ if eq(parameters.platform, 'Windows_NT_x64') }}: + # netcoreapp + - ${{ if eq(parameters.jobParameters.framework, 'netcoreapp') }}: + - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: + - Windows.7.Amd64.Open + - Windows.81.Amd64.Open + - Windows.10.Amd64.ServerRS5.Open + - Windows.10.Amd64.Server19H1.Open + - (Windows.Nano.1809.Amd64.Open)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-61052b7-20190723211353 + - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}: + - Windows.7.Amd64.Open + - Windows.81.Amd64.Open + - Windows.10.Amd64.Server19H1.ES.Open + - (Windows.Nano.1809.Amd64.Open)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-61052b7-20190723211353 + + # NETFX + - ${{ if eq(parameters.jobParameters.framework, 'netfx') }}: + - Windows.10.Amd64.Client19H1.Open + + # AllConfigurations + - ${{ if eq(parameters.jobParameters.framework, 'allConfigurations') }}: + - Windows.10.Amd64.Server19H1.Open + + # Windows_NT x86 + - ${{ if eq(parameters.platform, 'Windows_NT_x86') }}: + # netcoreapp + - ${{ if eq(parameters.jobParameters.framework, 'netcoreapp') }}: + - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: + - Windows.7.Amd64.Open + - Windows.81.Amd64.Open + - Windows.10.Amd64.ServerRS5.Open + - Windows.10.Amd64.Server19H1.Open + - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}: + - Windows.7.Amd64.Open + - Windows.81.Amd64.Open + - Windows.10.Amd64.Server19H1.ES.Open + + # NETFX + - ${{ if eq(parameters.jobParameters.framework, 'netfx') }}: + - Windows.10.Amd64.Client19H1.Open + + ${{ insert }}: ${{ parameters.jobParameters }} diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index 3359fd16743bdf..950638fdd7f934 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -6,52 +6,73 @@ schedules: branches: include: - master - - release/3.0 - -resources: - containers: - - container: centos7_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df28-20191023143754 - - - container: centos6_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-f39df28-20191023143802 - - - container: alpine_39_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-WithNode-0fc54a3-20190918214015 - - - container: alpine_37_arm64_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-406629a-20191023143847 - - - container: ubuntu_1604_arm64_cross_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-cfdd435-20191023143847 - - - container: ubuntu_1604_arm_cross_container - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-14.04-23cacb0-20191023143847 + - release/*.* variables: - template: variables.yml -stages: - # Windows outerloop legs - - ${{ if or(endsWith(variables['Build.DefinitionName'], 'windows'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }}: - - template: windows.yml - parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} +jobs: + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Release + platforms: + - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: + - Windows_NT_x86 + - ${{ if eq(variables['isFullMatrix'], true) }}: + - Windows_NT_x86 + - ${{ if eq(variables['includeLinuxOuterloop'], true) }}: + - Linux_rhel6_x64 + - ${{ if eq(variables['isFullMatrix'], true) }}: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 + - Linux_musl_arm64 + - ${{ if and(eq(variables['includeOsxOuterloop'], true), eq(variables['isFullMatrix'], true)) }}: + - OSX_x64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + isFullMatrix: ${{ variables['isFullMatrix'] }} + framework: netcoreapp + runTests: true testScope: outerloop - - # Linux outerloop legs - - ${{ if or(endsWith(variables['Build.DefinitionName'], 'linux'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }}: - - template: linux.yml + + - ${{ if eq(variables['isFullMatrix'], false) }}: + - template: /eng/pipelines/common/platform-matrix.yml parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - testScope: outerloop + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Debug + platforms: + - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: + - Windows_NT_x86 + - ${{ if eq(variables['includeLinuxOuterloop'], true) }}: + - Linux_x64 + - Linux_musl_x64 + - ${{ if eq(variables['includeOsxOuterloop'], true) }}: + - OSX_x64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + isFullMatrix: ${{ variables['isFullMatrix'] }} + framework: netcoreapp + runTests: true + testScope: outerloop - # MacOS outerloop legs - - ${{ if or(endsWith(variables['Build.DefinitionName'], 'osx'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }}: - - template: macos.yml + - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: + - template: /eng/pipelines/common/platform-matrix.yml parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - testScope: outerloop + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Release + platforms: + - Windows_NT_x86 + - ${{ if eq(variables['isFullMatrix'], true) }}: + - Windows_NT_x64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + isFullMatrix: ${{ variables['isFullMatrix'] }} + framework: netfx + runTests: true + testScope: outerloop \ No newline at end of file diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml new file mode 100644 index 00000000000000..a6ae0f941d1972 --- /dev/null +++ b/eng/pipelines/libraries/run-test-job.yml @@ -0,0 +1,63 @@ +parameters: + buildConfig: '' + osGroup: '' + osSubgroup: '' + archType: '' + framework: netcoreapp + isOfficialBuild: false + timeoutInMinutes: 150 + pool: '' + testScope: '' + helixQueues: [] + variables: {} + +jobs: + - template: /eng/pipelines/libraries/base-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + archType: ${{ parameters.archType }} + framework: ${{ parameters.framework }} + isOfficialBuild: ${{ parameters.isOfficialBuild }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + container: '' # we just send to helix, no need to use a container. + pool: ${{ parameters.pool }} + stepName: Run + + dependsOn: + - ${{ format('libraries_Build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('libraries_Test_{0}_{1}_x64_Release', parameters.framework, parameters.osGroup) }} + + variables: + - _archiveTestsParameter: /p:ArchiveTests=true + - _skipTestRestoreArg: /p:SkipTestRestore=false + - ${{ parameters.variables }} + + steps: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Build Assets + artifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + artifactFileName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}$(archiveExtension) + unpackFolder: $(Build.SourcesDirectory)/artifacts + + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Test Assets + cleanUnpackFolder: false + unpackFolder: $(Build.SourcesDirectory)/artifacts + artifactName: ${{ format('libraries_test_assets_{0}_{1}_x64_Release', parameters.framework, parameters.osGroup) }} + artifactFileName: ${{ format('libraries_test_assets_{0}_{1}_x64_Release', parameters.framework, parameters.osGroup) }}$(archiveExtension) + + - template: /eng/pipelines/libraries/helix.yml + parameters: + targetOS: ${{ parameters.osGroup }} + archGroup: ${{ parameters.archType }} + configuration: ${{ parameters.buildConfig }} + helixQueues: ${{ join('+', parameters.helixQueues) }} + msbuildScript: $(_msbuildCommand) + framework: ${{ parameters.framework }} + testScope: ${{ parameters.testScope }} + creator: dotnet-bot + helixToken: '' \ No newline at end of file diff --git a/eng/pipelines/libraries/variables.yml b/eng/pipelines/libraries/variables.yml index f0d229fc74f102..ca932a8fe9f180 100644 --- a/eng/pipelines/libraries/variables.yml +++ b/eng/pipelines/libraries/variables.yml @@ -7,3 +7,11 @@ variables: value: /eng/pipelines/libraries - name: isOfficialBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} + - name: isFullMatrix + value: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} + - name: includeWindowsOuterloop + value: ${{ or(endsWith(variables['Build.DefinitionName'], 'windows'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }} + - name: includeLinuxOuterloop + value: ${{ or(endsWith(variables['Build.DefinitionName'], 'linux'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }} + - name: includeOsxOuterloop + value: ${{ or(endsWith(variables['Build.DefinitionName'], 'osx'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }} diff --git a/eng/referenceAssemblies.props b/eng/referenceAssemblies.props index ba1e667fb4e2dd..dbb4b62aabc571 100644 --- a/eng/referenceAssemblies.props +++ b/eng/referenceAssemblies.props @@ -3,13 +3,13 @@ true - $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'ref')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', '$(MSBuildProjectName)')) - $([MSBuild]::NormalizeDirectory('$(ReferenceAssemblyOutputPath)', '$(MSBuildProjectName)')) + $([MSBuild]::NormalizeDirectory('$(ReferenceAssemblyOutputPath)', 'ref')) $(BaseOutputPath)$(Configuration) - $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'ref', '$(MSBuildProjectName)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', '$(MSBuildProjectName)', 'ref')) $(BaseIntermediateOutputPath)$(Configuration) diff --git a/src/libraries/build.proj b/src/libraries/build.proj index f132086061a01c..2d14f266047e6a 100644 --- a/src/libraries/build.proj +++ b/src/libraries/build.proj @@ -23,7 +23,7 @@ <_RestoreProjects Include="$(MSBuildThisFileDirectory)restore\dirs.proj" Build="true" /> - + <_RestoreProjects Include="$(MSBuildThisFileDirectory)tests.proj" Build="false" /> diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index c65ae3d5841aa6..c525db6c7bd1f2 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -10,8 +10,9 @@ - + + From 2121faa111e370268c7e78b09d17026dd1dff281 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Nov 2019 00:02:13 -0800 Subject: [PATCH 05/14] Removed unused yaml and projects --- eng/pipelines/libraries/base-job.yml | 2 +- eng/pipelines/libraries/corefx-base.yml | 217 ------------------------ eng/pipelines/libraries/freebsd.yml | 34 ---- eng/pipelines/libraries/linux.yml | 195 --------------------- eng/pipelines/libraries/macos.yml | 59 ------- eng/pipelines/libraries/outerloop.yml | 4 +- eng/pipelines/libraries/pre-publish.yml | 88 ---------- eng/pipelines/libraries/windows.yml | 162 ------------------ src/libraries/publishTestAssets.proj | 66 ------- 9 files changed, 4 insertions(+), 823 deletions(-) delete mode 100644 eng/pipelines/libraries/corefx-base.yml delete mode 100644 eng/pipelines/libraries/freebsd.yml delete mode 100644 eng/pipelines/libraries/linux.yml delete mode 100644 eng/pipelines/libraries/macos.yml delete mode 100644 eng/pipelines/libraries/pre-publish.yml delete mode 100644 eng/pipelines/libraries/windows.yml delete mode 100644 src/libraries/publishTestAssets.proj diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index b3cb574fa6adff..b3671d4bd0031d 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -86,7 +86,7 @@ jobs: steps: - checkout: none - clean: true + clean: true - download: current artifact: Checkout_bundle diff --git a/eng/pipelines/libraries/corefx-base.yml b/eng/pipelines/libraries/corefx-base.yml deleted file mode 100644 index aac9dbf563799f..00000000000000 --- a/eng/pipelines/libraries/corefx-base.yml +++ /dev/null @@ -1,217 +0,0 @@ -parameters: - # Required: TargetOS value to know what script commands to use - # Accepted values: - # - Windows_NT - # - Linux - # - OSX - # - FreeBSD - targetOS: '' - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Required: value to specify if the job is comming from an official build to run extra steps and sign binaries - # Default: false - isOfficialBuild: false - - # EACH JOB SHOULD INCLUDE THE FOLLOWING PROPERTIES (ASIDE FROM THE REQUIRED ONES IN THE JOB SCHEMA) - - # Required: as part of the strategy matrix, the following variables should be defined - # _BuildConfig: Debug | Release - # _architecture: x64 | x86 | arm | arm64 - # _framework: (netcoreapp, netfx). - # _helixQueues: Windows.Amd64 (Only needed if submitToHelix -> true.) -- Queues should be separated by + if multiple. - - # Required: empty | innerloop | outerloop | all - testScope: '' - - # Optional: _publishTests -> Boolean -> Publish test assets to blob storage if true. - # Default: false - # _publishTests: true | false - - # Required: submitToHelix -> Boolean -> Value to know if it should submit tests payloads to helix. - - # Optional: buildScriptPrefix -> String -> string to append to Unix build script. - # buildScriptPrefix: 'HOME=/home/ ' -> 'HOME=/home/ ./$(buildScriptName).sh ...' - - # Optional: buildExtraArguments -> String -> string to append at the end of the build arguments - # buildExtraArguments: /p:RuntimeOS=win10 -> '$(buildScriptName).cmd ... /p:RuntimeOS=win10' - - # Optional: customBuildSteps -> Array -> list of steps to run instead of the common build steps. - # customBuildSteps: - # - script: echo Hello World - # displayName: MyScript - - # Optional: preBuildSteps -> Array -> list of steps to be executed before common build steps. - # In example, to install build dependencies, or setup an environment. - # preBuildSteps: - # - script: brew install cmake - # displayName: Install Cmake - - # timeoutInMinutes: if a job doesn't define it, we will default it to 150 minutes - -jobs: - - ${{ each job in parameters.jobs }}: - - template: /eng/common/templates/job/job.yml - parameters: - variables: - - # pass along job variables - - ${{ each variable in job.variables }}: - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - - _msbuildCommonParameters: '' - - _archiveTestsParameter: '' - - _finalFrameworkArg: -framework $(_framework) - - _testScopeArg: '' - - - ${{ if ne(parameters.testScope, '') }}: - - _testScopeArg: -testscope ${{ parameters.testScope }} - - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - - - ${{ if eq(job.submitToHelix, 'true') }}: - - _archiveTestsParameter: /p:ArchiveTests=true - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - group: DotNet-HelixApi-Access - - - ${{ if ne(job.frameworkArg, '') }}: - - _finalFrameworkArg: ${{ job.frameworkArg }} - - - _args: -configuration $(_BuildConfig) -ci -arch $(_architecture) $(_finalFrameworkArg) $(_archiveTestsParameter) ${{ job.buildExtraArguments }} - - _commonArguments: $(_args) - - # Windows variables - - ${{ if eq(parameters.targetOS, 'Windows_NT') }}: - - _buildScript: $(buildScriptFileName).cmd - - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci - - # Non-Windows variables - - ${{ if ne(parameters.targetOS, 'Windows_NT') }}: - - _buildScript: ${{ job.buildScriptPrefix }}./$(buildScriptFileName).sh - - _msbuildCommand: ${{ job.buildScriptPrefix }}./eng/common/msbuild.sh --warnaserror false --ci - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - _commonArguments: $(_args) -stripSymbols - - # pass along job properties - ${{ each property in job }}: - ${{ if and(ne(property.key, 'job'), ne(property.key, 'variables')) }}: - ${{ property.key }}: ${{ property.value }} - - # enable helix telemetry -- we only send telemetry during official builds - enableTelemetry: ${{ parameters.isOfficialBuild }} - helixRepo: dotnet/corefx - - name: ${{ job.job }} - workspace: - clean: all - - # enabling publish build artifacts, will publish all build logs under /artifacts/log/ - enablePublishBuildArtifacts: true - - ${{ if eq(job.timeoutInMinutes, '') }}: - timeoutInMinutes: 150 - - steps: - - ${{ if ne(job.preBuildSteps[0], '') }}: - - ${{ job.preBuildSteps }} - - - script: $(_buildScript) -restore $(_commonArguments) - displayName: Restore Build Tools - - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - task: DotNetCoreCLI@2 - displayName: Restore internal tools - condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) - inputs: - command: restore - feedsToUse: config - projects: 'eng/common/internal/Tools.csproj' - nugetConfigPath: 'eng/internal/NuGet.config' - restoreDirectory: '$(Build.SourcesDirectory)\.packages' - verbosityRestore: 'normal' - externalFeedCredentials: 'dotnet-core-internal-tooling' - - - ${{ if eq(job.customBuildSteps[0], '') }}: - - script: $(_buildScript) - -build - -buildtests - $(_commonArguments) - $(_testScopeArg) - $(_msbuildCommonParameters) - displayName: Build Sources and Tests - - - ${{ if ne(job.customBuildSteps[0], '') }}: - - ${{ job.customBuildSteps }} - - - ${{ if eq(job.submitToHelix, 'true') }}: - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - script: $(_msbuildCommand) $(sourcesRoot)/publishTestAssets.proj - /p:FilesToPublishPattern=$(Build.SourcesDirectory)/artifacts/helix/**/*.zip - /p:AccountKey=$(dotnetfeed-storage-access-key-1) - /p:ExpectedFeedUrl=$(_dotnetFeedUrl) - /p:OSGroup=${{ parameters.targetOS }} - /p:ArchGroup=$(_architecture) - /p:ConfigurationGroup=$(_BuildConfig) - /p:TargetGroup=$(_framework) - /p:OfficialBuildId=$(Build.BuildNumber) - /p:ContinuousIntegrationBuild=true - /p:AssetManifestFileName=corefx-test-assets.xml - /p:ManifestBuildId=$(Build.BuildNumber) - /p:ManifestBuildData=Location=$(_dotnetFeedUrl) - /p:ManifestBranch=$(Build.SourceBranchName) - /p:ManifestCommit=$(Build.SourceVersion) - /p:ManifestRepoUri=$(Build.Repository.Uri) - displayName: Publish test assets to dotnet-core feed - condition: and(succeeded(), eq(variables['_publishTests'], 'true')) - - - task: PublishBuildArtifacts@1 - displayName: Publish test asset manifest to artifacts container - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/TestAssetsManifests - artifactName: $(Agent.Os)_$(Agent.JobName) - artifactType: container - condition: and(succeeded(), eq(variables['_publishTests'], 'true')) - - - template: /eng/pipelines/libraries/helix.yml - parameters: - # send tests to helix only on public builds, official scheduled builds or manual official builds. - condition: eq(${{ parameters.isOfficialBuild }}, False) - targetOS: ${{ parameters.targetOS }} - archGroup: $(_architecture) - configuration: $(_BuildConfig) - helixQueues: $(_helixQueues) - msbuildScript: $(_msbuildCommand) - framework: $(_framework) - testScope: ${{ coalesce(parameters.testScope, 'innerloop') }} # if parameters.testScope is empty use 'innerloop' - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - officialBuildId: $(Build.BuildNumber) - helixToken: $(HelixApiAccessToken) - - ${{ if eq(parameters.isOfficialBuild, 'false') }}: - # TODO: SET Creator to the PR owner whenever Azure DevOps supports a good way to retrieve it. - creator: dotnet-bot - helixToken: '' - - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish packages to artifacts container - inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/packages - artifactName: packages - artifactType: container - condition: and(succeeded(), ne(variables['_skipPublishPackages'], 'true')) diff --git a/eng/pipelines/libraries/freebsd.yml b/eng/pipelines/libraries/freebsd.yml deleted file mode 100644 index 5c18391311ee61..00000000000000 --- a/eng/pipelines/libraries/freebsd.yml +++ /dev/null @@ -1,34 +0,0 @@ -# FreeBSD leg -parameters: - # Required: value to specify if the job is comming from an official build to run extra steps and sign binaries - # Default: false - isOfficialBuild: false - # Required: value to specify if the full test matrix should be tested - # Default: false - fullMatrix: false - # Optional: value to scope the tests. - # Default: empty - testScope: '' - -jobs: -- template: corefx-base.yml - parameters: - isOfficialBuild: ${{ parameters.isOfficialBuild }} - testScope: ${{ parameters.testScope }} - targetOS: FreeBSD - jobs: - - - job: FreeBSD - strategy: - matrix: - x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: netcoreapp - _buildScriptPrefix: 'DotNetCoreSdkDir=/usr/local/dotnet/ DotNetRoot=/usr/local/dotnet/ ' - - pool: - name: dnceng-freebsd-internal - - buildScriptPrefix: $(_buildScriptPrefix) - submitToHelix: false diff --git a/eng/pipelines/libraries/linux.yml b/eng/pipelines/libraries/linux.yml deleted file mode 100644 index 2f58da1b2b66a6..00000000000000 --- a/eng/pipelines/libraries/linux.yml +++ /dev/null @@ -1,195 +0,0 @@ -# Linux legs -parameters: - # Required: value to specify if the job is comming from an official build to run extra steps and sign binaries - # Default: false - isOfficialBuild: false - # Required: value to specify if the full test matrix should be tested - # Default: false - fullMatrix: false - # Optional: value to scope the tests. - # Default: empty - testScope: '' - -stages: -- stage: LinuxStage - displayName: Linux - dependsOn: [] - jobs: - - template: corefx-base.yml - parameters: - isOfficialBuild: ${{ parameters.isOfficialBuild }} - testScope: ${{ parameters.testScope }} - targetOS: Linux - jobs: - - # Legs with Helix testing - - job: LinuxTest - displayName: Build - strategy: - matrix: - # Run RedHat6 in release mode on CI to cover Release configuration differences - RedHat6_x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: netcoreapp - _buildScriptPrefix: '' - _buildExtraArguments: /p:RuntimeOS=rhel.6 /p:PortableBuild=false - _dockerContainer: centos6_container - _helixQueues: $(redhatHelixQueue) - _publishTests: ${{ parameters.fullMatrix }} - - ${{ if eq(parameters.fullMatrix, 'false') }}: - x64_Debug: - _BuildConfig: Debug - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(linuxDefaultQueues) - _dockerContainer: centos7_container - _buildScriptPrefix: '' - _buildExtraArguments: '' - - musl_x64_Debug: - _BuildConfig: Debug - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(alpineQueues) - _dockerContainer: alpine_39_container - _buildScriptPrefix: '' - _buildExtraArguments: /p:RuntimeOS=linux-musl - - ${{ if eq(parameters.fullMatrix, 'true') }}: - x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(linuxDefaultQueues) - _dockerContainer: centos7_container - _buildScriptPrefix: '' - _buildExtraArguments: '' - _publishTests: true - - musl_x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(alpineQueues) - _dockerContainer: alpine_39_container - _buildScriptPrefix: '' - _buildExtraArguments: /p:RuntimeOS=linux-musl - _publishTests: true - - arm_Release: - _BuildConfig: Release - _architecture: arm - _framework: netcoreapp - _helixQueues: $(linuxArmQueues) - _dockerContainer: ubuntu_1604_arm_cross_container - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm ' - _buildExtraArguments: -warnAsError false - _publishTests: true - - musl_arm64_Release: - _BuildConfig: Release - _architecture: arm64 - _framework: netcoreapp - _helixQueues: $(alpineArm64Queues) - _dockerContainer: alpine_37_arm64_container - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' - _buildExtraArguments: -warnAsError false /p:RuntimeOS=linux-musl - _publishTests: true - - arm64_Release: - _BuildConfig: Release - _architecture: arm64 - _framework: netcoreapp - _helixQueues: $(linuxArm64Queues) - _dockerContainer: ubuntu_1604_arm64_cross_container - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' - _buildExtraArguments: -warnAsError false - _publishTests: true - - pool: - name: Hosted Ubuntu 1604 - - container: $[ variables['_dockerContainer'] ] - buildScriptPrefix: $(_buildScriptPrefix) - buildExtraArguments: $(_buildExtraArguments) - - submitToHelix: true - # Temporary till we reduced workloads on ARM64 - timeoutInMinutes: 240 - - variables: - - redhatHelixQueue: RedHat.6.Amd64.Open - - - ${{ if eq(parameters.fullMatrix, 'false') }}: - - linuxDefaultQueues: Centos.7.Amd64.Open+RedHat.7.Amd64.Open+Debian.9.Amd64.Open+Ubuntu.1604.Amd64.Open+Ubuntu.1804.Amd64.Open+SLES.15.Amd64.Open+\(Fedora.29.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-29-helix-09ca40b-20190508143249 - - alpineQueues: \(Alpine.310.Amd64.Open\)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.10-helix-3043688-20190918214010 - - - ${{ if eq(parameters.fullMatrix, 'true') }}: - - linuxDefaultQueues: Centos.7.Amd64.Open+RedHat.7.Amd64.Open+Debian.9.Amd64.Open+Ubuntu.1604.Amd64.Open+Ubuntu.1804.Amd64.Open+SLES.12.Amd64.Open+SLES.15.Amd64.Open+\(Fedora.29.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-29-helix-09ca40b-20190508143249+\(Ubuntu.1910.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64-cfcfd50-20191030180623+\(Debian.10.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-7c6abd3-20190620155928 - - linuxArmQueues: \(Debian.9.Arm32.Open\)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-9-helix-arm32v7-74c9941-20190620155841 - - alpineQueues: \(Alpine.38.Amd64.Open\)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.8-helix-09ca40b-20190620184125+\(Alpine.39.Amd64.Open\)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-helix-09ca40b-20190620184719+\(Alpine.310.Amd64.Open\)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.10-helix-3043688-20190918214010 - - alpineArm64Queues: \(Alpine.38.Arm64.Open\)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.8-helix-arm64v8-a45aeeb-20190620184035 - - linuxArm64Queues: \(Ubuntu.1804.ArmArch.Open\)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm64v8-a45aeeb-20190620160300 - - # Legs without helix testing - # Don't run these legs in outerloop builds - - ${{ if and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all')) }}: - - job: LinuxNoTest - displayName: Build - strategy: - matrix: - wasm_Release: - _BuildConfig: Release - _architecture: wasm - _framework: netcoreapp - _dockerContainer: ubuntu_1604 - _emsdkPath: $(Build.BinariesDirectory)/emsdk - _buildScriptPrefix: 'EMSDK_PATH=$(_emsdkPath) ' - _buildExtraArguments: -os WebAssembly -warnAsError false - _installEmscripten: true - - # Only run these legs in PRs. - ${{ if eq(parameters.fullMatrix, 'false') }}: - arm_Debug: - _BuildConfig: Debug - _architecture: arm - _framework: netcoreapp - _buildExtraArguments: /p:RuntimeOS=ubuntu.16.04 -warnAsError false - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm ' - _dockerContainer: ubuntu_1604_arm_cross_container - - musl_arm64_Debug: - _BuildConfig: Debug - _architecture: arm64 - _framework: netcoreapp - _dockerContainer: alpine_37_arm64_container - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' - _buildExtraArguments: -warnAsError false /p:RuntimeOS=linux-musl - - arm64_Debug: - _BuildConfig: Debug - _architecture: arm64 - _framework: netcoreapp - _dockerContainer: ubuntu_1604_arm64_cross_container - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' - _buildExtraArguments: --warnAsError false - - preBuildSteps: - - script: | - EMSCRIPTEN_VERSION=1.38.47 - git clone https://github.com/emscripten-core/emsdk.git $(_emsdkPath) - cd $(_emsdkPath) - ./emsdk install ${EMSCRIPTEN_VERSION}-upstream - ./emsdk activate --embedded ${EMSCRIPTEN_VERSION}-upstream - displayName: Install Emscripten - condition: and(succeeded(), eq(variables['_installEmscripten'], 'true')) - - pool: - name: Hosted Ubuntu 1604 - - container: $[ variables['_dockerContainer'] ] - buildExtraArguments: $(_buildExtraArguments) - buildScriptPrefix: $(_buildScriptPrefix) - submitToHelix: false diff --git a/eng/pipelines/libraries/macos.yml b/eng/pipelines/libraries/macos.yml deleted file mode 100644 index 6f78bd00c4350e..00000000000000 --- a/eng/pipelines/libraries/macos.yml +++ /dev/null @@ -1,59 +0,0 @@ -# macOS legs -parameters: - # Required: value to specify if the job is comming from an official build to run extra steps and sign binaries - # Default: false - isOfficialBuild: false - # Required: value to specify if the full test matrix should be tested - # Default: false - fullMatrix: false - # Optional: value to scope the tests. - # Default: empty - testScope: '' - -stages: -- stage: MacOSStage - displayName: MacOS - dependsOn: [] - jobs: - - template: corefx-base.yml - parameters: - isOfficialBuild: ${{ parameters.isOfficialBuild }} - testScope: ${{ parameters.testScope }} - targetOS: OSX - jobs: - - # Legs with Helix testing - - job: MacOS - displayName: Build - strategy: - matrix: - ${{ if eq(parameters.fullMatrix, 'false') }}: - x64_Debug: - _BuildConfig: Debug - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(macOSQueues) - - ${{ if eq(parameters.fullMatrix, 'true') }}: - x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(macOSQueues) - _publishTests: true - - pool: - name: Hosted macOS - - preBuildSteps: - - script: | - brew install pkgconfig openssl - ln -s /usr/local/opt/openssl/lib/pkgconfig/libcrypto.pc /usr/local/lib/pkgconfig/ - ln -s /usr/local/opt/openssl/lib/pkgconfig/libssl.pc /usr/local/lib/pkgconfig/ - ln -s /usr/local/opt/openssl/lib/pkgconfig/openssl.pc /usr/local/lib/pkgconfig/ - displayName: Install Build Dependencies - - submitToHelix: true - - variables: - - macOSQueues: OSX.1013.Amd64.Open+OSX.1014.Amd64.Open diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index 950638fdd7f934..c04696f1c8bb77 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -12,6 +12,8 @@ variables: - template: variables.yml jobs: + - template: /eng/pipelines/common/checkout-job.yml + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/libraries/build-job.yml @@ -20,7 +22,7 @@ jobs: - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: - Windows_NT_x86 - ${{ if eq(variables['isFullMatrix'], true) }}: - - Windows_NT_x86 + - Windows_NT_x64 - ${{ if eq(variables['includeLinuxOuterloop'], true) }}: - Linux_rhel6_x64 - ${{ if eq(variables['isFullMatrix'], true) }}: diff --git a/eng/pipelines/libraries/pre-publish.yml b/eng/pipelines/libraries/pre-publish.yml deleted file mode 100644 index 71752ad206fe78..00000000000000 --- a/eng/pipelines/libraries/pre-publish.yml +++ /dev/null @@ -1,88 +0,0 @@ -parameters: - buildConfiguration: Release - dependsOn: [] - -stages: -- stage: PrePublish - displayName: Pre-publish - dependsOn: ${{ parameters.dependsOn }} - jobs: - - template: /eng/common/templates/jobs/jobs.yml - parameters: - enableMicrobuild: true - enablePublishUsingPipelines: $(_PublishUsingPipelines) - enablePublishBuildAssets: true - enablePublishBuildArtifacts: true - jobs: - - job: SignAndPrepare - displayName: Sign and prepare to publish - timeoutInMinutes: 160 - - pool: - name: NetCoreInternal-Pool - queue: buildpool.windows.10.amd64.vs2017 - - workspace: - clean: all - - variables: - - group: DotNet-Versions-Publish - - _artifactsDir: $(Build.SourcesDirectory)/artifacts - - _TeamName: DotNetCore - - _SignType: real - - _BuildConfig: ${{ parameters.buildConfiguration }} - - steps: - - powershell: | - $prefix = "refs/heads/" - $branch = "$(Build.SourceBranch)" - $branchName = $branch - if ($branchName.StartsWith($prefix)) - { - $branchName = $branchName.Substring($prefix.Length) - } - Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName" - Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName" - displayName: Find true SourceBranchName - - - task: DownloadBuildArtifacts@0 - displayName: Download packages to publish - inputs: - artifactName: packages - downloadPath: $(_artifactsDir) - - - script: $(buildScriptFileName).cmd - -restore - -ci - displayName: Restore tools - - - script: $(buildScriptFileName).cmd - -sign - -ci - -configuration $(_BuildConfig) - /p:DotNetSignType=$(_SignType) - /p:OfficialBuildId=$(Build.BuildNumber) - displayName: Sign packages - - - script: $(buildScriptFileName).cmd - -publish - -ci - -configuration $(_BuildConfig) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) - /p:DotNetPublishBlobFeedUrl=$(_dotnetFeedUrl) - /p:DotNetPublishToBlobFeed=true - /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) - /p:OfficialBuildId=$(Build.BuildNumber) - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/PublishToArtifacts.binlog - displayName: Publish to artifacts and produce manifest - - - script: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 $(sourcesRoot)/build.proj - -warnaserror:0 -ci - /t:UpdatePublishedVersions - /p:GitHubAuthToken=$(AccessToken-dotnet-build-bot-public-repo) - /p:VersionsRepoOwner=dotnet - /p:VersionsRepo=versions - /p:VersionsRepoPath=build-info/dotnet/corefx/$(FullBranchName) - /p:ShippedNuGetPackageGlobPath=$(_artifactsDir)/packages/**/*.nupkg - displayName: Update dotnet/versions diff --git a/eng/pipelines/libraries/windows.yml b/eng/pipelines/libraries/windows.yml deleted file mode 100644 index c1825eaacad667..00000000000000 --- a/eng/pipelines/libraries/windows.yml +++ /dev/null @@ -1,162 +0,0 @@ -# Windows legs -parameters: - # Required: value to specify if the job is comming from an official build to run extra steps and sign binaries - # Default: false - isOfficialBuild: false - # Required: value to specify if the full test matrix should be tested - # Default: false - fullMatrix: false - # Optional: value to scope the tests. - # Default: empty - testScope: '' - -stages: -- stage: WindowsStage - displayName: Windows - dependsOn: [] - jobs: - - - template: corefx-base.yml - parameters: - isOfficialBuild: ${{ parameters.isOfficialBuild }} - testScope: ${{ parameters.testScope }} - targetOS: Windows_NT - jobs: - - # Legs with Helix testing - - job: WindowsTest - displayName: Build - strategy: - matrix: - # PR CI Matrix - ${{ if eq(parameters.fullMatrix, 'false') }}: - x64_Debug: - _BuildConfig: Debug - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(netcoreappWindowsQueues)+$(nanoQueues) - - x86_Release: - _BuildConfig: Release - _architecture: x86 - _framework: netcoreapp - _helixQueues: $(netcoreappWindowsQueues) - - NETFX_x86_Release: - _BuildConfig: Release - _architecture: x86 - _framework: netfx - _helixQueues: $(netFxQueues) - - # Full test matrix - ${{ if eq(parameters.fullMatrix, 'true') }}: - x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: netcoreapp - _helixQueues: $(netcoreappWindowsQueues)+$(nanoQueues) - _publishTests: true - - x86_Release: - _BuildConfig: Release - _architecture: x86 - _framework: netcoreapp - _helixQueues: $(netcoreappWindowsQueues) - _publishTests: true - - NETFX_x86_Release: - _BuildConfig: Release - _architecture: x86 - _framework: netfx - _helixQueues: $(netFxQueues) - _skipPublishPackages: true # In NETFX leg we don't produce packages - - NETFX_x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: netfx - _helixQueues: $(netFxQueues) - _skipPublishPackages: true # In NETFX leg we don't produce packages - - pool: - name: Hosted VS2017 - - submitToHelix: true - buildExtraArguments: /p:RuntimeOS=win10 - - variables: - - nanoQueues: "`(Windows.Nano.1809.Amd64.Open`)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-61052b7-20190723211353" - - netFxQueues: Windows.10.Amd64.Client19H1.Open - - - ${{ if eq(parameters.fullMatrix, 'false') }}: - - netcoreappWindowsQueues: Windows.7.Amd64.Open+Windows.81.Amd64.Open+Windows.10.Amd64.Server19H1.ES.Open - - - ${{ if eq(parameters.fullMatrix, 'true') }}: - - netcoreappWindowsQueues: Windows.7.Amd64.Open+Windows.81.Amd64.Open+Windows.10.Amd64.ServerRS5.Open+Windows.10.Amd64.Server19H1.Open - - # There is no point of running legs without outerloop tests, when in an outerloop build. - - ${{ if and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all')) }}: - # Packaging all configurations - - job: AllConfigurations - displayName: Packaging All Configurations - strategy: - matrix: - ${{ if eq(parameters.fullMatrix, 'false') }}: - x64_Debug: - _BuildConfig: Debug - _architecture: x64 - _framework: allConfigurations - _helixQueues: $(allConfigurationsQueues) - - ${{ if eq(parameters.fullMatrix, 'true') }}: - x64_Release: - _BuildConfig: Release - _architecture: x64 - _framework: allConfigurations - _helixQueues: $(allConfigurationsQueues) - - pool: - name: Hosted VS2017 - - submitToHelix: true - buildExtraArguments: /p:RuntimeOS=win10 - - variables: - - allConfigurationsQueues: Windows.10.Amd64.Server19H1.Open - - frameworkArg: -allConfigurations - customBuildSteps: - - script: $(buildScriptFileName).cmd - -build - -configuration $(_BuildConfig) - -ci - -buildtests - -allconfigurations - -arch $(_architecture) - /p:RuntimeOS=win10 - /p:ArchiveTests=true - $(_msbuildCommonParameters) - displayName: Build Packages and Tests - - # Legs without HELIX testing - - ${{ if eq(parameters.fullMatrix, 'true') }}: - - job: WindowsNoTest - displayName: Build - strategy: - matrix: - arm_Release: - _BuildConfig: Release - _architecture: arm - _framework: netcoreapp - - arm64_Release: - _BuildConfig: Release - _architecture: arm64 - _framework: netcoreapp - _publishTests: true - - pool: - name: Hosted VS2017 - - submitToHelix: false - buildExtraArguments: /p:RuntimeOS=win10 diff --git a/src/libraries/publishTestAssets.proj b/src/libraries/publishTestAssets.proj deleted file mode 100644 index eae1362f1bc7c0..00000000000000 --- a/src/libraries/publishTestAssets.proj +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - - - $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'TestAssetsManifests')) - $(AssetManifestDir)$(AssetManifestFileName) - - - - - - - - - <_TestAssetVersion>$(PackageVersion)-$(VersionSuffix) - <_FileRelativePathBase>corefx-tests/$(_TestAssetVersion)/$(OSGroup).$(ArchGroup)/$(TargetGroup)/ - - - - <_ItemsToPush Remove="@(_ItemsToPush)" /> - <_ItemsToPush Include="$(FilesToPublishPattern)" /> - <_ItemsToPush> - $(_FileRelativePathBase)$([System.String]::Copy('%(RecursiveDir)%(Filename)%(Extension)').Replace('\' ,'/')) - - - - - - - - - <_ManifestToPush Remove="@(_ManifestToPush)" /> - <_ManifestToPush Include="$(AssetManifestFilePath)"> - $(_FileRelativePathBase)$(AssetManifestFileName) - - - - - - - - From 253034bb15862c02c2973a56cb67190edc071e69 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Nov 2019 00:34:52 -0800 Subject: [PATCH 06/14] Flow test scope and move testScope arg to base template --- eng/pipelines/libraries/base-job.yml | 7 ++++++- eng/pipelines/libraries/build-job.yml | 2 ++ eng/pipelines/libraries/build-test-job.yml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index b3671d4bd0031d..e345b8973f81ff 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -12,6 +12,7 @@ parameters: variables: {} name: '' displayName: '' + testScope: '' pool: '' stepName: '' @@ -37,6 +38,10 @@ jobs: - _finalFrameworkArg: -framework ${{ parameters.framework }} - _buildScript: $(buildScriptFileName)$(scriptExt) - _warnAsErrorArg: '' + - _testScopeArg: '' + + - ${{ if ne(parameters.testScope, '') }}: + - _testScopeArg: -testscope ${{ parameters.testScope }} - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_rhel6')) }}: - _runtimeOSArg: /p:RuntimeOS=rhel.6 @@ -72,7 +77,7 @@ jobs: - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _stripSymbolsArg: -stripSymbols - - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) + - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) - ${{ parameters.variables }} dependsOn: diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index d2aca88a577f95..b2a270104a980f 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -11,6 +11,7 @@ parameters: variables: {} pool: '' runTests: false + testScope: '' jobs: - template: /eng/pipelines/libraries/base-job.yml @@ -25,6 +26,7 @@ jobs: preBuildSteps: ${{ parameters.preBuildSteps }} container: ${{ parameters.container }} pool: ${{ parameters.pool }} + testScope: ${{ parameters.testScope }} stepName: Build variables: diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index 4a401c7531ce48..a847b40cf7e235 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -25,6 +25,7 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: ${{ parameters.container }} pool: ${{ parameters.pool }} + testScope: ${{ parameters.testScope }} stepName: Test dependsOn: @@ -49,7 +50,6 @@ jobs: - script: $(_buildScript) -buildtests - -testScope ${{ parameters.testScope }} $(_buildArguments) $(_archiveTestsParameter) displayName: Build From 3fa0328d252b828b6704ecda3e106a8ee1efb56e Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Nov 2019 00:51:09 -0800 Subject: [PATCH 07/14] Fix dependencies --- eng/pipelines/libraries/.azure-ci.yml | 2 ++ eng/pipelines/libraries/run-test-job.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/.azure-ci.yml b/eng/pipelines/libraries/.azure-ci.yml index 33cda03d60c598..d0c722ab62f780 100644 --- a/eng/pipelines/libraries/.azure-ci.yml +++ b/eng/pipelines/libraries/.azure-ci.yml @@ -170,6 +170,8 @@ jobs: isFullMatrix: ${{ variables['isFullMatrix'] }} testScope: innerloop framework: netcoreapp + ${{ if eq(variables['isFullMatrix'], true) }}: + dependsOnBuildConfiguration: Release - ${{ if eq(variables['isFullMatrix'], false) }}: - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index a6ae0f941d1972..dc71c882dcf448 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -9,6 +9,7 @@ parameters: pool: '' testScope: '' helixQueues: [] + dependsOnBuildConfiguration: Debug variables: {} jobs: @@ -27,7 +28,7 @@ jobs: dependsOn: - ${{ format('libraries_Build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ format('libraries_Test_{0}_{1}_x64_Release', parameters.framework, parameters.osGroup) }} + - ${{ format('libraries_Test_{0}_{1}_x64_{2}', parameters.framework, parameters.osGroup, parameters.dependsOnBuildConfiguration) }} variables: - _archiveTestsParameter: /p:ArchiveTests=true From d28afe2225b35cdab29756f82816b9280c0dd11c Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Nov 2019 00:55:46 -0800 Subject: [PATCH 08/14] Remove excluded paths --- eng/pipelines/libraries/.azure-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/libraries/.azure-ci.yml b/eng/pipelines/libraries/.azure-ci.yml index d0c722ab62f780..91139132c1e3c5 100644 --- a/eng/pipelines/libraries/.azure-ci.yml +++ b/eng/pipelines/libraries/.azure-ci.yml @@ -22,7 +22,6 @@ trigger: - src/coreclr/* - eng/pipelines/coreclr/* - eng/pipelines/installer/* - - eng/pipelines/common/* pr: branches: @@ -43,7 +42,6 @@ pr: - src/coreclr/* - eng/pipelines/coreclr/* - eng/pipelines/installer/* - - eng/pipelines/common/* variables: - template: variables.yml From 0dc03a24159030e3a047aa396d7ff216618924ba Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Nov 2019 09:30:50 -0800 Subject: [PATCH 09/14] Fix download asset errors in run step --- eng/pipelines/libraries/run-test-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index dc71c882dcf448..d7b316af4e9555 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -48,8 +48,8 @@ jobs: displayName: Test Assets cleanUnpackFolder: false unpackFolder: $(Build.SourcesDirectory)/artifacts - artifactName: ${{ format('libraries_test_assets_{0}_{1}_x64_Release', parameters.framework, parameters.osGroup) }} - artifactFileName: ${{ format('libraries_test_assets_{0}_{1}_x64_Release', parameters.framework, parameters.osGroup) }}$(archiveExtension) + artifactName: ${{ format('libraries_test_assets_{0}_{1}_x64_{2}', parameters.framework, parameters.osGroup, parameters.dependsOnBuildConfiguration) }} + artifactFileName: ${{ format('libraries_test_assets_{0}_{1}_x64_{2}', parameters.framework, parameters.osGroup, parameters.dependsOnBuildConfiguration) }}$(archiveExtension) - template: /eng/pipelines/libraries/helix.yml parameters: From 592dc9d3153aac15192c8a09e5b89cce78611fdd Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Nov 2019 10:57:06 -0800 Subject: [PATCH 10/14] Fix bug in build-job when runtests:true --- eng/pipelines/libraries/build-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index b2a270104a980f..fcb5bb3e1c3e8e 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -114,7 +114,7 @@ jobs: targetOS: ${{ parameters.osGroup }} archGroup: ${{ parameters.archType }} configuration: ${{ parameters.buildConfig }} - helixQueues: ${{ join(';', parameters.helixQueues) }} + helixQueues: ${{ join('+', parameters.helixQueues) }} msbuildScript: $(_msbuildCommand) framework: ${{ parameters.framework }} testScope: ${{ parameters.testScope }} From 029c37f58edec20383f2c18951d21ee29aa93f69 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 26 Nov 2019 17:12:36 -0800 Subject: [PATCH 11/14] Fix issue where build configurations.props are not present --- eng/pipelines/libraries/build-job.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index fcb5bb3e1c3e8e..0ae7cc6093d34b 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -97,11 +97,17 @@ jobs: inputs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/testhost + + - task: CopyFiles@2 + displayName: Prepare artifacts toolset folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/toolset + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/toolset - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin - includeRootFolder: true + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts + includeRootFolder: false archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) From ab2c0cbbd751f9ee0086ef7ac106c1f124b96d72 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Wed, 27 Nov 2019 09:42:38 -0800 Subject: [PATCH 12/14] Fix nano queues syntax --- eng/pipelines/libraries/helix-queues-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 12716b9e4c56c6..3c1ea75aec6664 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -84,12 +84,12 @@ jobs: - Windows.81.Amd64.Open - Windows.10.Amd64.ServerRS5.Open - Windows.10.Amd64.Server19H1.Open - - (Windows.Nano.1809.Amd64.Open)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-61052b7-20190723211353 + - "`(Windows.Nano.1809.Amd64.Open`)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-61052b7-20190723211353" - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}: - Windows.7.Amd64.Open - Windows.81.Amd64.Open - Windows.10.Amd64.Server19H1.ES.Open - - (Windows.Nano.1809.Amd64.Open)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-61052b7-20190723211353 + - "`(Windows.Nano.1809.Amd64.Open`)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-61052b7-20190723211353" # NETFX - ${{ if eq(parameters.jobParameters.framework, 'netfx') }}: From 4957317453a62aa60a78d6f9d98012fed50a520d Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Thu, 28 Nov 2019 00:07:14 -0800 Subject: [PATCH 13/14] Don't split jobs until we make tests Configuration agnostic --- eng/pipelines/libraries/.azure-ci.yml | 115 +++++++++----------------- 1 file changed, 39 insertions(+), 76 deletions(-) diff --git a/eng/pipelines/libraries/.azure-ci.yml b/eng/pipelines/libraries/.azure-ci.yml index 91139132c1e3c5..d0eece45480159 100644 --- a/eng/pipelines/libraries/.azure-ci.yml +++ b/eng/pipelines/libraries/.azure-ci.yml @@ -68,7 +68,6 @@ jobs: platforms: - Linux_rhel6_x64 - Windows_NT_x86 - - WebAssembly_wasm - ${{ if eq(variables['isFullMatrix'], true) }}: - OSX_x64 - Windows_NT_x64 @@ -77,6 +76,45 @@ jobs: - Linux_arm64 - Linux_musl_x64 - Linux_musl_arm64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + framework: netcoreapp + ${{ if eq(variables['isOfficialBuild'], false) }}: + runTests: true + testScope: innerloop + + - ${{ if eq(variables['isFullMatrix'], false) }}: + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Debug + platforms: + - Windows_NT_x64 + - OSX_x64 + - Linux_x64 + - Linux_musl_x64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + isFullMatrix: ${{ variables['isFullMatrix'] }} + testScope: innerloop + framework: netcoreapp + runTests: true + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + ${{ if eq(variables['isFullMatrix'], false) }}: + buildConfig: Debug + ${{ if eq(variables['isFullMatrix'], true) }}: + buildConfig: Release + platforms: + - WebAssembly_wasm + - ${{ if eq(variables['isFullMatrix'], false) }}: + - Linux_arm + - Linux_arm64 + - Linux_musl_arm64 jobParameters: isOfficialBuild: ${{ variables['isOfficialBuild'] }} framework: netcoreapp @@ -112,78 +150,3 @@ jobs: framework: allConfigurations ${{ if eq(variables['isOfficialBuild'], false) }}: runTests: true - - - ${{ if eq(variables['isFullMatrix'], false) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Debug - platforms: - - Windows_NT_x64 - - OSX_x64 - - Linux_x64 - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 - - Linux_musl_arm64 - jobParameters: - isOfficialBuild: ${{ variables['isOfficialBuild'] }} - framework: netcoreapp - - - ${{ if eq(variables['isOfficialBuild'], false) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml - ${{ if eq(variables['isFullMatrix'], false) }}: - buildConfig: Debug - ${{ if eq(variables['isFullMatrix'], true) }}: - buildConfig: Release - platforms: - - Windows_NT_x64 - - OSX_x64 - - Linux_x64 - jobParameters: - isOfficialBuild: ${{ variables['isOfficialBuild'] }} - testScope: innerloop - framework: netcoreapp - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - Linux_rhel6_x64 - - Windows_NT_x86 - - ${{ if eq(variables['isFullMatrix'], true) }}: - - OSX_x64 - - Windows_NT_x64 - - Linux_x64 - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 - - Linux_musl_arm64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - jobParameters: - isOfficialBuild: ${{ variables['isOfficialBuild'] }} - isFullMatrix: ${{ variables['isFullMatrix'] }} - testScope: innerloop - framework: netcoreapp - ${{ if eq(variables['isFullMatrix'], true) }}: - dependsOnBuildConfiguration: Release - - - ${{ if eq(variables['isFullMatrix'], false) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Debug - platforms: - - Windows_NT_x64 - - OSX_x64 - - Linux_x64 - - Linux_musl_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - jobParameters: - isOfficialBuild: ${{ variables['isOfficialBuild'] }} - isFullMatrix: ${{ variables['isFullMatrix'] }} - testScope: innerloop - framework: netcoreapp From f985105abdd80d7e450e2dc05ad15e5a55fc32fa Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Thu, 28 Nov 2019 00:50:35 -0800 Subject: [PATCH 14/14] Include missing parameter --- eng/pipelines/libraries/.azure-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/libraries/.azure-ci.yml b/eng/pipelines/libraries/.azure-ci.yml index d0eece45480159..64b7e8919b940d 100644 --- a/eng/pipelines/libraries/.azure-ci.yml +++ b/eng/pipelines/libraries/.azure-ci.yml @@ -79,6 +79,7 @@ jobs: helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: isOfficialBuild: ${{ variables['isOfficialBuild'] }} + isFullMatrix: ${{ variables['isFullMatrix'] }} framework: netcoreapp ${{ if eq(variables['isOfficialBuild'], false) }}: runTests: true