Skip to content

Commit ef33d76

Browse files
safernsafern
authored
Use platform-matrix and split jobs by steps for libraries pipelines (#274)
1 parent 89f3ff1 commit ef33d76

19 files changed

+667
-944
lines changed

‎eng/pipelines/common/platform-matrix.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,22 @@ jobs:
159159
managedTestBuildOsGroup: OSX
160160
${{ insert }}: ${{ parameters.jobParameters }}
161161

162+
- ${{ if containsValue(parameters.platforms, 'WebAssembly_wasm') }}:
163+
- template: xplat-setup.yml
164+
parameters:
165+
jobTemplate: ${{ parameters.jobTemplate }}
166+
helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
167+
osGroup: WebAssembly
168+
archType: wasm
169+
platform: WebAssembly_wasm
170+
container:
171+
image: ubuntu-16.04-a50a721-20191023143845
172+
registry: mcr
173+
jobParameters:
174+
stagedBuild: ${{ parameters.stagedBuild }}
175+
buildConfig: ${{ parameters.buildConfig }}
176+
${{ insert }}: ${{ parameters.jobParameters }}
177+
162178
# FreeBSD
163179

164180
# FreeBSD machines are currently offline. Re-enable in the official build when

‎eng/pipelines/libraries/.azure-ci.yml‎

Lines changed: 90 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ trigger:
2222
- src/coreclr/*
2323
- eng/pipelines/coreclr/*
2424
- eng/pipelines/installer/*
25-
- eng/pipelines/common/*
2625

2726
pr:
2827
branches:
@@ -43,30 +42,6 @@ pr:
4342
- src/coreclr/*
4443
- eng/pipelines/coreclr/*
4544
- eng/pipelines/installer/*
46-
- eng/pipelines/common/*
47-
48-
resources:
49-
containers:
50-
- container: centos7_container
51-
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df28-20191023143754
52-
53-
- container: centos6_container
54-
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-f39df28-20191023143802
55-
56-
- container: alpine_39_container
57-
image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-WithNode-0fc54a3-20190918214015
58-
59-
- container: alpine_37_arm64_container
60-
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-406629a-20191023143847
61-
62-
- container: ubuntu_1604_arm64_cross_container
63-
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-cfdd435-20191023143847
64-
65-
- container: ubuntu_1604_arm_cross_container
66-
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-14.04-23cacb0-20191023143847
67-
68-
- container: ubuntu_1604
69-
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-a50a721-20191023143845
7045

7146
variables:
7247
- template: variables.yml
@@ -83,55 +58,96 @@ variables:
8358
- name: _DotNetValidationArtifactsCategory
8459
value: .NETCore
8560

86-
stages:
87-
# Windows legs
88-
- template: ${{ variables['pipelinesPath'] }}/windows.yml
89-
parameters:
90-
isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}
91-
fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }}
92-
93-
# Linux legs
94-
- template: ${{ variables['pipelinesPath'] }}/linux.yml
95-
parameters:
96-
isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}
97-
fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }}
61+
jobs:
62+
- template: /eng/pipelines/common/checkout-job.yml
9863

99-
# MacOS legs
100-
- template: ${{ variables['pipelinesPath'] }}/macos.yml
64+
- template: /eng/pipelines/common/platform-matrix.yml
10165
parameters:
102-
isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}
103-
fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }}
104-
105-
# FreeBSD leg is only for official builds
106-
# - template: ${{ variables['pipelinesPath'] }}/freebsd.yml
107-
# parameters:
108-
# isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}
109-
# fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }}
110-
111-
# Publish and validation steps. Only run in official builds
112-
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}:
113-
- template: ${{ variables['pipelinesPath'] }}/pre-publish.yml
66+
jobTemplate: /eng/pipelines/libraries/build-job.yml
67+
buildConfig: Release
68+
platforms:
69+
- Linux_rhel6_x64
70+
- Windows_NT_x86
71+
- ${{ if eq(variables['isFullMatrix'], true) }}:
72+
- OSX_x64
73+
- Windows_NT_x64
74+
- Linux_x64
75+
- Linux_arm
76+
- Linux_arm64
77+
- Linux_musl_x64
78+
- Linux_musl_arm64
79+
helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
80+
jobParameters:
81+
isOfficialBuild: ${{ variables['isOfficialBuild'] }}
82+
isFullMatrix: ${{ variables['isFullMatrix'] }}
83+
framework: netcoreapp
84+
${{ if eq(variables['isOfficialBuild'], false) }}:
85+
runTests: true
86+
testScope: innerloop
87+
88+
- ${{ if eq(variables['isFullMatrix'], false) }}:
89+
- template: /eng/pipelines/common/platform-matrix.yml
11490
parameters:
115-
dependsOn:
116-
- WindowsStage
117-
- LinuxStage
118-
- MacOSStage
119-
# - FreeBSD
120-
121-
- template: eng\common\templates\post-build\post-build.yml
122-
parameters:
123-
validateDependsOn:
124-
- PrePublish
125-
enableSymbolValidation: false # https://github.com/dotnet/arcade/issues/2871
126-
SDLValidationParameters:
127-
enable: true
128-
params: ' -SourceToolsList @("policheck","credscan")
129-
-TsaInstanceURL "$(TsaInstanceURL)"
130-
-TsaProjectName "$(TsaProjectName)"
131-
-TsaNotificationEmail "$(TsaNotificationEmail)"
132-
-TsaCodebaseAdmin "$(TsaCodebaseAdmin)"
133-
-TsaBugAreaPath "$(TsaBugAreaPath)"
134-
-TsaIterationPath "$(TsaIterationPath)"
135-
-TsaRepositoryName "CoreFX"
136-
-TsaCodebaseName "CoreFX"
137-
-TsaPublish $True'
91+
jobTemplate: /eng/pipelines/libraries/build-job.yml
92+
buildConfig: Debug
93+
platforms:
94+
- Windows_NT_x64
95+
- OSX_x64
96+
- Linux_x64
97+
- Linux_musl_x64
98+
helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
99+
jobParameters:
100+
isOfficialBuild: ${{ variables['isOfficialBuild'] }}
101+
isFullMatrix: ${{ variables['isFullMatrix'] }}
102+
testScope: innerloop
103+
framework: netcoreapp
104+
runTests: true
105+
106+
- template: /eng/pipelines/common/platform-matrix.yml
107+
parameters:
108+
jobTemplate: /eng/pipelines/libraries/build-job.yml
109+
${{ if eq(variables['isFullMatrix'], false) }}:
110+
buildConfig: Debug
111+
${{ if eq(variables['isFullMatrix'], true) }}:
112+
buildConfig: Release
113+
platforms:
114+
- WebAssembly_wasm
115+
- ${{ if eq(variables['isFullMatrix'], false) }}:
116+
- Linux_arm
117+
- Linux_arm64
118+
- Linux_musl_arm64
119+
jobParameters:
120+
isOfficialBuild: ${{ variables['isOfficialBuild'] }}
121+
framework: netcoreapp
122+
123+
- template: /eng/pipelines/common/platform-matrix.yml
124+
parameters:
125+
jobTemplate: /eng/pipelines/libraries/build-job.yml
126+
buildConfig: Release
127+
platforms:
128+
- Windows_NT_x86
129+
- ${{ if eq(variables['isFullMatrix'], true) }}:
130+
- Windows_NT_x64
131+
helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
132+
jobParameters:
133+
isOfficialBuild: ${{ variables['isOfficialBuild'] }}
134+
framework: netfx
135+
${{ if eq(variables['isOfficialBuild'], false) }}:
136+
runTests: true
137+
testScope: innerloop
138+
139+
- template: /eng/pipelines/common/platform-matrix.yml
140+
parameters:
141+
jobTemplate: /eng/pipelines/libraries/build-job.yml
142+
${{ if eq(variables['isFullMatrix'], false) }}:
143+
buildConfig: Debug
144+
${{ if eq(variables['isFullMatrix'], true) }}:
145+
buildConfig: Release
146+
platforms:
147+
- Windows_NT_x64
148+
helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
149+
jobParameters:
150+
isOfficialBuild: ${{ variables['isOfficialBuild'] }}
151+
framework: allConfigurations
152+
${{ if eq(variables['isOfficialBuild'], false) }}:
153+
runTests: true

‎eng/pipelines/libraries/base-job.yml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
parameters:
2+
buildConfig: ''
3+
osGroup: ''
4+
archType: ''
5+
osSubgroup: ''
6+
framework: ''
7+
isOfficialBuild: false
8+
timeoutInMinutes: 150
9+
container: ''
10+
steps: []
11+
dependsOn: []
12+
variables: {}
13+
name: ''
14+
displayName: ''
15+
testScope: ''
16+
pool: ''
17+
stepName: ''
18+
19+
jobs:
20+
- template: /eng/common/templates/job/job.yml
21+
parameters:
22+
name: ${{ format('libraries_{0}_{1}_{2}{3}_{4}_{5}', parameters.stepName, parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
23+
${{ if eq(parameters.framework, 'netcoreapp') }}:
24+
displayName: ${{ format('{0} {1}{2} {3} {4}', parameters.stepName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
25+
${{ if ne(parameters.framework, 'netcoreapp') }}:
26+
displayName: ${{ format('{0} {1} {2} {3} {4}', parameters.stepName, parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }}
27+
28+
enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed
29+
container: ${{ parameters.container }}
30+
helixRepo: dotnet/runtime
31+
pool: ${{ parameters.pool }}
32+
variables:
33+
- _BuildConfig: ${{ parameters.buildConfig }}
34+
- _msbuildCommonParameters: ''
35+
- _stripSymbolsArg: ''
36+
- _runtimeOSArg: ''
37+
- _portableArg: ''
38+
- _finalFrameworkArg: -framework ${{ parameters.framework }}
39+
- _buildScript: $(buildScriptFileName)$(scriptExt)
40+
- _warnAsErrorArg: ''
41+
- _testScopeArg: ''
42+
43+
- ${{ if ne(parameters.testScope, '') }}:
44+
- _testScopeArg: -testscope ${{ parameters.testScope }}
45+
46+
- ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_rhel6')) }}:
47+
- _runtimeOSArg: /p:RuntimeOS=rhel.6
48+
- _portableArg: /p:PortableBuild=false
49+
50+
- ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_musl')) }}:
51+
- _runtimeOSArg: /p:RuntimeOS=linux-musl
52+
53+
- ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, ''), eq(parameters.archType, 'arm')) }}:
54+
- _runtimeOSArg: /p:RuntimeOS=ubuntu.16.04
55+
56+
- ${{ if and(or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'WebAssembly')), ne(parameters.container, '')) }}:
57+
- _warnAsErrorArg: '-warnAsError false'
58+
59+
- ${{ if eq(parameters.osGroup, 'WebAssembly') }}:
60+
- _runtimeOSArg: -os ${{ parameters.osGroup }}
61+
62+
- ${{ if eq(parameters.framework, 'allConfigurations' ) }}:
63+
- _finalFrameworkArg: -allConfigurations
64+
65+
- ${{ if eq(parameters.isOfficialBuild, 'true') }}:
66+
- _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber)
67+
68+
# Windows variables
69+
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
70+
- _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci
71+
- _runtimeOSArg: /p:RuntimeOS=win10
72+
73+
# Non-Windows variables
74+
- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
75+
- _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci
76+
- _buildScript: ./$(buildScriptFileName)$(scriptExt)
77+
- ${{ if eq(parameters.isOfficialBuild, 'true') }}:
78+
- _stripSymbolsArg: -stripSymbols
79+
80+
- _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters)
81+
- ${{ parameters.variables }}
82+
83+
dependsOn:
84+
- checkout
85+
- ${{ parameters.dependsOn }}
86+
workspace:
87+
clean: all
88+
89+
enablePublishBuildArtifacts: true
90+
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
91+
92+
steps:
93+
- checkout: none
94+
clean: true
95+
96+
- download: current
97+
artifact: Checkout_bundle
98+
displayName: Download Checkout.bundle
99+
100+
- script: |
101+
git clone $(Pipeline.Workspace)/Checkout_bundle/Checkout.bundle .
102+
git remote set-url origin $(Build.Repository.Uri)
103+
displayName: Clone the repository from Checkout.bundle
104+
105+
- ${{ parameters.steps }}

0 commit comments

Comments
 (0)