Skip to content

Commit a2f405b

Browse files
michaelgsharpharishsk
michaelgsharp
authored and
harishsk
committed
Windows CI working (#5477)
* ci testing changes * comments from pr
1 parent bad973c commit a2f405b

File tree

4 files changed

+72
-76
lines changed

4 files changed

+72
-76
lines changed

‎.vsts-dotnet-ci.yml‎

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,47 +6,47 @@ resources:
66
containers:
77
- container: CentosContainer
88
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
9-
9+
1010
- container: UbuntuContainer
1111
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8
1212

1313
jobs:
14-
- template: /build/ci/job-template.yml
15-
parameters:
16-
name: Centos_x64_NetCoreApp31
17-
buildScript: ./build.sh
18-
container: CentosContainer
19-
customMatrixes:
20-
Debug_Build:
21-
_configuration: Debug-netcoreapp3_1
22-
_config_short: DI
23-
_includeBenchmarkData: false
24-
_targetFramework: netcoreapp3.1
25-
Release_Build:
26-
_configuration: Release-netcoreapp3_1
27-
_config_short: RI
28-
_includeBenchmarkData: true
29-
_targetFramework: netcoreapp3.1
30-
innerLoop: true
31-
pool:
32-
name: Hosted Ubuntu 1604
14+
# - template: /build/ci/job-template.yml
15+
# parameters:
16+
# name: Centos_x64_NetCoreApp31
17+
# buildScript: ./build.sh
18+
# container: CentosContainer
19+
# customMatrixes:
20+
# Debug_Build:
21+
# _configuration: Debug-netcoreapp3_1
22+
# _config_short: DI
23+
# _includeBenchmarkData: false
24+
# _targetFramework: netcoreapp3.1
25+
# Release_Build:
26+
# _configuration: Release-netcoreapp3_1
27+
# _config_short: RI
28+
# _includeBenchmarkData: true
29+
# _targetFramework: netcoreapp3.1
30+
# innerLoop: true
31+
# pool:
32+
# name: Hosted Ubuntu 1604
3333

34-
- template: /build/ci/job-template.yml
35-
parameters:
36-
name: Ubuntu_x64_NetCoreApp21
37-
buildScript: ./build.sh
38-
container: UbuntuContainer
39-
innerLoop: true
40-
pool:
41-
name: Hosted Ubuntu 1604
34+
# - template: /build/ci/job-template.yml
35+
# parameters:
36+
# name: Ubuntu_x64_NetCoreApp21
37+
# buildScript: ./build.sh
38+
# container: UbuntuContainer
39+
# innerLoop: true
40+
# pool:
41+
# name: Hosted Ubuntu 1604
4242

43-
- template: /build/ci/job-template.yml
44-
parameters:
45-
name: MacOS_x64_NetCoreApp21
46-
buildScript: ./build.sh
47-
innerLoop: true
48-
pool:
49-
name: Hosted macOS
43+
# - template: /build/ci/job-template.yml
44+
# parameters:
45+
# name: MacOS_x64_NetCoreApp21
46+
# buildScript: ./build.sh
47+
# innerLoop: true
48+
# pool:
49+
# name: Hosted macOS
5050

5151
- template: /build/ci/job-template.yml
5252
parameters:

‎build.cmd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
@echo off
2-
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -pack -warnAsError 0 %*"
2+
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -warnAsError 0 %*"
33
exit /b %ErrorLevel%

‎build/ci/job-template.yml‎

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#TODO: Need to update build documentation.
12
parameters:
23
name: ''
34
architecture: x64
@@ -21,7 +22,7 @@ jobs:
2122
timeoutInMinutes: 120
2223
cancelTimeoutInMinutes: 10
2324
variables:
24-
dotnetPath: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet
25+
dotnetPath: $(Build.SourcesDirectory)/.dotnet/dotnet
2526
nugetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
2627
nightlyBuildProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj
2728
nightlyBuildRunPath: $(Build.SourcesDirectory)/bin/AnyCPU.$(_configuration)/Microsoft.ML.NightlyBuild.Tests/$(_targetFramework)
@@ -43,7 +44,7 @@ jobs:
4344
_targetFramework: netcoreapp2.1
4445
${{ if ne(parameters.customMatrixes, '') }}:
4546
${{ insert }}: ${{ parameters.customMatrixes }}
46-
47+
4748
pool: ${{ parameters.pool }}
4849
${{ if ne(parameters.container, '') }}:
4950
container: ${{ parameters.container }}
@@ -63,7 +64,7 @@ jobs:
6364
- ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
6465
- bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
6566
displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
66-
- script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }}
67+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
6768
displayName: Build
6869
- ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
6970
- task: Bash@3
@@ -78,11 +79,12 @@ jobs:
7879
script: cd packages;find . -type d -path "*/runtimes/osx-*" -exec rm -rv {} +;find . -type d -path "*/runtimes/win-*" -exec rm -rv {} +;cd ..
7980
displayName: Clean up non-Linux runtime folders of NuGet Packages to save disk space
8081
- ${{ if eq(parameters.buildScript, 'build.cmd') }}:
81-
- task: PowerShell@2
82-
inputs:
83-
targetType: inline
84-
script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
85-
displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
82+
# TODO: We need to do this. THe packages are restored in a different folder so we need find the correct path.
83+
# - task: PowerShell@2
84+
# inputs:
85+
# targetType: inline
86+
# script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
87+
# displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
8688
- script: dir /s "bin"
8789
displayName: show bin folder disk usage
8890
- ${{ if eq(parameters.nightlyBuild, 'true') }}:
@@ -97,27 +99,22 @@ jobs:
9799
Get-ChildItem -Path '.\bin\AnyCPU.*' -Recurse |
98100
Select -ExpandProperty FullName |
99101
Where {$_ -notlike '*\Microsoft.ML.NightlyBuild.Tests*'} |
100-
sort length -Descending |
101-
Remove-Item -force
102+
sort length -Descending |
103+
Remove-Item -force
102104
Write-Output "Done cleaning up usless project..."
103105
displayName: Clean up useless project
104-
- script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" /p:Configuration=$(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
106+
- script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
105107
displayName: Build Nightly-Build Project with latest package versions
106108
- script: ${{ parameters.buildScript }} -$(_configuration) -runnightlybuildtests
107109
displayName: Run Nightly Build Tests
108110
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
109-
- script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
110-
displayName: Download Benchmark Data
111-
timeoutInMinutes: 10
112-
- script: ${{ parameters.buildScript }} -- /t:DownloadTensorflowMetaFiles /p:IncludeTensorflowMetaFile=true
113-
displayName: Download Tensorflow Meta File
114-
timeoutInMinutes: 20
115111
- ${{ if eq(parameters.innerLoop, 'false') }}:
116112
- ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}:
117-
- script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
113+
# TODO: Code coverage needs to be fixed.
114+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }}
118115
displayName: Run All Tests.
119116
- ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}:
120-
- script: ${{ parameters.buildScript }} -$(_configuration) -runSpecificTests -coverage=${{ parameters.codeCoverage }}
117+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -ci /p:TestRunnerAdditionalArguments=-trait%20Category=RunSpecificTest #-coverage=${{ parameters.codeCoverage }}
121118
displayName: Run Specific Tests.
122119
- ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}:
123120
- task: VSTest@2
@@ -127,10 +124,10 @@ jobs:
127124
testAssemblyVer2: |
128125
**\*test.dll
129126
**\*tests.dll
130-
!**\obj\**
127+
!**\obj\**
131128
runSettingsFile: $(Build.SourcesDirectory)/tools-local/vstest.runsettings
132129
searchFolder: '$(System.DefaultWorkingDirectory)'
133-
vstestLocationMethod: 'version'
130+
vstestLocationMethod: 'version'
134131
vsTestVersion: 'latest'
135132
runInParallel: False
136133
runTestsInIsolation: True
@@ -141,17 +138,17 @@ jobs:
141138
collectDumpOn: onAbortOnly
142139
publishRunAttachments: true
143140
- ${{ if eq(parameters.innerLoop, 'true') }}:
144-
- script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }}
141+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }}
145142
displayName: Run CI Tests.
146-
- script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild -restore build/Codecoverage.proj
143+
- script: $(dotnetPath) msbuild -restore build/Codecoverage.proj
147144
displayName: Upload coverage to codecov.io
148145
condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
149146
- task: PublishTestResults@2
150147
displayName: Publish Test Results
151148
condition: succeededOrFailed()
152149
inputs:
153150
testRunner: 'vSTest'
154-
searchFolder: '$(System.DefaultWorkingDirectory)/bin'
151+
searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults'
155152
testResultsFiles: '**/*.trx'
156153
testRunTitle: Machinelearning_Tests_${{ parameters.name }}_$(_configuration)_$(Build.BuildNumber)
157154
configuration: $(_configuration)
@@ -160,17 +157,16 @@ jobs:
160157
displayName: Stage build logs
161158
condition: not(succeeded())
162159
inputs:
163-
sourceFolder: $(Build.SourcesDirectory)
164-
contents: '?(msbuild.*|binclash.log|init-tools.log)'
165-
targetFolder: $(Build.ArtifactStagingDirectory)
160+
sourceFolder: $(Build.SourcesDirectory)/artifacts/log/
161+
contents: '**'
162+
targetFolder: $(Build.ArtifactStagingDirectory)artifacts/log/
166163
- task: CopyFiles@2
167164
displayName: Stage test output
168165
condition: not(succeeded())
169166
inputs:
170-
sourceFolder: $(Build.SourcesDirectory)/bin
167+
sourceFolder: $(Build.SourcesDirectory)/artifacts/TestResults
171168
contents: |
172-
**/TestOutput/**/*
173-
**/*.trx
169+
**
174170
targetFolder: $(Build.ArtifactStagingDirectory)
175171
- task: CopyFiles@2
176172
displayName: Stage process dump and pdb if any
@@ -180,7 +176,7 @@ jobs:
180176
contents: |
181177
*.dmp
182178
CrashDumps/*.dmp
183-
bin/**/*.pdb
179+
artifacts/bin/**/*.pdb
184180
targetFolder: $(Build.ArtifactStagingDirectory)
185181
- task: PublishBuildArtifacts@1
186182
displayName: Publish build and test logs
@@ -190,5 +186,5 @@ jobs:
190186
artifactName: ${{ parameters.name }} $(_config_short)
191187
artifactType: container
192188
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
193-
- script: ${{ parameters.buildScript }} -buildPackages
189+
- script: ${{ parameters.buildScript }} -pack -ci
194190
displayName: Build Packages

‎build/vsts-ci.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
################################################################################
2-
# ML.NET's official, signed build
2+
# ML.NET's official, signed build
33
################################################################################
44

55
resources:
@@ -80,7 +80,7 @@ phases:
8080
_TeamName: DotNetCore
8181
queue:
8282
name: DotNetCore-Build
83-
demands:
83+
demands:
8484
- agent.os -equals Windows_NT
8585
steps:
8686

@@ -98,7 +98,7 @@ phases:
9898
# Only build native assets to avoid conflicts.
9999
- script: ./build.cmd -buildNative -$(BuildConfig) -buildArch=x86 -skipRIDAgnosticAssets
100100
displayName: Build
101-
101+
102102
- task: MSBuild@1
103103
displayName: Sign Windows_x86 Binaries
104104
inputs:
@@ -132,7 +132,7 @@ phases:
132132
_TeamName: DotNetCore
133133
queue:
134134
name: DotNetCore-Build
135-
demands:
135+
demands:
136136
- agent.os -equals Windows_NT
137137
steps:
138138

@@ -147,7 +147,7 @@ phases:
147147
continueOnError: false
148148
condition: and(succeeded(), in(variables._SignType, 'real', 'test'))
149149

150-
# Build both native and managed assets.
150+
# Build both native and managed assets.
151151
- script: ./build.cmd -$(BuildConfig)
152152
displayName: Build
153153

@@ -157,7 +157,7 @@ phases:
157157
verbosity: 'Verbose'
158158
alertWarningLevel: 'High'
159159

160-
160+
161161
- task: MSBuild@1
162162
displayName: Sign Windows_x64 Binaries
163163
inputs:
@@ -199,7 +199,7 @@ phases:
199199
_MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
200200
queue:
201201
name: DotNetCore-Build
202-
demands:
202+
demands:
203203
- agent.os -equals Windows_NT
204204
steps:
205205

@@ -228,7 +228,7 @@ phases:
228228
inputs:
229229
sourceFolder: $(Build.SourcesDirectory)/bin/obj/packages/PackageAssets
230230
targetFolder: $(Build.SourcesDirectory)/bin/obj/packages
231-
231+
232232
- script: ./build.cmd -buildPackages
233233
displayName: Create Packages
234234

0 commit comments

Comments
 (0)