Skip to content

Commit 49f3ff3

Browse files
michaelgsharpmichaelgsharp
committed
Windows CI working (dotnet#5477)
* ci testing changes * comments from pr
1 parent be6a0a8 commit 49f3ff3

File tree

4 files changed

+72
-76
lines changed

4 files changed

+72
-76
lines changed

‎.vsts-dotnet-ci.yml‎

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,47 +6,47 @@ resources:
66
containers:
77
- container: CentosContainer
88
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
9-
9+
1010
- container: UbuntuContainer
1111
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8
1212

1313
jobs:
14-
- template: /build/ci/job-template.yml
15-
parameters:
16-
name: Centos_x64_NetCoreApp31
17-
buildScript: ./build.sh
18-
container: CentosContainer
19-
customMatrixes:
20-
Debug_Build:
21-
_configuration: Debug-netcoreapp3_1
22-
_config_short: DI
23-
_includeBenchmarkData: false
24-
_targetFramework: netcoreapp3.1
25-
Release_Build:
26-
_configuration: Release-netcoreapp3_1
27-
_config_short: RI
28-
_includeBenchmarkData: true
29-
_targetFramework: netcoreapp3.1
30-
innerLoop: true
31-
pool:
32-
name: Hosted Ubuntu 1604
14+
# - template: /build/ci/job-template.yml
15+
# parameters:
16+
# name: Centos_x64_NetCoreApp31
17+
# buildScript: ./build.sh
18+
# container: CentosContainer
19+
# customMatrixes:
20+
# Debug_Build:
21+
# _configuration: Debug-netcoreapp3_1
22+
# _config_short: DI
23+
# _includeBenchmarkData: false
24+
# _targetFramework: netcoreapp3.1
25+
# Release_Build:
26+
# _configuration: Release-netcoreapp3_1
27+
# _config_short: RI
28+
# _includeBenchmarkData: true
29+
# _targetFramework: netcoreapp3.1
30+
# innerLoop: true
31+
# pool:
32+
# name: Hosted Ubuntu 1604
3333

34-
- template: /build/ci/job-template.yml
35-
parameters:
36-
name: Ubuntu_x64_NetCoreApp21
37-
buildScript: ./build.sh
38-
container: UbuntuContainer
39-
innerLoop: true
40-
pool:
41-
name: Hosted Ubuntu 1604
34+
# - template: /build/ci/job-template.yml
35+
# parameters:
36+
# name: Ubuntu_x64_NetCoreApp21
37+
# buildScript: ./build.sh
38+
# container: UbuntuContainer
39+
# innerLoop: true
40+
# pool:
41+
# name: Hosted Ubuntu 1604
4242

43-
- template: /build/ci/job-template.yml
44-
parameters:
45-
name: MacOS_x64_NetCoreApp21
46-
buildScript: ./build.sh
47-
innerLoop: true
48-
pool:
49-
name: Hosted macOS
43+
# - template: /build/ci/job-template.yml
44+
# parameters:
45+
# name: MacOS_x64_NetCoreApp21
46+
# buildScript: ./build.sh
47+
# innerLoop: true
48+
# pool:
49+
# name: Hosted macOS
5050

5151
- template: /build/ci/job-template.yml
5252
parameters:

‎build.cmd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
@echo off
2-
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -pack -warnAsError 0 %*"
2+
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -warnAsError 0 %*"
33
exit /b %ErrorLevel%

‎build/ci/job-template.yml‎

Lines changed: 28 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#TODO: Need to update build documentation.
12
parameters:
23
name: ''
34
architecture: x64
@@ -21,7 +22,7 @@ jobs:
2122
timeoutInMinutes: 120
2223
cancelTimeoutInMinutes: 10
2324
variables:
24-
dotnetPath: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet
25+
dotnetPath: $(Build.SourcesDirectory)/.dotnet/dotnet
2526
nugetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
2627
nightlyBuildProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj
2728
nightlyBuildRunPath: $(Build.SourcesDirectory)/bin/AnyCPU.$(_configuration)/Microsoft.ML.NightlyBuild.Tests/$(_targetFramework)
@@ -43,7 +44,7 @@ jobs:
4344
_targetFramework: netcoreapp2.1
4445
${{ if ne(parameters.customMatrixes, '') }}:
4546
${{ insert }}: ${{ parameters.customMatrixes }}
46-
47+
4748
pool: ${{ parameters.pool }}
4849
${{ if ne(parameters.container, '') }}:
4950
container: ${{ parameters.container }}
@@ -55,7 +56,7 @@ jobs:
5556
- ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
5657
- bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
5758
displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
58-
- script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }}
59+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
5960
displayName: Build
6061
- ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
6162
- task: Bash@3
@@ -70,11 +71,12 @@ jobs:
7071
script: cd packages;find . -type d -path "*/runtimes/osx-*" -exec rm -rv {} +;find . -type d -path "*/runtimes/win-*" -exec rm -rv {} +;cd ..
7172
displayName: Clean up non-Linux runtime folders of NuGet Packages to save disk space
7273
- ${{ if eq(parameters.buildScript, 'build.cmd') }}:
73-
- task: PowerShell@2
74-
inputs:
75-
targetType: inline
76-
script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
77-
displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
74+
# TODO: We need to do this. THe packages are restored in a different folder so we need find the correct path.
75+
# - task: PowerShell@2
76+
# inputs:
77+
# targetType: inline
78+
# script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
79+
# displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
7880
- script: dir /s "bin"
7981
displayName: show bin folder disk usage
8082
- ${{ if eq(parameters.nightlyBuild, 'true') }}:
@@ -89,27 +91,22 @@ jobs:
8991
Get-ChildItem -Path '.\bin\AnyCPU.*' -Recurse |
9092
Select -ExpandProperty FullName |
9193
Where {$_ -notlike '*\Microsoft.ML.NightlyBuild.Tests*'} |
92-
sort length -Descending |
93-
Remove-Item -force
94+
sort length -Descending |
95+
Remove-Item -force
9496
Write-Output "Done cleaning up usless project..."
9597
displayName: Clean up useless project
96-
- script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" /p:Configuration=$(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
98+
- script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
9799
displayName: Build Nightly-Build Project with latest package versions
98100
- script: ${{ parameters.buildScript }} -$(_configuration) -runnightlybuildtests
99101
displayName: Run Nightly Build Tests
100102
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
101-
- script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
102-
displayName: Download Benchmark Data
103-
timeoutInMinutes: 10
104-
- script: ${{ parameters.buildScript }} -- /t:DownloadTensorflowMetaFiles /p:IncludeTensorflowMetaFile=true
105-
displayName: Download Tensorflow Meta File
106-
timeoutInMinutes: 20
107103
- ${{ if eq(parameters.innerLoop, 'false') }}:
108104
- ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}:
109-
- script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
105+
# TODO: Code coverage needs to be fixed.
106+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }}
110107
displayName: Run All Tests.
111108
- ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}:
112-
- script: ${{ parameters.buildScript }} -$(_configuration) -runSpecificTests -coverage=${{ parameters.codeCoverage }}
109+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -ci /p:TestRunnerAdditionalArguments=-trait%20Category=RunSpecificTest #-coverage=${{ parameters.codeCoverage }}
113110
displayName: Run Specific Tests.
114111
- ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}:
115112
- task: VSTest@2
@@ -119,10 +116,10 @@ jobs:
119116
testAssemblyVer2: |
120117
**\*test.dll
121118
**\*tests.dll
122-
!**\obj\**
119+
!**\obj\**
123120
runSettingsFile: $(Build.SourcesDirectory)/tools-local/vstest.runsettings
124121
searchFolder: '$(System.DefaultWorkingDirectory)'
125-
vstestLocationMethod: 'version'
122+
vstestLocationMethod: 'version'
126123
vsTestVersion: 'latest'
127124
runInParallel: False
128125
runTestsInIsolation: True
@@ -133,17 +130,17 @@ jobs:
133130
collectDumpOn: onAbortOnly
134131
publishRunAttachments: true
135132
- ${{ if eq(parameters.innerLoop, 'true') }}:
136-
- script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }}
133+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }}
137134
displayName: Run CI Tests.
138-
- script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild -restore build/Codecoverage.proj
135+
- script: $(dotnetPath) msbuild -restore build/Codecoverage.proj
139136
displayName: Upload coverage to codecov.io
140137
condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
141138
- task: PublishTestResults@2
142139
displayName: Publish Test Results
143140
condition: succeededOrFailed()
144141
inputs:
145142
testRunner: 'vSTest'
146-
searchFolder: '$(System.DefaultWorkingDirectory)/bin'
143+
searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults'
147144
testResultsFiles: '**/*.trx'
148145
testRunTitle: Machinelearning_Tests_${{ parameters.name }}_$(_configuration)_$(Build.BuildNumber)
149146
configuration: $(_configuration)
@@ -152,17 +149,16 @@ jobs:
152149
displayName: Stage build logs
153150
condition: not(succeeded())
154151
inputs:
155-
sourceFolder: $(Build.SourcesDirectory)
156-
contents: '?(msbuild.*|binclash.log|init-tools.log)'
157-
targetFolder: $(Build.ArtifactStagingDirectory)
152+
sourceFolder: $(Build.SourcesDirectory)/artifacts/log/
153+
contents: '**'
154+
targetFolder: $(Build.ArtifactStagingDirectory)artifacts/log/
158155
- task: CopyFiles@2
159156
displayName: Stage test output
160157
condition: not(succeeded())
161158
inputs:
162-
sourceFolder: $(Build.SourcesDirectory)/bin
159+
sourceFolder: $(Build.SourcesDirectory)/artifacts/TestResults
163160
contents: |
164-
**/TestOutput/**/*
165-
**/*.trx
161+
**
166162
targetFolder: $(Build.ArtifactStagingDirectory)
167163
- task: CopyFiles@2
168164
displayName: Stage process dump and pdb if any
@@ -172,7 +168,7 @@ jobs:
172168
contents: |
173169
*.dmp
174170
CrashDumps/*.dmp
175-
bin/**/*.pdb
171+
artifacts/bin/**/*.pdb
176172
targetFolder: $(Build.ArtifactStagingDirectory)
177173
- task: PublishBuildArtifacts@1
178174
displayName: Publish build and test logs
@@ -182,5 +178,5 @@ jobs:
182178
artifactName: ${{ parameters.name }} $(_config_short)
183179
artifactType: container
184180
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
185-
- script: ${{ parameters.buildScript }} -buildPackages
181+
- script: ${{ parameters.buildScript }} -pack -ci
186182
displayName: Build Packages

‎build/vsts-ci.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
################################################################################
2-
# ML.NET's official, signed build
2+
# ML.NET's official, signed build
33
################################################################################
44

55
resources:
@@ -73,7 +73,7 @@ phases:
7373
_TeamName: DotNetCore
7474
queue:
7575
name: DotNetCore-Build
76-
demands:
76+
demands:
7777
- agent.os -equals Windows_NT
7878
steps:
7979

@@ -91,7 +91,7 @@ phases:
9191
# Only build native assets to avoid conflicts.
9292
- script: ./build.cmd -buildNative -$(BuildConfig) -buildArch=x86 -skipRIDAgnosticAssets
9393
displayName: Build
94-
94+
9595
- task: MSBuild@1
9696
displayName: Sign Windows_x86 Binaries
9797
inputs:
@@ -125,7 +125,7 @@ phases:
125125
_TeamName: DotNetCore
126126
queue:
127127
name: DotNetCore-Build
128-
demands:
128+
demands:
129129
- agent.os -equals Windows_NT
130130
steps:
131131

@@ -140,7 +140,7 @@ phases:
140140
continueOnError: false
141141
condition: and(succeeded(), in(variables._SignType, 'real', 'test'))
142142

143-
# Build both native and managed assets.
143+
# Build both native and managed assets.
144144
- script: ./build.cmd -$(BuildConfig)
145145
displayName: Build
146146

@@ -150,7 +150,7 @@ phases:
150150
verbosity: 'Verbose'
151151
alertWarningLevel: 'High'
152152

153-
153+
154154
- task: MSBuild@1
155155
displayName: Sign Windows_x64 Binaries
156156
inputs:
@@ -192,7 +192,7 @@ phases:
192192
_MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
193193
queue:
194194
name: DotNetCore-Build
195-
demands:
195+
demands:
196196
- agent.os -equals Windows_NT
197197
steps:
198198

@@ -221,7 +221,7 @@ phases:
221221
inputs:
222222
sourceFolder: $(Build.SourcesDirectory)/bin/obj/packages/PackageAssets
223223
targetFolder: $(Build.SourcesDirectory)/bin/obj/packages
224-
224+
225225
- script: ./build.cmd -buildPackages
226226
displayName: Create Packages
227227

0 commit comments

Comments
 (0)