From 622efb84c6ece709e511a37d6bd95d7c8ffa2110 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 25 Jun 2019 21:17:38 -0700 Subject: [PATCH 1/6] Move from phases to jobs and use bring your own cloud pool --- .vsts-dotnet-ci.yml | 37 ++++++++-------- .../{phase-template.yml => job-template.yml} | 43 +++++++++---------- build/codecoverage-ci.yml | 2 +- 3 files changed, 41 insertions(+), 41 deletions(-) rename build/ci/{phase-template.yml => job-template.yml} (69%) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index b3668f511b..61388f280c 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -10,11 +10,12 @@ resources: - container: UbuntuContainer image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303 -phases: -- template: /build/ci/phase-template.yml +jobs: +- template: /build/ci/job-template.yml parameters: name: Centos_x64_NetCoreApp30 buildScript: ./build.sh + container: CentosContainer customMatrixes: Debug_Build: _configuration: Debug-Intrinsics @@ -24,26 +25,25 @@ phases: _configuration: Release-Intrinsics _config_short: RI _includeBenchmarkData: true - queue: + pool: name: Hosted Ubuntu 1604 - container: CentosContainer -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh - queue: + container: UbuntuContainer + pool: name: Hosted Ubuntu 1604 - container: UbuntuContainer -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - queue: + pool: name: Hosted macOS -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64_NetCoreApp30 buildScript: build.cmd @@ -56,17 +56,18 @@ phases: _configuration: Release-Intrinsics _config_short: RI _includeBenchmarkData: true - queue: - name: Hosted VS2017 + pool: + name: NetCorePublic-Pool + queue: buildpool.windows.10.amd64.vs2017 -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - queue: + pool: name: Hosted VS2017 -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64_NetFx461 buildScript: build.cmd @@ -79,13 +80,13 @@ phases: _configuration: Release-netfx _config_short: RFX _includeBenchmarkData: false - queue: + pool: name: Hosted VS2017 -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - queue: + pool: name: Hosted VS2017 diff --git a/build/ci/phase-template.yml b/build/ci/job-template.yml similarity index 69% rename from build/ci/phase-template.yml rename to build/ci/job-template.yml index 6d70c14bda..2f00bd0555 100644 --- a/build/ci/phase-template.yml +++ b/build/ci/job-template.yml @@ -2,23 +2,18 @@ parameters: name: '' architecture: x64 buildScript: '' - queue: {} + pool: {} customMatrixes: '' codeCoverage: false + container: '' -phases: - - phase: ${{ parameters.name }} - variables: - _buildScript: ${{ parameters.buildScript }} - _phaseName: ${{ parameters.name }} - _arch: ${{ parameters.architecture }} - _codeCoverage: ${{ parameters.codeCoverage }} - queue: - ${{ if eq(variables._codeCoverage, 'false') }}: - timeoutInMinutes: 30 - ${{ if eq(variables._codeCoverage, 'true') }}: - timeoutInMinutes: 60 - parallel: 99 +jobs: + - job: ${{ parameters.name }} + ${{ if eq(parameters.codeCoverage, 'false') }}: + timeoutInMinutes: 30 + ${{ if eq(parameters.codeCoverage, 'true') }}: + timeoutInMinutes: 60 + strategy: matrix: ${{ if eq(parameters.customMatrixes, '') }}: Debug_Build: @@ -31,20 +26,24 @@ phases: _includeBenchmarkData: true ${{ if ne(parameters.customMatrixes, '') }}: ${{ insert }}: ${{ parameters.customMatrixes }} - ${{ insert }}: ${{ parameters.queue }} + + pool: ${{ parameters.pool }} + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + steps: - - ${{ if eq(parameters.queue.name, 'Hosted macOS') }}: + - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}: - script: brew update && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb && brew install mono-libgdiplus gettext && brew link gettext --force && brew link libomp --force displayName: Install build dependencies - - script: $(_buildScript) -$(_configuration) -buildArch=$(_arch) + - script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }} displayName: Build - - script: $(_buildScript) -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData) + - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData) displayName: Download Benchmark Data - - script: $(_buildScript) -$(_configuration) -runtests -coverage=$(_codeCoverage) + - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }} displayName: Run Tests. - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild build/Codecoverage.proj /p:CodeCovToken=$(CODECOV_TOKEN) displayName: Upload coverage to codecov.io - condition: and(succeeded(), eq(variables._codeCoverage, 'true')) + condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True)) - task: PublishTestResults@2 displayName: Publish Test Results condition: succeededOrFailed() @@ -52,7 +51,7 @@ phases: testRunner: 'vSTest' searchFolder: '$(System.DefaultWorkingDirectory)/bin' testResultsFiles: '**/*.trx' - testRunTitle: Machinelearning_Tests_$(_phaseName)_$(_configuration)_$(Build.BuildNumber) + testRunTitle: Machinelearning_Tests_${{ parameters.name }}_$(_configuration)_$(Build.BuildNumber) configuration: $(_configuration) mergeTestResults: true - task: CopyFiles@2 @@ -78,5 +77,5 @@ phases: pathToPublish: $(Build.ArtifactStagingDirectory) artifactName: ${{ parameters.name }} $(_config_short) artifactType: container - - script: $(_buildScript) -buildPackages + - script: ${{ parameters.buildScript }} -buildPackages displayName: Build Packages diff --git a/build/codecoverage-ci.yml b/build/codecoverage-ci.yml index 7dc3910255..3f6005f53e 100644 --- a/build/codecoverage-ci.yml +++ b/build/codecoverage-ci.yml @@ -3,7 +3,7 @@ ################################################################################ phases: -- template: /build/ci/phase-template.yml +- template: /build/ci/job-template.yml parameters: name: Windows_x64 buildScript: build.cmd From 278045047af7896f1e4b6e38c2a72104d4644742 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Tue, 25 Jun 2019 22:34:22 -0700 Subject: [PATCH 2/6] Fix queue --- .vsts-dotnet-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 61388f280c..3c727fc99a 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -58,7 +58,7 @@ jobs: _includeBenchmarkData: true pool: name: NetCorePublic-Pool - queue: buildpool.windows.10.amd64.vs2017 + queue: buildpool.windows.10.amd64.vs2017.open - template: /build/ci/job-template.yml parameters: From 87406937c729af39b0263e13c7d60322af354261 Mon Sep 17 00:00:00 2001 From: Wei-Sheng Chin Date: Wed, 26 Jun 2019 11:43:25 -0700 Subject: [PATCH 3/6] Try again --- build/Dependencies.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index 2ed9835fb4..d63101377e 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -15,7 +15,7 @@ 3.5.1 2.2.3 2.1.0 - 0.3.0 + 0.4.0 0.0.0.9 2.1.3 4.5.0 From 99264f40e4b474d8dd00ad44a504c446c24ab43d Mon Sep 17 00:00:00 2001 From: Wei-Sheng Chin Date: Wed, 26 Jun 2019 12:46:13 -0700 Subject: [PATCH 4/6] Increase tol --- test/Microsoft.ML.Tests/FeatureContributionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/FeatureContributionTests.cs b/test/Microsoft.ML.Tests/FeatureContributionTests.cs index 02cf7b3d1b..e47a9572b0 100644 --- a/test/Microsoft.ML.Tests/FeatureContributionTests.cs +++ b/test/Microsoft.ML.Tests/FeatureContributionTests.cs @@ -124,7 +124,7 @@ public void TestSVMBinary() [Fact] public void TestLogisticRegressionBinary() { - TestFeatureContribution(ML.BinaryClassification.Trainers.LbfgsLogisticRegression(), GetSparseDataset(TaskType.BinaryClassification, 100), "LogisticRegressionBinary"); + TestFeatureContribution(ML.BinaryClassification.Trainers.LbfgsLogisticRegression(), GetSparseDataset(TaskType.BinaryClassification, 100), "LogisticRegressionBinary", 4); } [Fact] From 75b029810a30aad2122aebe9ab9ff3418eac56c9 Mon Sep 17 00:00:00 2001 From: Wei-Sheng Chin Date: Wed, 26 Jun 2019 13:35:28 -0700 Subject: [PATCH 5/6] Increase tol again --- test/Microsoft.ML.Tests/FeatureContributionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/FeatureContributionTests.cs b/test/Microsoft.ML.Tests/FeatureContributionTests.cs index e47a9572b0..7e9c4b67c2 100644 --- a/test/Microsoft.ML.Tests/FeatureContributionTests.cs +++ b/test/Microsoft.ML.Tests/FeatureContributionTests.cs @@ -124,7 +124,7 @@ public void TestSVMBinary() [Fact] public void TestLogisticRegressionBinary() { - TestFeatureContribution(ML.BinaryClassification.Trainers.LbfgsLogisticRegression(), GetSparseDataset(TaskType.BinaryClassification, 100), "LogisticRegressionBinary", 4); + TestFeatureContribution(ML.BinaryClassification.Trainers.LbfgsLogisticRegression(), GetSparseDataset(TaskType.BinaryClassification, 100), "LogisticRegressionBinary", 3); } [Fact] From d4e7e141ba80f505e1e2632dc99363f724fdd1fe Mon Sep 17 00:00:00 2001 From: Wei-Sheng Chin Date: Wed, 26 Jun 2019 15:07:16 -0700 Subject: [PATCH 6/6] Increase build time --- build/ci/job-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index 2f00bd0555..82415d87ba 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -10,7 +10,7 @@ parameters: jobs: - job: ${{ parameters.name }} ${{ if eq(parameters.codeCoverage, 'false') }}: - timeoutInMinutes: 30 + timeoutInMinutes: 40 ${{ if eq(parameters.codeCoverage, 'true') }}: timeoutInMinutes: 60 strategy: