Skip to content

Commit 97c4533

Browse files
frank-dong-ms-zzfrank-dong-ms-zz
authored
separate build pipelines (#4688)
* remove flaky test from CI * refine text * remove timeout setting * also remove benchmarktest from inner loop * fix skip theory * fix define constants * skip benchmark properly * skip tests in compile time * skip one more tensorflow test from CI
1 parent 112a92f commit 97c4533

File tree

21 files changed

+237
-18
lines changed

21 files changed

+237
-18
lines changed

‎.vsts-dotnet-ci.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ jobs:
2525
_configuration: Release-netcoreapp3_0
2626
_config_short: RI
2727
_includeBenchmarkData: true
28+
innerLoop: true
2829
pool:
2930
name: Hosted Ubuntu 1604
3031

@@ -33,13 +34,15 @@ jobs:
3334
name: Ubuntu_x64_NetCoreApp21
3435
buildScript: ./build.sh
3536
container: UbuntuContainer
37+
innerLoop: true
3638
pool:
3739
name: Hosted Ubuntu 1604
3840

3941
- template: /build/ci/job-template.yml
4042
parameters:
4143
name: MacOS_x64_NetCoreApp21
4244
buildScript: ./build.sh
45+
innerLoop: true
4346
pool:
4447
name: Hosted macOS High Sierra
4548

@@ -56,6 +59,7 @@ jobs:
5659
_configuration: Release-netcoreapp3_0
5760
_config_short: RI
5861
_includeBenchmarkData: true
62+
innerLoop: true
5963
pool:
6064
name: NetCorePublic-Pool
6165
queue: buildpool.windows.10.amd64.vs2017.open
@@ -64,6 +68,7 @@ jobs:
6468
parameters:
6569
name: Windows_x64_NetCoreApp21
6670
buildScript: build.cmd
71+
innerLoop: true
6772
pool:
6873
name: Hosted VS2017
6974

@@ -80,6 +85,7 @@ jobs:
8085
_configuration: Release-netfx
8186
_config_short: RFX
8287
_includeBenchmarkData: false
88+
innerLoop: true
8389
pool:
8490
name: Hosted VS2017
8591

@@ -88,5 +94,6 @@ jobs:
8894
name: Windows_x86_NetCoreApp21
8995
architecture: x86
9096
buildScript: build.cmd
97+
innerLoop: true
9198
pool:
9299
name: Hosted VS2017

‎build.proj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@
9797
Targets="RunTests" />
9898
</Target>
9999

100+
<Target Name="RunCITests">
101+
<MSBuild Projects="test\run-tests.proj"
102+
Targets="RunCITests" />
103+
</Target>
104+
100105
<Target Name="RunNightlyBuildTests">
101106
<MSBuild Projects="test\run-night-build-tests.proj"
102107
Targets="RunNightlyBuildTests" />

‎build/.outer-loop-build.yml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
################################################################################
2+
# ML.NET's outer loop build that will run all tests
3+
################################################################################
4+
5+
# no PR builds
6+
pr: none
7+
8+
# no CI builds
9+
trigger: none
10+
11+
# scheduled trigger, runs at UTC 8:00 every day which is midnight of GMT-8
12+
schedules:
13+
- cron: "0 8 * * *"
14+
displayName: outer loop build at midnight
15+
branches:
16+
include:
17+
- master
18+
- releases/1.0
19+
- features/automl
20+
- features/integrationPackage
21+
always: true
22+
23+
24+
resources:
25+
containers:
26+
- container: CentosContainer
27+
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
28+
29+
- container: UbuntuContainer
30+
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303
31+
32+
jobs:
33+
- template: /build/ci/job-template.yml
34+
parameters:
35+
name: Centos_x64_NetCoreApp30
36+
buildScript: ./build.sh
37+
container: CentosContainer
38+
customMatrixes:
39+
Debug_Build:
40+
_configuration: Debug-netcoreapp3_0
41+
_config_short: DI
42+
_includeBenchmarkData: false
43+
Release_Build:
44+
_configuration: Release-netcoreapp3_0
45+
_config_short: RI
46+
_includeBenchmarkData: true
47+
pool:
48+
name: Hosted Ubuntu 1604
49+
50+
- template: /build/ci/job-template.yml
51+
parameters:
52+
name: Ubuntu_x64_NetCoreApp21
53+
buildScript: ./build.sh
54+
container: UbuntuContainer
55+
pool:
56+
name: Hosted Ubuntu 1604
57+
58+
- template: /build/ci/job-template.yml
59+
parameters:
60+
name: MacOS_x64_NetCoreApp21
61+
buildScript: ./build.sh
62+
pool:
63+
name: Hosted macOS High Sierra
64+
65+
- template: /build/ci/job-template.yml
66+
parameters:
67+
name: Windows_x64_NetCoreApp30
68+
buildScript: build.cmd
69+
customMatrixes:
70+
Debug_Build:
71+
_configuration: Debug-netcoreapp3_0
72+
_config_short: DI
73+
_includeBenchmarkData: false
74+
Release_Build:
75+
_configuration: Release-netcoreapp3_0
76+
_config_short: RI
77+
_includeBenchmarkData: true
78+
pool:
79+
name: NetCorePublic-Pool
80+
queue: buildpool.windows.10.amd64.vs2017.open
81+
82+
- template: /build/ci/job-template.yml
83+
parameters:
84+
name: Windows_x64_NetCoreApp21
85+
buildScript: build.cmd
86+
pool:
87+
name: Hosted VS2017
88+
89+
- template: /build/ci/job-template.yml
90+
parameters:
91+
name: Windows_x64_NetFx461
92+
buildScript: build.cmd
93+
customMatrixes:
94+
Debug_Build:
95+
_configuration: Debug-netfx
96+
_config_short: DFX
97+
_includeBenchmarkData: false
98+
Release_Build:
99+
_configuration: Release-netfx
100+
_config_short: RFX
101+
_includeBenchmarkData: false
102+
pool:
103+
name: Hosted VS2017
104+
105+
- template: /build/ci/job-template.yml
106+
parameters:
107+
name: Windows_x86_NetCoreApp21
108+
architecture: x86
109+
buildScript: build.cmd
110+
pool:
111+
name: Hosted VS2017

‎build/ci/job-template.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ parameters:
66
customMatrixes: ''
77
codeCoverage: false
88
nightlyBuild: false
9+
innerLoop: false
910
container: ''
1011

1112
jobs:
@@ -64,8 +65,12 @@ jobs:
6465
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
6566
- script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
6667
displayName: Download Benchmark Data
67-
- script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
68-
displayName: Run Tests.
68+
- ${{ if eq(parameters.innerLoop, 'false') }}:
69+
- script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
70+
displayName: Run All Tests.
71+
- ${{ if eq(parameters.innerLoop, 'true') }}:
72+
- script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }}
73+
displayName: Run CI Tests.
6974
- script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild -restore build/Codecoverage.proj
7075
displayName: Upload coverage to codecov.io
7176
condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))

‎config.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,12 @@
7878
"values": [],
7979
"defaultValue": ""
8080
},
81+
"RunCITests": {
82+
"description": "MsBuild target that run CI tests. Call this after building.",
83+
"valueType": "target",
84+
"values": [],
85+
"defaultValue": ""
86+
},
8187
"RunNightlyBuildTests": {
8288
"description": "MsBuild target that run the nightly build tests. Call this after building.",
8389
"valueType": "target",
@@ -166,6 +172,12 @@
166172
"RunTests": "default"
167173
}
168174
},
175+
"runCITests": {
176+
"description": "Runs CI tests. Call this after building.",
177+
"settings": {
178+
"RunCITests": "default"
179+
}
180+
},
169181
"runnightlybuildtests": {
170182
"description": "Runs the nightly build tests. Call this after building.",
171183
"settings": {

‎test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ where Attribute.IsDefined(type, typeof(CIBenchmark))
4545

4646
[BenchmarkTheory]
4747
[MemberData(nameof(GetBenchmarks))]
48+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
49+
[Trait("Category", "SkipInCI")]
4850
public void BenchmarksProjectIsNotBroken(Type type)
4951
{
5052
var summary = BenchmarkRunner.Run(type, new TestConfig().With(new OutputLogger(Output)));

‎test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2123,6 +2123,8 @@ public void EntryPointMulticlassPipelineEnsemble()
21232123
}
21242124

21252125
[LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
2126+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
2127+
[Trait("Category", "SkipInCI")]
21262128
public void EntryPointPipelineEnsembleGetSummary()
21272129
{
21282130
var dataPath = GetDataPath("breast-cancer-withheader.txt");
@@ -2638,6 +2640,8 @@ public void EntryPointEvaluateMulticlass()
26382640
}
26392641

26402642
[Fact]
2643+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
2644+
[Trait("Category", "SkipInCI")]
26412645
public void EntryPointEvaluateRegression()
26422646
{
26432647
var dataPath = GetDataPath(TestDatasets.generatedRegressionDatasetmacro.trainFilename);
@@ -2759,7 +2763,9 @@ public void EntryPointSDCAMulticlass()
27592763
TestEntryPointRoutine("iris.txt", "Trainers.StochasticDualCoordinateAscentClassifier");
27602764
}
27612765

2762-
[RetryFact]
2766+
[Fact]
2767+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
2768+
[Trait("Category", "SkipInCI")]
27632769
public void EntryPointSDCARegression()
27642770
{
27652771
TestEntryPointRoutine(TestDatasets.generatedRegressionDatasetmacro.trainFilename, "Trainers.StochasticDualCoordinateAscentRegressor", loader: TestDatasets.generatedRegressionDatasetmacro.loaderSettings);
@@ -3846,7 +3852,9 @@ public void EntryPointChainedTrainTestMacros()
38463852
validateAuc(metrics);
38473853
}
38483854

3849-
[RetryFact]
3855+
[Fact]
3856+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
3857+
[Trait("Category", "SkipInCI")]
38503858
public void EntryPointChainedCrossValMacros()
38513859
{
38523860
string inputGraph = @"
@@ -5496,6 +5504,8 @@ public void TestCrossValidationMacroMulticlassWithWarnings()
54965504
}
54975505

54985506
[Fact]
5507+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
5508+
[Trait("Category", "SkipInCI")]
54995509
public void TestCrossValidationMacroWithStratification()
55005510
{
55015511
var dataPath = GetDataPath(@"breast-cancer.txt");
@@ -6026,7 +6036,9 @@ public void TestCrossValidationMacroWithNonDefaultNames()
60266036
}
60276037
}
60286038

6029-
[RetryFact]
6039+
[Fact]
6040+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
6041+
[Trait("Category", "SkipInCI")]
60306042
public void TestOvaMacro()
60316043
{
60326044
var dataPath = GetDataPath(@"iris.txt");
@@ -6190,7 +6202,9 @@ public void TestOvaMacro()
61906202
}
61916203
}
61926204

6193-
[RetryFact]
6205+
[Fact]
6206+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
6207+
[Trait("Category", "SkipInCI")]
61946208
public void TestOvaMacroWithUncalibratedLearner()
61956209
{
61966210
var dataPath = GetDataPath(@"iris.txt");

‎test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,8 @@ public void InspectLdaModelParameters()
203203
/// Introspective Training: Linear model parameters may be inspected.
204204
/// </summary>
205205
[Fact]
206+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
207+
[Trait("Category", "SkipInCI")]
206208
public void InpsectLinearModelParameters()
207209
{
208210
var mlContext = new MLContext(seed: 1);

‎test/Microsoft.ML.Predictor.Tests/TestPredictors.cs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -182,10 +182,12 @@ public void MulticlassSdcaTest()
182182
/// <summary>
183183
/// Multiclass Logistic Regression test with a tree featurizer.
184184
/// </summary>
185-
[RetryX64Fact("x86 output differs from Baseline")]
185+
[X64Fact("x86 output differs from Baseline")]
186186
[TestCategory("Multiclass")]
187187
[TestCategory("Logistic Regression")]
188188
[TestCategory("FastTree")]
189+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
190+
[Trait("Category", "SkipInCI")]
189191
public void MulticlassTreeFeaturizedLRTest()
190192
{
191193
RunMTAThread(() =>
@@ -243,6 +245,8 @@ public void KMeansClusteringTest()
243245
[X64Fact("x86 output differs from Baseline")]
244246
[TestCategory("Binary")]
245247
[TestCategory("SDCA")]
248+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
249+
[Trait("Category", "SkipInCI")]
246250
public void LinearClassifierTest()
247251
{
248252
var binaryPredictors = new[]
@@ -271,8 +275,10 @@ public void BinaryClassifierLogisticRegressionTest()
271275
Done();
272276
}
273277

274-
[RetryX64Fact("x86 output differs from Baseline")]
278+
[X64Fact("x86 output differs from Baseline")]
275279
[TestCategory("Binary")]
280+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
281+
[Trait("Category", "SkipInCI")]
276282
public void BinaryClassifierSymSgdTest()
277283
{
278284
//Results sometimes go out of error tolerance on OS X.
@@ -311,6 +317,8 @@ public void BinaryClassifierLogisticRegressionNormTest()
311317
///</summary>
312318
[LessThanNetCore30OrNotNetCoreAndX64Fact("netcoreapp3.0 and x86 output differs from Baseline")]
313319
[TestCategory("Binary")]
320+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
321+
[Trait("Category", "SkipInCI")]
314322
public void BinaryClassifierLogisticRegressionNonNegativeTest()
315323
{
316324
var binaryPredictors = new[] { TestLearners.logisticRegressionNonNegative };
@@ -322,8 +330,10 @@ public void BinaryClassifierLogisticRegressionNonNegativeTest()
322330
/// <summary>
323331
///A test for binary classifiers
324332
///</summary>
325-
[RetryLessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
333+
[LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
326334
[TestCategory("Binary")]
335+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
336+
[Trait("Category", "SkipInCI")]
327337
public void BinaryClassifierLogisticRegressionBinNormTest()
328338
{
329339
var binaryPredictors = new[] { TestLearners.logisticRegressionBinNorm };

‎test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1022,6 +1022,8 @@ public void SavePipeTrainAndScoreFccFastTree()
10221022

10231023
[TestCategory("DataPipeSerialization")]
10241024
[Fact]
1025+
//Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
1026+
[Trait("Category", "SkipInCI")]
10251027
public void SavePipeTrainAndScoreFccTransformStr()
10261028
{
10271029
TestCore(null, false,

0 commit comments

Comments
 (0)