diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 0de370b0a0..3d76920355 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -283,6 +283,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.CodeGenerator" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Vision", "src\Microsoft.ML.Vision\Microsoft.ML.Vision.csproj", "{419F93D5-4135-4DA0-A76E-EFC23E04093D}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TestFrameworkCommon", "test\Microsoft.ML.TestFrameworkCommon\Microsoft.ML.TestFrameworkCommon.csproj", "{A22FAD27-77E8-4460-8B92-EC7090B7173A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1690,6 +1692,30 @@ Global {419F93D5-4135-4DA0-A76E-EFC23E04093D}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU {419F93D5-4135-4DA0-A76E-EFC23E04093D}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU {419F93D5-4135-4DA0-A76E-EFC23E04093D}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug|x64.ActiveCfg = Debug|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug|x64.Build.0 = Debug|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netcoreapp3_0|Any CPU.ActiveCfg = Debug-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netcoreapp3_0|Any CPU.Build.0 = Debug-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netcoreapp3_0|x64.ActiveCfg = Debug-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netcoreapp3_0|x64.Build.0 = Debug-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release|Any CPU.Build.0 = Release|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release|x64.ActiveCfg = Release|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release|x64.Build.0 = Release|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netcoreapp3_0|Any CPU.ActiveCfg = Release-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netcoreapp3_0|Any CPU.Build.0 = Release-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netcoreapp3_0|x64.ActiveCfg = Release-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netcoreapp3_0|x64.Build.0 = Release-netcoreapp3_0|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU + {A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1779,6 +1805,7 @@ Global {46CC5637-3DDF-4100-93FC-44BB87B2DB81} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {3817A875-278C-4140-BF66-3C4A8CA55F0D} = {D3D38B03-B557-484D-8348-8BADEE4DF592} {419F93D5-4135-4DA0-A76E-EFC23E04093D} = {09EADF06-BE25-4228-AB53-95AE3E15B530} + {A22FAD27-77E8-4460-8B92-EC7090B7173A} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 65282a640d..de2fff0eff 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.TestFramework.Attributes; using Xunit; using static Microsoft.ML.DataOperationsCatalog; diff --git a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs b/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs index 96fa26f313..12fd86d292 100644 --- a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs +++ b/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs @@ -6,11 +6,11 @@ using BenchmarkDotNet.Attributes; using Microsoft.ML.Data; using Microsoft.ML.Trainers.LightGbm; -using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Transforms; +using Microsoft.ML.TestFrameworkCommon; namespace Microsoft.ML.Benchmarks { diff --git a/test/Microsoft.ML.Benchmarks/RffTransform.cs b/test/Microsoft.ML.Benchmarks/RffTransform.cs index d497541209..121584c9fa 100644 --- a/test/Microsoft.ML.Benchmarks/RffTransform.cs +++ b/test/Microsoft.ML.Benchmarks/RffTransform.cs @@ -8,6 +8,7 @@ using Microsoft.ML.Data; using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Transforms; namespace Microsoft.ML.Benchmarks diff --git a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs b/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs index 1cc953e5c4..4c39766d74 100644 --- a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs +++ b/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs @@ -10,6 +10,7 @@ using Microsoft.ML.TestFramework; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; +using Microsoft.ML.TestFrameworkCommon; namespace Microsoft.ML.Benchmarks { diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/CoreBaseTestClass.cs b/test/Microsoft.ML.Core.Tests/UnitTests/CoreBaseTestClass.cs index 8badda7c76..6c25102de3 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/CoreBaseTestClass.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/CoreBaseTestClass.cs @@ -9,6 +9,7 @@ using Microsoft.ML.Internal.Utilities; using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; +using Microsoft.ML.TestFrameworkCommon; using Xunit.Abstractions; namespace Microsoft.ML.Core.Tests.UnitTests @@ -25,16 +26,6 @@ protected bool Failed() return false; } - protected bool EqualTypes(DataViewType type1, DataViewType type2, bool exactTypes) - { - Contracts.AssertValue(type1); - Contracts.AssertValue(type2); - - if (type1.Equals(type2)) - return true; - return !exactTypes && type1 is VectorDataViewType vt1 && type2 is VectorDataViewType vt2 && vt1.ItemType.Equals(vt2.ItemType) && vt1.Size == vt2.Size; - } - protected Func GetIdComparer(DataViewRow r1, DataViewRow r2, out ValueGetter idGetter) { var g1 = r1.GetIdGetter(); @@ -85,75 +76,10 @@ protected Func GetComparerVec(DataViewRow r1, DataViewRow r2, int col, { g1(ref v1); g2(ref v2); - return CompareVec(in v1, in v2, size, fn); + return TestCommon.CompareVec(in v1, in v2, size, fn); }; } - protected bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) - { - return CompareVec(in v1, in v2, size, (i, x, y) => fn(x, y)); - } - - protected bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) - { - Contracts.Assert(size == 0 || v1.Length == size); - Contracts.Assert(size == 0 || v2.Length == size); - Contracts.Assert(v1.Length == v2.Length); - - var v1Values = v1.GetValues(); - var v2Values = v2.GetValues(); - - if (v1.IsDense && v2.IsDense) - { - for (int i = 0; i < v1.Length; i++) - { - var x1 = v1Values[i]; - var x2 = v2Values[i]; - if (!fn(i, x1, x2)) - return false; - } - return true; - } - - Contracts.Assert(!v1.IsDense || !v2.IsDense); - int iiv1 = 0; - int iiv2 = 0; - var v1Indices = v1.GetIndices(); - var v2Indices = v2.GetIndices(); - for (; ; ) - { - int iv1 = v1.IsDense ? iiv1 : iiv1 < v2Indices.Length ? v1Indices[iiv1] : v1.Length; - int iv2 = v2.IsDense ? iiv2 : iiv2 < v2Indices.Length ? v2Indices[iiv2] : v2.Length; - T x1, x2; - int iv; - if (iv1 == iv2) - { - if (iv1 == v1.Length) - return true; - x1 = v1Values[iiv1]; - x2 = v2Values[iiv2]; - iv = iv1; - iiv1++; - iiv2++; - } - else if (iv1 < iv2) - { - x1 = v1Values[iiv1]; - x2 = default(T); - iv = iv1; - iiv1++; - } - else - { - x1 = default(T); - x2 = v2Values[iiv2]; - iv = iv2; - iiv2++; - } - if (!fn(iv, x1, x2)) - return false; - } - } protected Func GetColumnComparer(DataViewRow r1, DataViewRow r2, int col, DataViewType type, bool exactDoubles) { if (type is VectorDataViewType vecType) @@ -331,7 +257,7 @@ protected bool CheckSameValues(DataViewRowCursor curs1, DataViewRowCursor curs2, { var type1 = curs1.Schema[col].Type; var type2 = curs2.Schema[col].Type; - if (!EqualTypes(type1, type2, exactTypes)) + if (!TestCommon.EqualTypes(type1, type2, exactTypes)) { Fail($"Different types {type1} and {type2}"); return Failed(); @@ -420,7 +346,7 @@ protected bool CheckSameValues(DataViewRowCursor curs1, IDataView view2, bool ex Contracts.Assert(cursors[col] != null); var type1 = curs1.Schema[col].Type; var type2 = cursors[col].Schema[col].Type; - if (!EqualTypes(type1, type2, exactTypes)) + if (!TestCommon.EqualTypes(type1, type2, exactTypes)) { Fail("Different types"); return Failed(); diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 87a0c61f1c..bd425c05ff 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -17,6 +17,7 @@ using Microsoft.ML.Model.OnnxConverter; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.Ensemble; using Microsoft.ML.Trainers.FastTree; diff --git a/test/Microsoft.ML.Functional.Tests/DataIO.cs b/test/Microsoft.ML.Functional.Tests/DataIO.cs index c073629fb7..2cff7132c8 100644 --- a/test/Microsoft.ML.Functional.Tests/DataIO.cs +++ b/test/Microsoft.ML.Functional.Tests/DataIO.cs @@ -4,7 +4,7 @@ using System.IO; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Xunit; using Xunit.Abstractions; @@ -13,7 +13,7 @@ namespace Microsoft.ML.Functional.Tests /// /// Test data input and output formats. /// - public class DataIO : BaseTestClass + public class DataIO : FunctionalTestBaseClass { // Separators to test private readonly char[] _separators; @@ -118,7 +118,7 @@ public void WriteAndReadAFromABinaryFile() private string SerializeDatasetToFile(MLContext mlContext, IDataView data, char separator) { - var filePath = GetOutputPath(Path.GetRandomFileName()); + var filePath = TestCommon.GetOutputPath(OutDir, Path.GetRandomFileName()); using (var file = File.Create(filePath)) mlContext.Data.SaveAsText(data, file, separatorChar: separator, headerRow: true); @@ -127,7 +127,7 @@ private string SerializeDatasetToFile(MLContext mlContext, IDataView data, char private string SerializeDatasetToBinaryFile(MLContext mlContext, IDataView data) { - var filePath = GetOutputPath(Path.GetRandomFileName()); + var filePath = TestCommon.GetOutputPath(OutDir, Path.GetRandomFileName()); using (var file = File.Create(filePath)) mlContext.Data.SaveAsBinary(data, file); diff --git a/test/Microsoft.ML.Functional.Tests/DataTransformation.cs b/test/Microsoft.ML.Functional.Tests/DataTransformation.cs index 5689be7320..f9bd039dde 100644 --- a/test/Microsoft.ML.Functional.Tests/DataTransformation.cs +++ b/test/Microsoft.ML.Functional.Tests/DataTransformation.cs @@ -4,8 +4,7 @@ using System; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms.Text; using Xunit; @@ -13,7 +12,7 @@ namespace Microsoft.ML.Functional.Tests { - public class DataTransformation : BaseTestClass + public class DataTransformation : FunctionalTestBaseClass { public DataTransformation(ITestOutputHelper output) : base(output) { @@ -30,7 +29,7 @@ void ExtensibilityAddAColumnAsAFunctionOfMultipleColumns() // Load the Iris dataset var data = mlContext.Data.LoadFromTextFile( - GetDataPath(TestDatasets.iris.trainFilename), + TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -83,7 +82,7 @@ void ExtensibilityAddingTwoColumns() // Load the Iris dataset var data = mlContext.Data.LoadFromTextFile( - GetDataPath(TestDatasets.iris.trainFilename), + TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -129,7 +128,7 @@ void ExtensibilityModifyTextFeaturization() // Concurrency must be 1 to assure that the mapping is done sequentially. var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator); @@ -167,7 +166,7 @@ void ExtensibilityNormalizeColumns() // Load the Iris dataset. var data = mlContext.Data.LoadFromTextFile( - GetDataPath(TestDatasets.iris.trainFilename), + TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); diff --git a/test/Microsoft.ML.Functional.Tests/Debugging.cs b/test/Microsoft.ML.Functional.Tests/Debugging.cs index 5e25f7f3b0..0989a1269b 100644 --- a/test/Microsoft.ML.Functional.Tests/Debugging.cs +++ b/test/Microsoft.ML.Functional.Tests/Debugging.cs @@ -5,8 +5,7 @@ using System.Collections.Generic; using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms.Text; using Xunit; @@ -14,7 +13,7 @@ namespace Microsoft.ML.Functional.Tests { - public class Debugging : BaseTestClass + public class Debugging : FunctionalTestBaseClass { public Debugging(ITestOutputHelper output) : base(output) { @@ -101,7 +100,7 @@ public void InspectPipelineSchema() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Define a pipeline var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -130,7 +129,7 @@ public void InspectSchemaUponLoadingData() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Verify the column names. int i = 0; @@ -168,7 +167,7 @@ public void ViewTrainingOutput() mlContext.Log += logWatcher.ObserveEvent; // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Define a pipeline var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) diff --git a/test/Microsoft.ML.Functional.Tests/Evaluation.cs b/test/Microsoft.ML.Functional.Tests/Evaluation.cs index 57c2d7f29e..caf60a75d4 100644 --- a/test/Microsoft.ML.Functional.Tests/Evaluation.cs +++ b/test/Microsoft.ML.Functional.Tests/Evaluation.cs @@ -4,9 +4,8 @@ using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; -using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; +using Microsoft.ML.TestFrameworkCommon.Attributes; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Xunit; @@ -14,7 +13,7 @@ namespace Microsoft.ML.Functional.Tests { - public class Evaluation : BaseTestClass + public class Evaluation : FunctionalTestBaseClass { public Evaluation(ITestOutputHelper output): base(output) { @@ -30,10 +29,10 @@ public void TrainAndEvaluateAnomalyDetection() var trainData = MnistOneClass.GetTextLoader(mlContext, TestDatasets.mnistOneClass.fileHasHeader, TestDatasets.mnistOneClass.fileSeparator) - .Load(GetDataPath(TestDatasets.mnistOneClass.trainFilename)); + .Load(TestCommon.GetDataPath(DataDir, TestDatasets.mnistOneClass.trainFilename)); var testData = MnistOneClass.GetTextLoader(mlContext, TestDatasets.mnistOneClass.fileHasHeader, TestDatasets.mnistOneClass.fileSeparator) - .Load(GetDataPath(TestDatasets.mnistOneClass.testFilename)); + .Load(TestCommon.GetDataPath(DataDir, TestDatasets.mnistOneClass.testFilename)); // Create a training pipeline. var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca(); @@ -58,7 +57,7 @@ public void TrainAndEvaluateBinaryClassification() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator); @@ -87,7 +86,7 @@ public void TrainAndEvaluateBinaryClassificationWithCalibration() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator); @@ -116,7 +115,7 @@ public void TrainAndEvaluateClustering() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.iris.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -144,7 +143,7 @@ public void TrainAndEvaluateMulticlassClassification() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.iris.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -169,7 +168,7 @@ public void TrainAndEvaluateMulticlassClassification() private IDataView GetScoredDataForRankingEvaluation(MLContext mlContext) { var data = Iris.LoadAsRankingProblem(mlContext, - GetDataPath(TestDatasets.iris.trainFilename), + TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -230,7 +229,7 @@ public void TrainAndEvaluateRecommendation() // Get the dataset. var data = TrivialMatrixFactorization.LoadAndFeaturizeFromTextFile( mlContext, - GetDataPath(TestDatasets.trivialMatrixFactorization.trainFilename), + TestCommon.GetDataPath(DataDir, TestDatasets.trivialMatrixFactorization.trainFilename), TestDatasets.trivialMatrixFactorization.fileHasHeader, TestDatasets.trivialMatrixFactorization.fileSeparator); @@ -265,7 +264,7 @@ public void TrainAndEvaluateRegression() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) .Append(mlContext.Regression.Trainers.FastForest(new FastForestRegressionTrainer.Options { NumberOfThreads = 1 })); @@ -292,7 +291,7 @@ public void TrainAndEvaluateWithPrecisionRecallCurves() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator); diff --git a/test/Microsoft.ML.Functional.Tests/Explainability.cs b/test/Microsoft.ML.Functional.Tests/Explainability.cs index a63da1abea..b470a63336 100644 --- a/test/Microsoft.ML.Functional.Tests/Explainability.cs +++ b/test/Microsoft.ML.Functional.Tests/Explainability.cs @@ -4,8 +4,7 @@ using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Xunit; @@ -16,7 +15,7 @@ namespace Microsoft.ML.Functional.Tests /// /// Test explainability features. /// - public class Explainability : BaseTestClass + public class Explainability : FunctionalTestBaseClass { public Explainability(ITestOutputHelper output) : base(output) { @@ -33,7 +32,7 @@ public void GlobalFeatureImportanceWithPermutationFeatureImportance(bool saveMod var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -50,7 +49,7 @@ public void GlobalFeatureImportanceWithPermutationFeatureImportance(bool saveMod ITransformer loadedModel; // Load and save the model - var modelAndSchemaPath = GetOutputPath("TestFunctionalTestPFI.zip"); + var modelAndSchemaPath = TestCommon.GetOutputPath(OutDir, "TestFunctionalTestPFI.zip"); mlContext.Model.Save(model, data.Schema, modelAndSchemaPath); loadedModel = mlContext.Model.Load(modelAndSchemaPath, out var schema); @@ -91,7 +90,7 @@ public void GlobalFeatureImportanceForLinearModelThroughWeights() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -115,7 +114,7 @@ public void GlobalFeatureImportanceForFastTreeThroughFeatureGain() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -142,7 +141,7 @@ public void GlobalFeatureImportanceForFastForestThroughFeatureGain() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -169,7 +168,7 @@ public void LocalFeatureImportanceForLinearModel() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -206,7 +205,7 @@ public void LocalFeatureImportanceForFastTreeModel() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -243,7 +242,7 @@ public void LocalFeatureImportanceForFastForestModel() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -281,7 +280,7 @@ public void LocalFeatureImportanceForGamModel() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) diff --git a/test/Microsoft.ML.Functional.Tests/FunctionalTestBaseClass.cs b/test/Microsoft.ML.Functional.Tests/FunctionalTestBaseClass.cs new file mode 100644 index 0000000000..0209731458 --- /dev/null +++ b/test/Microsoft.ML.Functional.Tests/FunctionalTestBaseClass.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Globalization; +using System.IO; +using System.Reflection; +using System.Threading; +using Microsoft.ML.TestFrameworkCommon; +using Xunit.Abstractions; + +namespace Microsoft.ML.Functional.Tests +{ + public class FunctionalTestBaseClass : IDisposable + { + static FunctionalTestBaseClass() + { + RootDir = TestCommon.GetRepoRoot(); + DataDir = Path.Combine(RootDir, "test", "data"); + } + + public string TestName { get; set; } + public string FullTestName { get; set; } + public string OutDir { get; } + protected static string RootDir { get; } + protected static string DataDir { get; } + protected ITestOutputHelper Output { get; } + + public FunctionalTestBaseClass(ITestOutputHelper output) + { + //This locale is currently set for tests only so that the produced output + //files can be compared on systems with other locales to give set of known + //correct results that are on en-US locale. + Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); + +#if NETFRAMEWORK + string codeBaseUri = typeof(FunctionalTestBaseClass).Assembly.CodeBase; + string path = new Uri(codeBaseUri).AbsolutePath; + var currentAssemblyLocation = new FileInfo(Directory.GetParent(path).FullName); +#else + // There is an extra folder in the netfx path representing the runtime identifier. + var currentAssemblyLocation = new FileInfo(typeof(FunctionalTestBaseClass).Assembly.Location); +#endif + OutDir = Path.Combine(currentAssemblyLocation.Directory.FullName, "TestOutput"); + Directory.CreateDirectory(OutDir); + Output = output; + + ITest test = (ITest)output.GetType().GetField("test", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(output); + FullTestName = test.TestCase.TestMethod.TestClass.Class.Name + "." + test.TestCase.TestMethod.Method.Name; + TestName = test.TestCase.TestMethod.Method.Name; + + // write to the console when a test starts and stops so we can identify any test hangs/deadlocks in CI + Console.WriteLine($"Starting test: {FullTestName}"); + Initialize(); + } + + void IDisposable.Dispose() + { + Cleanup(); + Console.WriteLine($"Finished test: {FullTestName}"); + } + + protected virtual void Initialize() + { + } + + protected virtual void Cleanup() + { + } + } +} diff --git a/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs b/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs index c60165558f..1b0331a2e7 100644 --- a/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs +++ b/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs @@ -8,8 +8,7 @@ using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Transforms; @@ -18,7 +17,7 @@ namespace Microsoft.ML.Functional.Tests { - public class IntrospectiveTraining : BaseTestClass + public class IntrospectiveTraining : FunctionalTestBaseClass { public IntrospectiveTraining(ITestOutputHelper output) : base(output) { @@ -33,7 +32,7 @@ public void InspectFastForestRegresionTrees() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -73,7 +72,7 @@ public void InspectFastTreeModelParameters() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -136,7 +135,7 @@ void IntrospectGamShapeFunctions() // Load the Iris dataset. var data = mlContext.Data.LoadFromTextFile( - GetDataPath(TestDatasets.iris.trainFilename), + TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -176,7 +175,7 @@ public void InspectLdaModelParameters() var mlContext = new MLContext(seed: 1); // Load the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -208,7 +207,7 @@ public void InpsectLinearModelParameters() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -248,7 +247,7 @@ void IntrospectNormalization() // Load the Iris dataset. var data = mlContext.Data.LoadFromTextFile( - GetDataPath(TestDatasets.iris.trainFilename), + TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -278,7 +277,7 @@ public void InspectPipelineContents() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -321,7 +320,7 @@ public void InspectSlotNamesForReversibleHash() var mlContext = new MLContext(seed: 1); // Load the Adult dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.adult.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename), hasHeader: TestDatasets.adult.fileHasHeader, separatorChar: TestDatasets.adult.fileSeparator); @@ -378,7 +377,7 @@ public void InspectNestedPipeline() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.iris.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); diff --git a/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj b/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj index 436168ea20..03ee7f5be6 100644 --- a/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj +++ b/test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj @@ -17,14 +17,13 @@ - - + diff --git a/test/Microsoft.ML.Functional.Tests/ModelFiles.cs b/test/Microsoft.ML.Functional.Tests/ModelFiles.cs index 1de28f9f39..41a5003b5f 100644 --- a/test/Microsoft.ML.Functional.Tests/ModelFiles.cs +++ b/test/Microsoft.ML.Functional.Tests/ModelFiles.cs @@ -10,7 +10,7 @@ using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Transforms; using Xunit; @@ -18,7 +18,7 @@ namespace Microsoft.ML.Functional.Tests { - public partial class ModelFiles : TestDataPipeBase + public partial class ModelFiles : FunctionalTestBaseClass { public ModelFiles(ITestOutputHelper output) : base(output) { @@ -42,7 +42,7 @@ public void DetermineNugetVersionFromModel() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -53,7 +53,7 @@ public void DetermineNugetVersionFromModel() var model = pipeline.Fit(data); // Save model to a file. - var modelPath = DeleteOutputPath("determineNugetVersionFromModel.zip"); + var modelPath = TestCommon.DeleteOutputPath(OutDir, "determineNugetVersionFromModel.zip"); mlContext.Model.Save(model, data.Schema, modelPath); // Check that the version can be extracted from the model. @@ -89,7 +89,7 @@ public void FitPipelineSaveModelAndPredict() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -99,7 +99,7 @@ public void FitPipelineSaveModelAndPredict() // Fit the pipeline. var model = pipeline.Fit(data); - var modelPath = DeleteOutputPath("fitPipelineSaveModelAndPredict.zip"); + var modelPath = TestCommon.DeleteOutputPath(OutDir, "fitPipelineSaveModelAndPredict.zip"); // Save model to a file. mlContext.Model.Save(model, data.Schema, modelPath); @@ -108,7 +108,7 @@ public void FitPipelineSaveModelAndPredict() using (var file = File.OpenRead(modelPath)) { serializedModel = mlContext.Model.Load(file, out var serializedSchema); - CheckSameSchemas(data.Schema, serializedSchema); + TestCommon.CheckSameSchemas(data.Schema, serializedSchema); } // Create prediction engine and test predictions. @@ -124,21 +124,21 @@ public void FitPipelineSaveModelAndPredict() // Check that the predictions are identical. Assert.Equal(originalPrediction.Score, serializedPrediction.Score); } - - Done(); } [Fact] public void LoadModelAndExtractPredictor() { - var file = new MultiFileSource(GetDataPath(TestDatasets.adult.trainFilename)); - var loader = ML.Data.CreateTextLoader(hasHeader: true, dataSample: file); + var mlContext = new MLContext(); + + var file = new MultiFileSource(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename)); + var loader = mlContext.Data.CreateTextLoader(hasHeader: true, dataSample: file); var data = loader.Load(file); // Pipeline. - var pipeline = ML.BinaryClassification.Trainers.Gam(); + var pipeline = mlContext.BinaryClassification.Trainers.Gam(); // Define the same pipeline starting with the loader. - var pipeline1 = loader.Append(ML.BinaryClassification.Trainers.Gam()); + var pipeline1 = loader.Append(mlContext.BinaryClassification.Trainers.Gam()); // Train. var transformerModel = pipeline.Fit(data); @@ -147,35 +147,35 @@ public void LoadModelAndExtractPredictor() // Save and reload the "same" model with some differences in structure. // In this case we are saving the transformer model, but *not* the loader, just the schema from that loader. - string modelAndSchemaPath = GetOutputPath(FullTestName + "-model-schema.zip"); - ML.Model.Save(transformerModel, data.Schema, modelAndSchemaPath); + string modelAndSchemaPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-model-schema.zip"); + mlContext.Model.Save(transformerModel, data.Schema, modelAndSchemaPath); // In this case we have combined the loader with the transformer model to form a "composite" loader, and are just // saving that one loader to this file. - string compositeLoaderModelPath = GetOutputPath(FullTestName + "-composite-model.zip"); - ML.Model.Save(null, compositeLoaderModel, compositeLoaderModelPath); + string compositeLoaderModelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-composite-model.zip"); + mlContext.Model.Save(null, compositeLoaderModel, compositeLoaderModelPath); // In this case we are saving the transformer model, as well as the associated data loader. - string loaderAndTransformerModelPath = GetOutputPath(FullTestName + "-loader-transformer.zip"); - ML.Model.Save(transformerModel, loader, loaderAndTransformerModelPath); + string loaderAndTransformerModelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-loader-transformer.zip"); + mlContext.Model.Save(transformerModel, loader, loaderAndTransformerModelPath); ITransformer loadedTransformerModel; IDataLoader loadedCompositeLoader; ITransformer loadedTransformerModel1; using (var fs = File.OpenRead(modelAndSchemaPath)) - loadedTransformerModel = ML.Model.Load(fs, out var loadedSchema); + loadedTransformerModel = mlContext.Model.Load(fs, out var loadedSchema); using (var fs = File.OpenRead(compositeLoaderModelPath)) { // This model can be loaded either as a composite data loader, // a transformer model + an input schema, or a transformer model + a data loader. - var t = ML.Model.LoadWithDataLoader(fs, out loadedCompositeLoader); + var t = mlContext.Model.LoadWithDataLoader(fs, out loadedCompositeLoader); // This is a bit strange, as it seems to test that it can reload from the same // stream twice opened only once, which as far as I know is not really a requirement // of the design or API, but we are nonetheless testing it. If this winds up failing, // I'm not sure we should really insist on this as a design requirement. - var t1 = ML.Model.Load(fs, out var s); + var t1 = mlContext.Model.Load(fs, out var s); - CheckSameSchemas(loadedCompositeLoader.GetOutputSchema(), s); + TestCommon.CheckSameSchemas(loadedCompositeLoader.GetOutputSchema(), s); // We combined the GAM with the loader, so the remaining chain should just be empty. Assert.Empty(Assert.IsType>(t)); Assert.Empty(Assert.IsType>(t1)); @@ -184,10 +184,10 @@ public void LoadModelAndExtractPredictor() { // This model can be loaded either as a composite data loader, // a transformer model + an input schema, or a transformer model + a data loader. - var t = ML.Model.Load(fs, out var s); - CheckSameSchemas(loader.GetOutputSchema(), s); + var t = mlContext.Model.Load(fs, out var s); + TestCommon.CheckSameSchemas(loader.GetOutputSchema(), s); - loadedTransformerModel1 = ML.Model.LoadWithDataLoader(fs, out var l); + loadedTransformerModel1 = mlContext.Model.LoadWithDataLoader(fs, out var l); } void AssertIsGam(ITransformer trans) @@ -209,8 +209,6 @@ void AssertIsGam(ITransformer trans) var fromComposite = Assert.Single(Assert.IsType>( Assert.IsType>(loadedCompositeLoader).Transformer)); AssertIsGam(fromComposite); - - Done(); } public class ModelInput @@ -238,7 +236,7 @@ public void LoadModelWithOptionalColumnTransform() var mlContext = new MLContext(); ITransformer trainedModel; DataViewSchema dataViewSchema; - trainedModel = mlContext.Model.Load(GetDataPath("backcompat", "modelwithoptionalcolumntransform.zip"), out dataViewSchema); + trainedModel = mlContext.Model.Load(TestCommon.GetDataPath(DataDir, "backcompat", "modelwithoptionalcolumntransform.zip"), out dataViewSchema); var model = mlContext.Model.CreatePredictionEngine(trainedModel, inputSchemaDefinition: inputSchemaDefinition); var prediction = model.Predict(new ModelInput() { CategoricalFeatures = new[] { "ABC", "ABC", "ABC", "ABC", "ABC" }, NumericalFeatures = new float [] { 1, 1, 1 } }); @@ -248,19 +246,21 @@ public void LoadModelWithOptionalColumnTransform() [Fact] public void SaveAndLoadModelWithLoader() { - var file = new MultiFileSource(GetDataPath(TestDatasets.adult.trainFilename)); - var loader = ML.Data.CreateTextLoader(hasHeader: true, dataSample: file); + var mlContext = new MLContext(); + + var file = new MultiFileSource(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename)); + var loader = mlContext.Data.CreateTextLoader(hasHeader: true, dataSample: file); var data = loader.Load(file); // Pipeline. - var pipeline = ML.BinaryClassification.Trainers.Gam(); + var pipeline = mlContext.BinaryClassification.Trainers.Gam(); // Train. var model = pipeline.Fit(data); // Save and reload. - string modelPath = GetOutputPath(FullTestName + "-model.zip"); - ML.Model.Save(model, loader, modelPath); + string modelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-model.zip"); + mlContext.Model.Save(model, loader, modelPath); IDataLoader loadedLoader; ITransformer loadedModelWithoutLoader; @@ -268,17 +268,17 @@ public void SaveAndLoadModelWithLoader() DataViewSchema loadedSchema; using (var fs = File.OpenRead(modelPath)) { - loadedModelWithLoader = ML.Model.LoadWithDataLoader(fs, out loadedLoader); + loadedModelWithLoader = mlContext.Model.LoadWithDataLoader(fs, out loadedLoader); Assert.IsAssignableFrom>(loadedModelWithLoader); - loadedModelWithoutLoader = ML.Model.Load(fs, out loadedSchema); + loadedModelWithoutLoader = mlContext.Model.Load(fs, out loadedSchema); Assert.IsAssignableFrom>(loadedModelWithoutLoader); - CheckSameSchemas(loadedLoader.GetOutputSchema(), loadedSchema); + TestCommon.CheckSameSchemas(loadedLoader.GetOutputSchema(), loadedSchema); } // When using a novel data source other than one derived from the loader, we will not have // the slot names. - data = ML.Data.LoadFromEnumerable(new[] { new InputData() }); + data = mlContext.Data.LoadFromEnumerable(new[] { new InputData() }); data = loadedModelWithoutLoader.Transform(data); Assert.False(data.Schema["Features"].HasSlotNames()); // When we plumb the loaded schema through the transformer though, we should have slot names. @@ -303,44 +303,48 @@ public void SaveAndLoadModelWithLoader() [Fact] public void LoadSchemaAndCreateNewData() { - var file = new MultiFileSource(GetDataPath(TestDatasets.adult.trainFilename)); - var loader = ML.Data.CreateTextLoader(hasHeader: true, dataSample: file); + var mlContext = new MLContext(); + + var file = new MultiFileSource(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename)); + var loader = mlContext.Data.CreateTextLoader(hasHeader: true, dataSample: file); var data = loader.Load(file); // Pipeline. - var pipeline = ML.Transforms.NormalizeMinMax("Features"); + var pipeline = mlContext.Transforms.NormalizeMinMax("Features"); // Train. var model = pipeline.Fit(data); // Save and reload. - string modelPath = GetOutputPath(FullTestName + "-model.zip"); - ML.Model.Save(model, loader, modelPath); + string modelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-model.zip"); + mlContext.Model.Save(model, loader, modelPath); ITransformer loadedModel; DataViewSchema loadedSchema; using (var fs = File.OpenRead(modelPath)) - loadedModel = ML.Model.Load(fs, out loadedSchema); + loadedModel = mlContext.Model.Load(fs, out loadedSchema); // Without using the schema from the model we lose the slot names. - data = ML.Data.LoadFromEnumerable(new[] { new InputData() }); + data = mlContext.Data.LoadFromEnumerable(new[] { new InputData() }); data = loadedModel.Transform(data); Assert.True(!data.Schema["Features"].HasSlotNames()); - data = ML.Data.LoadFromEnumerable(new[] { new InputData() }, loadedSchema); + data = mlContext.Data.LoadFromEnumerable(new[] { new InputData() }, loadedSchema); Assert.True(data.Schema["Features"].HasSlotNames()); } [Fact] public void SaveTextLoaderAndLoad() { - var file = new MultiFileSource(GetDataPath(TestDatasets.adult.trainFilename)); - var loader = ML.Data.CreateTextLoader(hasHeader: true, dataSample: file); + var mlContext = new MLContext(); - string modelPath = GetOutputPath(FullTestName + "-model.zip"); - ML.Model.Save(null, loader, modelPath); + var file = new MultiFileSource(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename)); + var loader = mlContext.Data.CreateTextLoader(hasHeader: true, dataSample: file); - Load(modelPath, out var loadedWithSchema, out var loadedSchema, + string modelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-model.zip"); + mlContext.Model.Save(null, loader, modelPath); + + Load(mlContext, modelPath, out var loadedWithSchema, out var loadedSchema, out var loadedWithLoader, out var loadedLoaderWithTransformer); Assert.True(loadedWithSchema is TransformerChain); Assert.False((loadedWithSchema as TransformerChain).Any()); @@ -361,15 +365,17 @@ public void SaveTextLoaderAndLoad() [Fact] public void SaveCompositeLoaderAndLoad() { - var file = new MultiFileSource(GetDataPath(TestDatasets.adult.trainFilename)); - var loader = ML.Data.CreateTextLoader(hasHeader: true, dataSample: file); - var composite = loader.Append(ML.Transforms.NormalizeMinMax("Features")); + var mlContext = new MLContext(); + + var file = new MultiFileSource(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename)); + var loader = mlContext.Data.CreateTextLoader(hasHeader: true, dataSample: file); + var composite = loader.Append(mlContext.Transforms.NormalizeMinMax("Features")); var loaderWithEmbeddedModel = composite.Fit(file); - string modelPath = GetOutputPath(FullTestName + "-model.zip"); - ML.Model.Save(null, loaderWithEmbeddedModel, modelPath); + string modelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-model.zip"); + mlContext.Model.Save(null, loaderWithEmbeddedModel, modelPath); - Load(modelPath, out var loadedWithSchema, out var loadedSchema, + Load(mlContext, modelPath, out var loadedWithSchema, out var loadedSchema, out var loadedWithLoader, out var loadedLoaderWithTransformer); // Because we saved the transform model as part of the composite loader, with no transforms, // the transform that should be loaded should be an empty transformer chain, since the "model," @@ -383,25 +389,25 @@ public void SaveCompositeLoaderAndLoad() Assert.NotNull(expectedSchema.GetColumnOrNull("Features")); Assert.True(expectedSchema["Features"].HasSlotNames()); - CheckSameSchemas(loaderWithEmbeddedModel.GetOutputSchema(), loadedSchema); + TestCommon.CheckSameSchemas(loaderWithEmbeddedModel.GetOutputSchema(), loadedSchema); var schemaFromLoadedLoader = loadedLoaderWithTransformer.GetOutputSchema(); - CheckSameSchemas(loaderWithEmbeddedModel.GetOutputSchema(), schemaFromLoadedLoader); + TestCommon.CheckSameSchemas(loaderWithEmbeddedModel.GetOutputSchema(), schemaFromLoadedLoader); // The type of the loader itself should be a composite data loader, and its single transformer // should be the normalizing transformer. var compositeLoader = Assert.IsType>(loadedLoaderWithTransformer); var chainFromLoader = compositeLoader.Transformer; Assert.IsType(Assert.Single(compositeLoader.Transformer)); - - Done(); } [Fact] public void SaveLoaderAndTransformerAndLoad() { - var file = new MultiFileSource(GetDataPath(TestDatasets.adult.trainFilename)); - var loader = ML.Data.CreateTextLoader(hasHeader: true, dataSample: file); - var estimator = ML.Transforms.NormalizeMinMax("Features"); + var mlContext = new MLContext(); + + var file = new MultiFileSource(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename)); + var loader = mlContext.Data.CreateTextLoader(hasHeader: true, dataSample: file); + var estimator = mlContext.Transforms.NormalizeMinMax("Features"); var data = loader.Load(file); var model = estimator.Fit(data); @@ -412,35 +418,35 @@ public void SaveLoaderAndTransformerAndLoad() Assert.NotNull(expectedInputSchema.GetColumnOrNull("Features")); Assert.True(expectedInputSchema["Features"].HasSlotNames()); - string modelPath = GetOutputPath(FullTestName + "-model.zip"); - ML.Model.Save(model, loader, modelPath); + string modelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-model.zip"); + mlContext.Model.Save(model, loader, modelPath); // Reload the loader and schema. - Load(modelPath, out var loadedWithSchema, out var loadedInputSchema, + Load(mlContext, modelPath, out var loadedWithSchema, out var loadedInputSchema, out var loadedWithLoader, out var loadedLoaderWithTransformer); Assert.IsType(loadedWithSchema); Assert.IsType(loadedWithLoader); Assert.IsType(loadedLoaderWithTransformer); - CheckSameSchemas(expectedInputSchema, loadedInputSchema); + TestCommon.CheckSameSchemas(expectedInputSchema, loadedInputSchema); var reloadedLoaderInputSchema = loadedLoaderWithTransformer.GetOutputSchema(); - CheckSameSchemas(expectedInputSchema, reloadedLoaderInputSchema); - - Done(); + TestCommon.CheckSameSchemas(expectedInputSchema, reloadedLoaderInputSchema); } [Fact] public void SaveTransformerAndSchemaAndLoad() { - var file = new MultiFileSource(GetDataPath(TestDatasets.adult.trainFilename)); - var loader = ML.Data.CreateTextLoader(hasHeader: true, dataSample: file); - var estimator = ML.Transforms.NormalizeMinMax("Features"); + var mlContext = new MLContext(); + + var file = new MultiFileSource(TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename)); + var loader = mlContext.Data.CreateTextLoader(hasHeader: true, dataSample: file); + var estimator = mlContext.Transforms.NormalizeMinMax("Features"); var model = estimator.Fit(loader.Load(file)); - string modelPath = GetOutputPath(FullTestName + "-model.zip"); - ML.Model.Save(model, loader.GetOutputSchema(), modelPath); + string modelPath = TestCommon.GetOutputPath(OutDir, FullTestName + "-model.zip"); + mlContext.Model.Save(model, loader.GetOutputSchema(), modelPath); - Load(modelPath, out var loadedWithSchema, out var loadedSchema, + Load(mlContext, modelPath, out var loadedWithSchema, out var loadedSchema, out var loadedWithLoader, out var loadedLoaderWithTransformer); Assert.True(loadedWithSchema is NormalizingTransformer); Assert.True(loadedSchema.Count == 2 && @@ -451,15 +457,15 @@ public void SaveTransformerAndSchemaAndLoad() Assert.Null(loadedLoaderWithTransformer); } - private void Load(string filename, out ITransformer loadedWithSchema, out DataViewSchema loadedSchema, + private void Load(MLContext mlContext, string filename, out ITransformer loadedWithSchema, out DataViewSchema loadedSchema, out ITransformer loadedWithLoader, out IDataLoader loadedLoaderWithTransformer) { using (var fs = File.OpenRead(filename)) { - loadedWithSchema = ML.Model.Load(fs, out loadedSchema); + loadedWithSchema = mlContext.Model.Load(fs, out loadedSchema); try { - loadedWithLoader = ML.Model.LoadWithDataLoader(fs, out loadedLoaderWithTransformer); + loadedWithLoader = mlContext.Model.LoadWithDataLoader(fs, out loadedLoaderWithTransformer); } catch (Exception) { diff --git a/test/Microsoft.ML.Functional.Tests/ONNX.cs b/test/Microsoft.ML.Functional.Tests/ONNX.cs index eaaab95981..d438ad33d2 100644 --- a/test/Microsoft.ML.Functional.Tests/ONNX.cs +++ b/test/Microsoft.ML.Functional.Tests/ONNX.cs @@ -5,9 +5,8 @@ using System.IO; using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; -using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; +using Microsoft.ML.TestFrameworkCommon.Attributes; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Xunit; @@ -15,7 +14,7 @@ namespace Microsoft.ML.Functional.Tests { - public class ONNX : BaseTestClass + public class ONNX : FunctionalTestBaseClass { public ONNX(ITestOutputHelper output) : base(output) { @@ -30,7 +29,7 @@ public void SaveOnnxModelLoadAndScoreFastTree() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -44,7 +43,7 @@ public void SaveOnnxModelLoadAndScoreFastTree() // Serialize the pipeline to a file. var modelFileName = "SaveOnnxLoadAndScoreFastTreeModel.onnx"; - var modelPath = DeleteOutputPath(modelFileName); + var modelPath = TestCommon.DeleteOutputPath(OutDir, modelFileName); using (var file = File.Create(modelPath)) mlContext.Model.ConvertToOnnx(model, data, file); @@ -85,7 +84,7 @@ public void SaveOnnxModelLoadAndScoreKMeans() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -99,7 +98,7 @@ public void SaveOnnxModelLoadAndScoreKMeans() // Serialize the pipeline to a file. var modelFileName = "SaveOnnxLoadAndScoreKMeansModel.onnx"; - var modelPath = DeleteOutputPath(modelFileName); + var modelPath = TestCommon.DeleteOutputPath(OutDir, modelFileName); using (var file = File.Create(modelPath)) mlContext.Model.ConvertToOnnx(model, data, file); @@ -137,7 +136,7 @@ public void SaveOnnxModelLoadAndScoreSDCA() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -151,7 +150,7 @@ public void SaveOnnxModelLoadAndScoreSDCA() // Serialize the pipeline to a file. var modelFileName = "SaveOnnxLoadAndScoreSdcaModel.onnx"; - var modelPath = DeleteOutputPath(modelFileName); + var modelPath = TestCommon.DeleteOutputPath(OutDir, modelFileName); using (var file = File.Create(modelPath)) mlContext.Model.ConvertToOnnx(model, data, file); diff --git a/test/Microsoft.ML.Functional.Tests/Prediction.cs b/test/Microsoft.ML.Functional.Tests/Prediction.cs index c19770bf9e..8292ff709a 100644 --- a/test/Microsoft.ML.Functional.Tests/Prediction.cs +++ b/test/Microsoft.ML.Functional.Tests/Prediction.cs @@ -7,15 +7,14 @@ using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; using Xunit.Abstractions; namespace Microsoft.ML.Functional.Tests { - public class PredictionScenarios : BaseTestClass + public class PredictionScenarios : FunctionalTestBaseClass { public PredictionScenarios(ITestOutputHelper output) : base(output) { @@ -37,7 +36,7 @@ public void ReconfigurablePrediction() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), hasHeader: TestDatasets.Sentiment.fileHasHeader, separatorChar: TestDatasets.Sentiment.fileSeparator); diff --git a/test/Microsoft.ML.Functional.Tests/SchemaDefinitionTests.cs b/test/Microsoft.ML.Functional.Tests/SchemaDefinitionTests.cs index 11255b6885..47aec19755 100644 --- a/test/Microsoft.ML.Functional.Tests/SchemaDefinitionTests.cs +++ b/test/Microsoft.ML.Functional.Tests/SchemaDefinitionTests.cs @@ -5,14 +5,13 @@ using System; using System.Linq; using Microsoft.ML.Data; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Xunit; using Xunit.Abstractions; namespace Microsoft.ML.Functional.Tests { - public class SchemaDefinitionTests : BaseTestClass + public class SchemaDefinitionTests : FunctionalTestBaseClass { private MLContext _ml; @@ -25,13 +24,12 @@ protected override void Initialize() base.Initialize(); _ml = new MLContext(42); - _ml.AddStandardComponents(); } [Fact] public void SchemaDefinitionForPredictionEngine() { - var fileName = GetDataPath(TestDatasets.adult.trainFilename); + var fileName = TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename); var loader = _ml.Data.CreateTextLoader(new TextLoader.Options(), new MultiFileSource(fileName)); var data = loader.Load(new MultiFileSource(fileName)); var pipeline1 = _ml.Transforms.Categorical.OneHotEncoding("Cat", "Workclass", maximumNumberOfKeys: 3) @@ -61,7 +59,7 @@ public void SchemaDefinitionForPredictionEngine() [Fact] public void SchemaDefinitionForCustomMapping() { - var fileName = GetDataPath(TestDatasets.adult.trainFilename); + var fileName = TestCommon.GetDataPath(DataDir, TestDatasets.adult.trainFilename); var data = new MultiFileSource(fileName); var loader = _ml.Data.CreateTextLoader(new TextLoader.Options(), new MultiFileSource(fileName)); var pipeline = _ml.Transforms.Categorical.OneHotEncoding("Categories") diff --git a/test/Microsoft.ML.Functional.Tests/Training.cs b/test/Microsoft.ML.Functional.Tests/Training.cs index 6f8d264805..26b268d2d6 100644 --- a/test/Microsoft.ML.Functional.Tests/Training.cs +++ b/test/Microsoft.ML.Functional.Tests/Training.cs @@ -5,8 +5,7 @@ using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Xunit; @@ -14,7 +13,7 @@ namespace Microsoft.ML.Functional.Tests { - public class Training : BaseTestClass + public class Training : FunctionalTestBaseClass { public Training(ITestOutputHelper output) : base(output) { @@ -29,7 +28,7 @@ public void CompareTrainerEvaluations() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), separatorChar: TestDatasets.Sentiment.fileSeparator, hasHeader: TestDatasets.Sentiment.fileHasHeader, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -83,7 +82,7 @@ public void ContinueTrainingAveragePerceptron() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), separatorChar: TestDatasets.Sentiment.fileSeparator, hasHeader: TestDatasets.Sentiment.fileHasHeader, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -127,7 +126,7 @@ public void ContinueTrainingFieldAwareFactorizationMachine() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), separatorChar: TestDatasets.Sentiment.fileSeparator, hasHeader: TestDatasets.Sentiment.fileHasHeader, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -171,7 +170,7 @@ public void ContinueTrainingLinearSupportVectorMachine() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), separatorChar: TestDatasets.Sentiment.fileSeparator, hasHeader: TestDatasets.Sentiment.fileHasHeader, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -215,7 +214,7 @@ public void ContinueTrainingLogisticRegression() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), separatorChar: TestDatasets.Sentiment.fileSeparator, hasHeader: TestDatasets.Sentiment.fileHasHeader, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -258,7 +257,7 @@ public void ContinueTrainingLogisticRegressionMulticlass() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.iris.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -309,7 +308,7 @@ public void ContinueTrainingOnlineGradientDescent() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), separatorChar: TestDatasets.housing.fileSeparator, hasHeader: TestDatasets.housing.fileHasHeader); @@ -353,7 +352,7 @@ public void ContinueTrainingPoissonRegression() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), separatorChar: TestDatasets.housing.fileSeparator, hasHeader: TestDatasets.housing.fileHasHeader); @@ -397,7 +396,7 @@ public void ContinueTrainingSymbolicStochasticGradientDescent() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.Sentiment.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.Sentiment.trainFilename), separatorChar: TestDatasets.Sentiment.fileSeparator, hasHeader: TestDatasets.Sentiment.fileHasHeader, allowQuoting: TestDatasets.Sentiment.allowQuoting); @@ -445,7 +444,7 @@ public void MetacomponentsFunctionAsExpectedOva() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.iris.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); @@ -476,7 +475,7 @@ public void MetacomponentsFunctionWithKeyHandling() { var mlContext = new MLContext(seed: 1); - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.iris.trainFilename), + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.iris.trainFilename), hasHeader: TestDatasets.iris.fileHasHeader, separatorChar: TestDatasets.iris.fileSeparator); diff --git a/test/Microsoft.ML.Functional.Tests/Validation.cs b/test/Microsoft.ML.Functional.Tests/Validation.cs index c269720140..64d6352aaf 100644 --- a/test/Microsoft.ML.Functional.Tests/Validation.cs +++ b/test/Microsoft.ML.Functional.Tests/Validation.cs @@ -4,8 +4,7 @@ using Microsoft.ML.Data; using Microsoft.ML.Functional.Tests.Datasets; -using Microsoft.ML.RunTests; -using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Trainers.FastTree; using Xunit; @@ -13,7 +12,7 @@ namespace Microsoft.ML.Functional.Tests { - public class Validation : BaseTestClass + public class Validation : FunctionalTestBaseClass { public Validation(ITestOutputHelper output) : base(output) { @@ -32,7 +31,7 @@ void CrossValidation() var mlContext = new MLContext(seed: 1); // Get the dataset - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create a pipeline to train on the housing data. var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features) @@ -61,7 +60,7 @@ public void TrainWithValidationSet() var mlContext = new MLContext(seed: 1); // Get the dataset. - var data = mlContext.Data.LoadFromTextFile(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true); + var data = mlContext.Data.LoadFromTextFile(TestCommon.GetDataPath(DataDir, TestDatasets.housing.trainFilename), hasHeader: true); // Create the train and validation set. var dataSplit = mlContext.Data.TrainTestSplit(data, testFraction: 0.2); diff --git a/test/Microsoft.ML.OnnxTransformerTest/DnnImageFeaturizerTest.cs b/test/Microsoft.ML.OnnxTransformerTest/DnnImageFeaturizerTest.cs index 9f791ad572..614e86026b 100644 --- a/test/Microsoft.ML.OnnxTransformerTest/DnnImageFeaturizerTest.cs +++ b/test/Microsoft.ML.OnnxTransformerTest/DnnImageFeaturizerTest.cs @@ -10,6 +10,7 @@ using Microsoft.ML.Model; using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon.Attributes; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs index f8dd41084d..8236c75d69 100644 --- a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs +++ b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs @@ -17,6 +17,7 @@ using Xunit; using Xunit.Abstractions; using Microsoft.ML.Transforms.Onnx; +using Microsoft.ML.TestFrameworkCommon.Attributes; namespace Microsoft.ML.Tests { diff --git a/test/Microsoft.ML.Predictor.Tests/TestConcurrency.cs b/test/Microsoft.ML.Predictor.Tests/TestConcurrency.cs index 6d7c49612b..04388d6a4e 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestConcurrency.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestConcurrency.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using Microsoft.ML.Runtime; +using Microsoft.ML.TestFrameworkCommon; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 6daceea70e..faa7122b27 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -23,6 +23,7 @@ namespace Microsoft.ML.RunTests using Xunit; using Xunit.Abstractions; using TestLearners = TestLearnersBase; + using Microsoft.ML.TestFrameworkCommon; /// /// Tests using maml commands (IDV) functionality. diff --git a/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreAndX64FactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreAndX64FactAttribute.cs index c6bddec8ab..d2facf6c47 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreAndX64FactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreAndX64FactAttribute.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.ML.TestFrameworkCommon.Attributes; + namespace Microsoft.ML.TestFramework.Attributes { /// diff --git a/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreFactAttribute.cs index d2a2e20663..83df75cd9e 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreFactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/LessThanNetCore30OrNotNetCoreFactAttribute.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.ML.TestFrameworkCommon.Attributes; + namespace Microsoft.ML.TestFramework.Attributes { /// diff --git a/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs index 37002d545a..fbedec7676 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/LightGBMFactAttribute.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.ML.TestFrameworkCommon.Attributes; + namespace Microsoft.ML.TestFramework.Attributes { /// diff --git a/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs index 0bc1606be4..f076ca43f3 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/OnnxTheoryAttribute.cs @@ -4,6 +4,7 @@ using System; using System.Runtime.InteropServices; +using Microsoft.ML.TestFrameworkCommon.Attributes; namespace Microsoft.ML.TestFramework.Attributes { diff --git a/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs index 5a84fba1b3..f7ae0ecf91 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.ML.TestFrameworkCommon.Attributes; + namespace Microsoft.ML.TestFramework.Attributes { /// diff --git a/test/Microsoft.ML.TestFramework/Attributes/X64FactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/X64FactAttribute.cs index ca654d93da..6b00d475bd 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/X64FactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/X64FactAttribute.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.ML.TestFrameworkCommon.Attributes; + namespace Microsoft.ML.TestFramework.Attributes { /// diff --git a/test/Microsoft.ML.TestFramework/BaseTestClass.cs b/test/Microsoft.ML.TestFramework/BaseTestClass.cs index d4809721c7..569c34b3a1 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestClass.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestClass.cs @@ -8,6 +8,7 @@ using System.Reflection; using System.Threading; using Microsoft.ML.Internal.Internallearn.Test; +using Microsoft.ML.TestFrameworkCommon; using Xunit.Abstractions; namespace Microsoft.ML.TestFramework @@ -20,30 +21,10 @@ public class BaseTestClass : IDisposable static BaseTestClass() { GlobalBase.AssemblyInit(); - RootDir = GetRepoRoot(); + RootDir = TestCommon.GetRepoRoot(); DataDir = Path.Combine(RootDir, "test", "data"); } - private static string GetRepoRoot() - { -#if NETFRAMEWORK - string directory = AppDomain.CurrentDomain.BaseDirectory; -#else - string directory = AppContext.BaseDirectory; -#endif - - while (!Directory.Exists(Path.Combine(directory, ".git")) && directory != null) - { - directory = Directory.GetParent(directory).FullName; - } - - if (directory == null) - { - return null; - } - return directory; - } - public BaseTestClass(ITestOutputHelper output) { //This locale is currently set for tests only so that the produced output @@ -105,38 +86,21 @@ public static string GetDataPath(string subDir, string name) return Path.GetFullPath(Path.Combine(DataDir, subDir, name)); } - protected void EnsureOutputDir(string subDir) - { - Directory.CreateDirectory(Path.Combine(OutDir, subDir)); - } protected string GetOutputPath(string name) { - if (string.IsNullOrWhiteSpace(name)) - return null; - return Path.Combine(OutDir, name); + return TestCommon.GetOutputPath(OutDir, name); } protected string GetOutputPath(string subDir, string name) { - if (string.IsNullOrWhiteSpace(subDir)) - return GetOutputPath(name); - EnsureOutputDir(subDir); - if (string.IsNullOrWhiteSpace(name)) - return null; - return Path.Combine(OutDir, subDir, name); // REVIEW: put the path in in braces in case the path has spaces + return TestCommon.GetOutputPath(OutDir, subDir, name); } protected string DeleteOutputPath(string subDir, string name) { - string path = GetOutputPath(subDir, name); - if (!string.IsNullOrWhiteSpace(path)) - File.Delete(path); - return path; + return TestCommon.DeleteOutputPath(OutDir, subDir, name); } protected string DeleteOutputPath(string name) { - string path = GetOutputPath(name); - if (!string.IsNullOrWhiteSpace(path)) - File.Delete(path); - return path; + return TestCommon.DeleteOutputPath(OutDir, name); } } -} +} \ No newline at end of file diff --git a/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs b/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs index 2cfceafebf..20734fa4e2 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestPredictorsMaml.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Runtime.InteropServices; using Microsoft.ML.Runtime; +using Microsoft.ML.TestFrameworkCommon; namespace Microsoft.ML.RunTests { diff --git a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs index f645e11d2d..bc4cc77a7d 100644 --- a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs +++ b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs @@ -9,6 +9,7 @@ using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Runtime; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Text; using Xunit; @@ -231,7 +232,7 @@ public void SavePipeKeyToVec() { getters[i](ref v1); getters[i + 1](ref v2); - Check(CompareVec(in v1, in v2, v1.Length, fn), "Mismatch"); + Check(TestCommon.CompareVec(in v1, in v2, v1.Length, fn), "Mismatch"); } } } @@ -587,7 +588,7 @@ public void SavePipeWordHash() { get1(ref bag1); get2(ref bag2); - if (!CompareVec(in bag1, in bag2, bag1.Length, (x1, x2) => x1 <= x2)) + if (!TestCommon.CompareVec(in bag1, in bag2, bag1.Length, (x1, x2) => x1 <= x2)) { Fail("Values don't match in columns F13, F23"); return; @@ -629,7 +630,7 @@ public void SavePipeWordHashUnordered() { get1(ref bag1); get2(ref bag2); - if (!CompareVec(in bag1, in bag2, bag1.Length, (x1, x2) => 2 * x1 == x2)) + if (!TestCommon.CompareVec(in bag1, in bag2, bag1.Length, (x1, x2) => 2 * x1 == x2)) { Fail("Values don't match"); return; @@ -781,7 +782,7 @@ public void SavePipeWordBagManyToOne() { get1(ref b1); get2(ref b2); - if (!CompareVec(in b1, in b2, b1.Length, (x1, x2) => 2 * x1 == x2)) + if (!TestCommon.CompareVec(in b1, in b2, b1.Length, (x1, x2) => 2 * x1 == x2)) { Fail("Unexpected values in row {0}", c.Position); break; @@ -924,7 +925,7 @@ Action action while (c.MoveNext()) { getter(ref buffer); - CompareVec(in buffer, in expected[index++], buffer.GetValues().Length, (s1, s2) => s1.Span.SequenceEqual(s2.Span)); + TestCommon.CompareVec(in buffer, in expected[index++], buffer.GetValues().Length, (s1, s2) => s1.Span.SequenceEqual(s2.Span)); } } }; diff --git a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs index 4afa6781d9..116031fbc4 100644 --- a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs +++ b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs @@ -13,6 +13,7 @@ using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Xunit; namespace Microsoft.ML.RunTests @@ -97,7 +98,7 @@ protected void TestEstimatorCore(IEstimator estimator, { var mapper = transformer.GetRowToRowMapper(data.Schema); Check(mapper.InputSchema == data.Schema, "InputSchemas were not identical to actual input schema"); - CheckSameSchemas(schema, mapper.OutputSchema); + TestCommon.CheckSameSchemas(schema, mapper.OutputSchema); } else { @@ -105,21 +106,21 @@ protected void TestEstimatorCore(IEstimator estimator, } // Loaded transformer needs to have the same schema propagation. - CheckSameSchemas(schema, loadedTransformer.GetOutputSchema(data.Schema)); + TestCommon.CheckSameSchemas(schema, loadedTransformer.GetOutputSchema(data.Schema)); // Loaded schema needs to have the same schema as data. - CheckSameSchemas(data.Schema, loadedInputSchema); + TestCommon.CheckSameSchemas(data.Schema, loadedInputSchema); var scoredTrain = transformer.Transform(data); var scoredTrain2 = loadedTransformer.Transform(data); // The schema of the transformed data must match the schema provided by schema propagation. - CheckSameSchemas(schema, scoredTrain.Schema); + TestCommon.CheckSameSchemas(schema, scoredTrain.Schema); // The schema and data of scored dataset must be identical between loaded // and original transformer. // This in turn means that the schema of loaded transformer matches for // Transform and GetOutputSchema calls. - CheckSameSchemas(scoredTrain.Schema, scoredTrain2.Schema); + TestCommon.CheckSameSchemas(scoredTrain.Schema, scoredTrain2.Schema); CheckSameValues(scoredTrain, scoredTrain2, exactDoubles: false); }; @@ -199,7 +200,7 @@ internal ILegacyDataLoader TestCore(string pathData, bool keepHidden, string[] a if (!CheckMetadataTypes(reappliedPipe.Schema)) Failed(); - if (!CheckSameSchemas(pipe1.Schema, reappliedPipe.Schema)) + if (!TestCommon.CheckSameSchemas(pipe1.Schema, reappliedPipe.Schema)) Failed(); else if (!CheckSameValues(pipe1, reappliedPipe, checkId: checkId)) Failed(); @@ -232,7 +233,7 @@ internal ILegacyDataLoader TestCore(string pathData, bool keepHidden, string[] a if (!CheckMetadataTypes(pipe2.Schema)) Failed(); - if (!CheckSameSchemas(pipe1.Schema, pipe2.Schema)) + if (!TestCommon.CheckSameSchemas(pipe1.Schema, pipe2.Schema)) Failed(); else if (!CheckSameValues(pipe1, pipe2, checkId: checkId)) Failed(); @@ -422,7 +423,7 @@ protected bool SaveLoadText(IDataView view, IHostEnvironment env, if (!CheckMetadataTypes(loadedData.Schema)) Failed(); - if (!CheckSameSchemas(view.Schema, loadedData.Schema, exactTypes: false, keyNames: false)) + if (!TestCommon.CheckSameSchemas(view.Schema, loadedData.Schema, exactTypes: false, keyNames: false)) return Failed(); if (!CheckSameValues(view, loadedData, exactTypes: false, exactDoubles: false, checkId: false)) return Failed(); @@ -516,137 +517,6 @@ protected bool CheckMetadataTypes(DataViewSchema sch) return true; } - protected bool CheckSameSchemas(DataViewSchema sch1, DataViewSchema sch2, bool exactTypes = true, bool keyNames = true) - { - if (sch1.Count != sch2.Count) - { - Fail("column count mismatch: {0} vs {1}", sch1.Count, sch2.Count); - return Failed(); - } - - for (int col = 0; col < sch1.Count; col++) - { - string name1 = sch1[col].Name; - string name2 = sch2[col].Name; - if (name1 != name2) - { - Fail("column name mismatch at index {0}: {1} vs {2}", col, name1, name2); - return Failed(); - } - var type1 = sch1[col].Type; - var type2 = sch2[col].Type; - if (!EqualTypes(type1, type2, exactTypes)) - { - Fail("column type mismatch at index {0}", col); - return Failed(); - } - - // This ensures that the two schemas map names to the same column indices. - int col1, col2; - bool f1 = sch1.TryGetColumnIndex(name1, out col1); - bool f2 = sch2.TryGetColumnIndex(name2, out col2); - if (!Check(f1, "TryGetColumnIndex unexpectedly failed")) - return Failed(); - if (!Check(f2, "TryGetColumnIndex unexpectedly failed")) - return Failed(); - if (col1 != col2) - { - Fail("TryGetColumnIndex on '{0}' produced different results: '{1}' vs '{2}'", name1, col1, col2); - return Failed(); - } - - // This checks that an unknown metadata kind does the right thing. - if (!CheckMetadataNames("PurpleDragonScales", 0, sch1, sch2, col, exactTypes, true)) - return Failed(); - - ulong vsize = type1 is VectorDataViewType vectorType ? (ulong)vectorType.Size : 0; - if (!CheckMetadataNames(AnnotationUtils.Kinds.SlotNames, vsize, sch1, sch2, col, exactTypes, true)) - return Failed(); - - if (!keyNames) - continue; - - ulong ksize = type1.GetItemType() is KeyDataViewType keyType ? keyType.Count : 0; - if (!CheckMetadataNames(AnnotationUtils.Kinds.KeyValues, ksize, sch1, sch2, col, exactTypes, false)) - return Failed(); - } - - return true; - } - - protected bool CheckMetadataNames(string kind, ulong size, DataViewSchema sch1, DataViewSchema sch2, int col, bool exactTypes, bool mustBeText) - { - var names1 = default(VBuffer>); - var names2 = default(VBuffer>); - - var t1 = sch1[col].Annotations.Schema.GetColumnOrNull(kind)?.Type; - var t2 = sch2[col].Annotations.Schema.GetColumnOrNull(kind)?.Type; - if ((t1 == null) != (t2 == null)) - { - Fail("Different null-ness of {0} metadata types", kind); - return Failed(); - } - if (t1 == null) - { - if (!CheckMetadataCallFailure(kind, sch1, col, ref names1)) - return Failed(); - if (!CheckMetadataCallFailure(kind, sch2, col, ref names2)) - return Failed(); - return true; - } - if (size > int.MaxValue) - Fail(nameof(KeyDataViewType) + "." + nameof(KeyDataViewType.Count) + "is larger than int.MaxValue"); - if (!EqualTypes(t1, t2, exactTypes)) - { - Fail("Different {0} metadata types: {0} vs {1}", kind, t1, t2); - return Failed(); - } - if (!(t1.GetItemType() is TextDataViewType)) - { - if (!mustBeText) - { - Log("Metadata '{0}' was not text so skipping comparison", kind); - return true; // REVIEW: Do something a bit more clever here. - } - Fail("Unexpected {0} metadata type", kind); - return Failed(); - } - - if ((int)size != t1.GetVectorSize()) - { - Fail("{0} metadata type wrong size: {1} vs {2}", kind, t1.GetVectorSize(), size); - return Failed(); - } - - sch1[col].Annotations.GetValue(kind, ref names1); - sch2[col].Annotations.GetValue(kind, ref names2); - if (!CompareVec(in names1, in names2, (int)size, (a, b) => a.Span.SequenceEqual(b.Span))) - { - Fail("Different {0} metadata values", kind); - return Failed(); - } - return true; - } - - protected bool CheckMetadataCallFailure(string kind, DataViewSchema sch, int col, ref VBuffer> names) - { - try - { - sch[col].Annotations.GetValue(kind, ref names); - Fail("Getting {0} metadata unexpectedly succeeded", kind); - return Failed(); - } - catch (InvalidOperationException ex) - { - if (ex.Message != "Invalid call to 'GetValue'") - { - Fail("Message from GetValue failed call doesn't match expected message: {0}", ex.Message); - return Failed(); - } - } - return true; - } - protected bool SaveLoad(IDataView view, IHostEnvironment env, string suffix = "") { var saverArgs = new BinarySaver.Arguments(); @@ -686,7 +556,7 @@ protected bool SaveLoad(IDataView view, IHostEnvironment env, string suffix = "" if (!CheckMetadataTypes(loader.Schema)) return Failed(); - if (!CheckSameSchemas(view.Schema, loader.Schema)) + if (!TestCommon.CheckSameSchemas(view.Schema, loader.Schema)) return Failed(); if (!CheckSameValues(view, loader, checkId: false)) return Failed(); @@ -737,7 +607,7 @@ protected bool SaveLoadTransposed(IDataView view, IHostEnvironment env, string s if (!CheckMetadataTypes(loader.Schema)) return Failed(); - if (!CheckSameSchemas(view.Schema, loader.Schema)) + if (!TestCommon.CheckSameSchemas(view.Schema, loader.Schema)) return Failed(); if (!CheckSameValues(view, loader, checkId: false)) return Failed(); @@ -1148,81 +1018,15 @@ protected Func GetComparerVec(DataViewRow r1, DataViewRow r2, int col, { g1(ref v1); g2(ref v2); - return CompareVec(in v1, in v2, size, fn); + return TestCommon.CompareVec(in v1, in v2, size, fn); }; } - protected bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) - { - return CompareVec(in v1, in v2, size, (i, x, y) => fn(x, y)); - } - - protected bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) - { - Contracts.Assert(size == 0 || v1.Length == size); - Contracts.Assert(size == 0 || v2.Length == size); - Contracts.Assert(v1.Length == v2.Length); - - var v1Values = v1.GetValues(); - var v2Values = v2.GetValues(); - - if (v1.IsDense && v2.IsDense) - { - for (int i = 0; i < v1.Length; i++) - { - var x1 = v1Values[i]; - var x2 = v2Values[i]; - if (!fn(i, x1, x2)) - return false; - } - return true; - } - - var v1Indices = v1.GetIndices(); - var v2Indices = v2.GetIndices(); - - Contracts.Assert(!v1.IsDense || !v2.IsDense); - int iiv1 = 0; - int iiv2 = 0; - for (; ; ) - { - int iv1 = v1.IsDense ? iiv1 : iiv1 < v1Indices.Length ? v1Indices[iiv1] : v1.Length; - int iv2 = v2.IsDense ? iiv2 : iiv2 < v2Indices.Length ? v2Indices[iiv2] : v2.Length; - T x1, x2; - int iv; - if (iv1 == iv2) - { - if (iv1 == v1.Length) - return true; - x1 = v1Values[iiv1]; - x2 = v2Values[iiv2]; - iv = iv1; - iiv1++; - iiv2++; - } - else if (iv1 < iv2) - { - x1 = v1Values[iiv1]; - x2 = default(T); - iv = iv1; - iiv1++; - } - else - { - x1 = default(T); - x2 = v2Values[iiv2]; - iv = iv2; - iiv2++; - } - if (!fn(iv, x1, x2)) - return false; - } - } - // Verifies the equality of the values returned by the single valued getters passed in as parameters. protected void VerifyOneEquality(ValueGetter oneGetter, ValueGetter oneNGetter) { T f1 = default(T); + T f1n = default(T); oneGetter(ref f1); oneNGetter(ref f1n); @@ -1236,7 +1040,7 @@ protected void VerifyVecEquality(ValueGetter> vecGetter, ValueGett VBuffer fvn = default(VBuffer); vecGetter(ref fv); vecNGetter(ref fvn); - Assert.True(CompareVec(in fv, in fvn, size, compare)); + Assert.True(TestCommon.CompareVec(in fv, in fvn, size, compare)); } } } diff --git a/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj b/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj index 00e1199c4f..2609486574 100644 --- a/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj +++ b/test/Microsoft.ML.TestFramework/Microsoft.ML.TestFramework.csproj @@ -12,9 +12,10 @@ + - + @@ -22,6 +23,6 @@ - + diff --git a/test/Microsoft.ML.TestFramework/TestCommandBase.cs b/test/Microsoft.ML.TestFramework/TestCommandBase.cs index a02c98ab04..29c6440699 100644 --- a/test/Microsoft.ML.TestFramework/TestCommandBase.cs +++ b/test/Microsoft.ML.TestFramework/TestCommandBase.cs @@ -16,6 +16,7 @@ using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Tools; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.TestFramework/Attributes/AttributeHelpers.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/AttributeHelpers.cs similarity index 85% rename from test/Microsoft.ML.TestFramework/Attributes/AttributeHelpers.cs rename to test/Microsoft.ML.TestFrameworkCommon/Attributes/AttributeHelpers.cs index f2b77c311c..abdc70cb87 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/AttributeHelpers.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/AttributeHelpers.cs @@ -5,11 +5,11 @@ using System; using System.Runtime.InteropServices; -namespace Microsoft.ML.TestFramework.Attributes +namespace Microsoft.ML.TestFrameworkCommon.Attributes { - internal static class AttributeHelpers + public static class AttributeHelpers { - internal static bool CheckLibcVersionGreaterThanMinimum(Version minVersion) + public static bool CheckLibcVersionGreaterThanMinimum(Version minVersion) { #if !NETFRAMEWORK Version version; diff --git a/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/EnvironmentSpecificFactAttribute.cs similarity index 96% rename from test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificFactAttribute.cs rename to test/Microsoft.ML.TestFrameworkCommon/Attributes/EnvironmentSpecificFactAttribute.cs index fe754b6eb2..79e6262cad 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/EnvironmentSpecificFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/EnvironmentSpecificFactAttribute.cs @@ -5,7 +5,7 @@ using System; using Xunit; -namespace Microsoft.ML.TestFramework.Attributes +namespace Microsoft.ML.TestFrameworkCommon.Attributes { /// /// A base class for environment-specific fact attributes. diff --git a/test/Microsoft.ML.TestFramework/Attributes/MatrixFactorizationFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/MatrixFactorizationFactAttribute.cs similarity index 92% rename from test/Microsoft.ML.TestFramework/Attributes/MatrixFactorizationFactAttribute.cs rename to test/Microsoft.ML.TestFrameworkCommon/Attributes/MatrixFactorizationFactAttribute.cs index 4b9969c54f..279e507ca4 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/MatrixFactorizationFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/MatrixFactorizationFactAttribute.cs @@ -3,7 +3,7 @@ // See the LICENSE file in the project root for more information. using System; -namespace Microsoft.ML.TestFramework.Attributes +namespace Microsoft.ML.TestFrameworkCommon.Attributes { /// /// A fact for tests requiring matrix factorization. diff --git a/test/Microsoft.ML.TestFramework/Attributes/OnnxFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/OnnxFactAttribute.cs similarity index 94% rename from test/Microsoft.ML.TestFramework/Attributes/OnnxFactAttribute.cs rename to test/Microsoft.ML.TestFrameworkCommon/Attributes/OnnxFactAttribute.cs index 6254f50213..80d498d89d 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/OnnxFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/OnnxFactAttribute.cs @@ -5,7 +5,7 @@ using System; using System.Runtime.InteropServices; -namespace Microsoft.ML.TestFramework.Attributes +namespace Microsoft.ML.TestFrameworkCommon.Attributes { /// /// A fact for tests requiring Onnx. diff --git a/test/Microsoft.ML.TestFramework/Datasets.cs b/test/Microsoft.ML.TestFrameworkCommon/Datasets.cs similarity index 99% rename from test/Microsoft.ML.TestFramework/Datasets.cs rename to test/Microsoft.ML.TestFrameworkCommon/Datasets.cs index 2a0dc6e6cf..a4060aa23c 100644 --- a/test/Microsoft.ML.TestFramework/Datasets.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Datasets.cs @@ -5,7 +5,7 @@ using System; using Microsoft.ML.Data; -namespace Microsoft.ML.RunTests +namespace Microsoft.ML.TestFrameworkCommon { public class TestDataset { diff --git a/test/Microsoft.ML.TestFrameworkCommon/Microsoft.ML.TestFrameworkCommon.csproj b/test/Microsoft.ML.TestFrameworkCommon/Microsoft.ML.TestFrameworkCommon.csproj new file mode 100644 index 0000000000..0959503ee0 --- /dev/null +++ b/test/Microsoft.ML.TestFrameworkCommon/Microsoft.ML.TestFrameworkCommon.csproj @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/Microsoft.ML.TestFrameworkCommon/TestCommon.cs b/test/Microsoft.ML.TestFrameworkCommon/TestCommon.cs new file mode 100644 index 0000000000..52e9a55669 --- /dev/null +++ b/test/Microsoft.ML.TestFrameworkCommon/TestCommon.cs @@ -0,0 +1,284 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.ML.Data; +using Xunit; + +namespace Microsoft.ML.TestFrameworkCommon +{ + public static class TestCommon + { + public static string GetOutputPath(string outDir, string name) + { + if (string.IsNullOrWhiteSpace(name)) + return null; + return Path.Combine(outDir, name); + } + public static string GetOutputPath(string outDir, string subDir, string name) + { + if (string.IsNullOrWhiteSpace(subDir)) + return GetOutputPath(outDir, name); + EnsureOutputDir(subDir, outDir); + if (string.IsNullOrWhiteSpace(name)) + return null; + return Path.Combine(outDir, subDir, name); // REVIEW: put the path in in braces in case the path has spaces + } + + public static string GetDataPath(string dataDir, string name) + { + if (string.IsNullOrWhiteSpace(name)) + return null; + return Path.GetFullPath(Path.Combine(dataDir, name)); + } + public static string GetDataPath(string dataDir, string subDir, string name) + { + if (string.IsNullOrWhiteSpace(name)) + return null; + return Path.GetFullPath(Path.Combine(dataDir, subDir, name)); + } + + public static string DeleteOutputPath(string outDir, string subDir, string name) + { + string path = GetOutputPath(outDir, subDir, name); + if (!string.IsNullOrWhiteSpace(path)) + File.Delete(path); + return path; + } + public static string DeleteOutputPath(string outDir, string name) + { + string path = GetOutputPath(outDir, name); + if (!string.IsNullOrWhiteSpace(path)) + File.Delete(path); + return path; + } + + public static string GetRepoRoot() + { +#if NETFRAMEWORK + string directory = AppDomain.CurrentDomain.BaseDirectory; +#else + string directory = AppContext.BaseDirectory; +#endif + + while (!Directory.Exists(Path.Combine(directory, ".git")) && directory != null) + { + directory = Directory.GetParent(directory).FullName; + } + + if (directory == null) + { + return null; + } + return directory; + } + + public static bool CheckSameSchemas(DataViewSchema sch1, DataViewSchema sch2, bool exactTypes = true, bool keyNames = true) + { + Assert.True(sch1.Count == sch2.Count, $"column count mismatch: {sch1.Count} vs {sch2.Count}"); + + for (int col = 0; col < sch1.Count; col++) + { + string name1 = sch1[col].Name; + string name2 = sch2[col].Name; + Assert.True(name1 == name2, $"column name mismatch at index {col}: {name1} vs {name2}"); + + var type1 = sch1[col].Type; + var type2 = sch2[col].Type; + Assert.True(EqualTypes(type1, type2, exactTypes), $"column type mismatch at index {col}"); + + // This ensures that the two schemas map names to the same column indices. + int col1, col2; + bool f1 = sch1.TryGetColumnIndex(name1, out col1); + bool f2 = sch2.TryGetColumnIndex(name2, out col2); + + Assert.True(f1, "TryGetColumnIndex unexpectedly failed"); + Assert.True(f2, "TryGetColumnIndex unexpectedly failed"); + Assert.True(col1 == col2, $"TryGetColumnIndex on '{name1}' produced different results: '{col1}' vs '{col2}'"); + + // This checks that an unknown metadata kind does the right thing. + if (!CheckMetadataNames("PurpleDragonScales", 0, sch1, sch2, col, exactTypes, true)) + return false; + + ulong vsize = type1 is VectorDataViewType vectorType ? (ulong)vectorType.Size : 0; + if (!CheckMetadataNames("SlotNames", vsize, sch1, sch2, col, exactTypes, true)) + return false; + + if (!keyNames) + continue; + + ulong ksize = type1.GetItemType() is KeyDataViewType keyType ? keyType.Count : 0; + if (!CheckMetadataNames("KeyValues", ksize, sch1, sch2, col, exactTypes, false)) + return false; + } + + return true; + } + + public static bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) + { + return CompareVec(in v1, in v2, size, (i, x, y) => fn(x, y)); + } + + public static bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) + { + Assert.True(size == 0 || v1.Length == size); + Assert.True(size == 0 || v2.Length == size); + Assert.True(v1.Length == v2.Length); + + var v1Values = v1.GetValues(); + var v2Values = v2.GetValues(); + + if (v1.IsDense && v2.IsDense) + { + for (int i = 0; i < v1.Length; i++) + { + var x1 = v1Values[i]; + var x2 = v2Values[i]; + if (!fn(i, x1, x2)) + return false; + } + return true; + } + + Assert.True(!v1.IsDense || !v2.IsDense); + int iiv1 = 0; + int iiv2 = 0; + var v1Indices = v1.GetIndices(); + var v2Indices = v2.GetIndices(); + for (; ; ) + { + int iv1 = v1.IsDense ? iiv1 : iiv1 < v1Indices.Length ? v1Indices[iiv1] : v1.Length; + int iv2 = v2.IsDense ? iiv2 : iiv2 < v2Indices.Length ? v2Indices[iiv2] : v2.Length; + T x1, x2; + int iv; + if (iv1 == iv2) + { + if (iv1 == v1.Length) + return true; + x1 = v1Values[iiv1]; + x2 = v2Values[iiv2]; + iv = iv1; + iiv1++; + iiv2++; + } + else if (iv1 < iv2) + { + x1 = v1Values[iiv1]; + x2 = default(T); + iv = iv1; + iiv1++; + } + else + { + x1 = default(T); + x2 = v2Values[iiv2]; + iv = iv2; + iiv2++; + } + if (!fn(iv, x1, x2)) + return false; + } + } + + public static bool EqualTypes(DataViewType type1, DataViewType type2, bool exactTypes) + { + Assert.NotNull(type1); + Assert.NotNull(type2); + + return exactTypes ? type1.Equals(type2) : type1.SameSizeAndItemType(type2); + } + + /// + /// Equivalent to calling Equals(ColumnType) for non-vector types. For vector type, + /// returns true if current and other vector types have the same size and item type. + /// + private static bool SameSizeAndItemType(this DataViewType columnType, DataViewType other) + { + if (other == null) + return false; + + if (columnType.Equals(other)) + return true; + + // For vector types, we don't care about the factoring of the dimensions. + if (!(columnType is VectorDataViewType vectorType) || !(other is VectorDataViewType otherVectorType)) + return false; + if (!vectorType.ItemType.Equals(otherVectorType.ItemType)) + return false; + return vectorType.Size == otherVectorType.Size; + } + + private static bool TryGetColumnIndex(this DataViewSchema schema, string name, out int col) + { + col = schema.GetColumnOrNull(name)?.Index ?? -1; + return col >= 0; + } + + private static bool CheckMetadataNames(string kind, ulong size, DataViewSchema sch1, DataViewSchema sch2, int col, bool exactTypes, bool mustBeText) + { + var names1 = default(VBuffer>); + var names2 = default(VBuffer>); + + var t1 = sch1[col].Annotations.Schema.GetColumnOrNull(kind)?.Type; + var t2 = sch2[col].Annotations.Schema.GetColumnOrNull(kind)?.Type; + Assert.False((t1 == null) != (t2 == null), $"Different null-ness of {kind} metadata types"); + + if (t1 == null) + { + Assert.True(CheckMetadataCallFailure(kind, sch1, col, ref names1)); + Assert.True(CheckMetadataCallFailure(kind, sch2, col, ref names2)); + + return true; + } + + Assert.False(size > int.MaxValue, $"{nameof(KeyDataViewType)}.{nameof(KeyDataViewType.Count)} is larger than int.MaxValue"); + Assert.True(EqualTypes(t1, t2, exactTypes), $"Different {kind} metadata types: {t1} vs {t2}"); + + if (!(t1.GetItemType() is TextDataViewType)) + { + if (!mustBeText) + return true; + + Assert.False(mustBeText, $"Unexpected {kind} metadata type"); + } + + Assert.True((int)size == t1.GetVectorSize(), $"{kind} metadata type wrong size: {t1.GetVectorSize()} vs {size}"); + + sch1[col].Annotations.GetValue(kind, ref names1); + sch2[col].Annotations.GetValue(kind, ref names2); + Assert.True(CompareVec(in names1, in names2, (int)size, (a, b) => a.Span.SequenceEqual(b.Span)), $"Different {kind} metadata values"); + + return true; + } + + private static bool CheckMetadataCallFailure(string kind, DataViewSchema sch, int col, ref VBuffer> names) + { + try + { + sch[col].Annotations.GetValue(kind, ref names); + + return false; + } + catch (InvalidOperationException ex) + { + if (ex.Message != "Invalid call to 'GetValue'") + { + return false; + } + } + return true; + } + + private static DataViewType GetItemType(this DataViewType columnType) => (columnType as VectorDataViewType)?.ItemType ?? columnType; + + private static int GetVectorSize(this DataViewType columnType) => (columnType as VectorDataViewType)?.Size ?? 0; + + private static void EnsureOutputDir(string subDir, string outDir) + { + Directory.CreateDirectory(Path.Combine(outDir, subDir)); + } + } +} diff --git a/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs b/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs index 2135d2ca67..020b5792b1 100644 --- a/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs +++ b/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs @@ -7,6 +7,7 @@ using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index ec991e5985..9595d35af1 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -10,6 +10,7 @@ using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 9e6f9040c4..0a79681b0b 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -15,6 +15,8 @@ using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; +using Microsoft.ML.TestFrameworkCommon.Attributes; using Microsoft.ML.Tools; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; diff --git a/test/Microsoft.ML.Tests/OnnxSequenceTypeWithAttributesTest.cs b/test/Microsoft.ML.Tests/OnnxSequenceTypeWithAttributesTest.cs index 11849c3372..d6c5bc9a97 100644 --- a/test/Microsoft.ML.Tests/OnnxSequenceTypeWithAttributesTest.cs +++ b/test/Microsoft.ML.Tests/OnnxSequenceTypeWithAttributesTest.cs @@ -13,6 +13,7 @@ using System.Linq; using System.IO; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon.Attributes; namespace Microsoft.ML.Tests { diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs index 98125a9d72..76d628526c 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs @@ -8,6 +8,7 @@ using Microsoft.ML.Data; using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Text; diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/DecomposableTrainAndPredict.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/DecomposableTrainAndPredict.cs index 78e6305d65..8d58fc403a 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/DecomposableTrainAndPredict.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/DecomposableTrainAndPredict.cs @@ -5,6 +5,7 @@ using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Xunit; diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/Extensibility.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/Extensibility.cs index e3553df150..efa509a8c6 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/Extensibility.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/Extensibility.cs @@ -6,6 +6,7 @@ using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Xunit; diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/MultithreadedPrediction.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/MultithreadedPrediction.cs index 5d5298e3cb..7b6a8a122d 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/MultithreadedPrediction.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/MultithreadedPrediction.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/PredictAndMetadata.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/PredictAndMetadata.cs index 149dd7af31..f4058959ce 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/PredictAndMetadata.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/PredictAndMetadata.cs @@ -6,6 +6,7 @@ using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/SimpleTrainAndPredict.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/SimpleTrainAndPredict.cs index 0e7da9c206..c511c6bcfb 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/SimpleTrainAndPredict.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/SimpleTrainAndPredict.cs @@ -4,6 +4,7 @@ using System.Linq; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainWithInitialPredictor.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainWithInitialPredictor.cs index 356c14ef2c..1c55100343 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainWithInitialPredictor.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainWithInitialPredictor.cs @@ -4,6 +4,7 @@ using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; diff --git a/test/Microsoft.ML.Tests/Scenarios/GetColumnTests.cs b/test/Microsoft.ML.Tests/Scenarios/GetColumnTests.cs index 4c59f177bb..34544a7004 100644 --- a/test/Microsoft.ML.Tests/Scenarios/GetColumnTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/GetColumnTests.cs @@ -9,6 +9,7 @@ using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs index 5f89e1ad79..4181ce3b19 100644 --- a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs @@ -5,6 +5,7 @@ using Microsoft.ML.Data; using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs index 36eb2d1b6b..e02d3fdc97 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs @@ -4,6 +4,7 @@ using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 6c4695e4e8..c015a309dd 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -14,6 +14,7 @@ using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Image; using Microsoft.ML.TensorFlow; diff --git a/test/Microsoft.ML.Tests/TextLoaderTests.cs b/test/Microsoft.ML.Tests/TextLoaderTests.cs index 91b78d9142..c439ae1a03 100644 --- a/test/Microsoft.ML.Tests/TextLoaderTests.cs +++ b/test/Microsoft.ML.Tests/TextLoaderTests.cs @@ -11,6 +11,7 @@ using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFrameworkCommon; using Newtonsoft.Json.Linq; using Xunit; using Xunit.Abstractions; diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/FAFMEstimator.cs b/test/Microsoft.ML.Tests/TrainerEstimators/FAFMEstimator.cs index 83f1bd374c..b322a208a0 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/FAFMEstimator.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/FAFMEstimator.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Xunit; diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index f67f1280cb..e3c0623cdc 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -11,6 +11,8 @@ using Microsoft.ML.Internal.CpuMath; using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; +using Microsoft.ML.TestFrameworkCommon.Attributes; using Microsoft.ML.Trainers; using Xunit; diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MetalinearEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MetalinearEstimators.cs index 9f94bc2560..e026ff0a9e 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MetalinearEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MetalinearEstimators.cs @@ -6,6 +6,7 @@ using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Xunit; diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/PriorRandomTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/PriorRandomTests.cs index d5c2ef7c6a..94858f6704 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/PriorRandomTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/PriorRandomTests.cs @@ -4,6 +4,7 @@ using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Xunit; namespace Microsoft.ML.Tests.TrainerEstimators diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs index 1322827ac2..ac4f2a6de3 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs @@ -4,6 +4,7 @@ using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Text; diff --git a/test/Microsoft.ML.Tests/Transformers/CopyColumnEstimatorTests.cs b/test/Microsoft.ML.Tests/Transformers/CopyColumnEstimatorTests.cs index 903da2df18..70025d8b6a 100644 --- a/test/Microsoft.ML.Tests/Transformers/CopyColumnEstimatorTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/CopyColumnEstimatorTests.cs @@ -7,6 +7,7 @@ using Microsoft.ML.Data; using Microsoft.ML.Model; using Microsoft.ML.Runtime; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Tools; using Microsoft.ML.Transforms; using Xunit; @@ -146,7 +147,7 @@ void TestMetadataCopy() var type2 = result.Schema[copyIndex].Type; result.Schema[termIndex].GetKeyValues(ref names1); result.Schema[copyIndex].GetKeyValues(ref names2); - Assert.True(CompareVec(in names1, in names2, size, (a, b) => a.Span.SequenceEqual(b.Span))); + Assert.True(TestCommon.CompareVec(in names1, in names2, size, (a, b) => a.Span.SequenceEqual(b.Span))); } [Fact] @@ -182,72 +183,6 @@ private void ValidateCopyColumnTransformer(IDataView result) } } } - private bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) - { - return CompareVec(in v1, in v2, size, (i, x, y) => fn(x, y)); - } - - private bool CompareVec(in VBuffer v1, in VBuffer v2, int size, Func fn) - { - Contracts.Assert(size == 0 || v1.Length == size); - Contracts.Assert(size == 0 || v2.Length == size); - Contracts.Assert(v1.Length == v2.Length); - - var v1Values = v1.GetValues(); - var v2Values = v2.GetValues(); - - if (v1.IsDense && v2.IsDense) - { - for (int i = 0; i < v1.Length; i++) - { - var x1 = v1Values[i]; - var x2 = v2Values[i]; - if (!fn(i, x1, x2)) - return false; - } - return true; - } - - var v1Indices = v1.GetIndices(); - var v2Indices = v2.GetIndices(); - - Contracts.Assert(!v1.IsDense || !v2.IsDense); - int iiv1 = 0; - int iiv2 = 0; - for (; ; ) - { - int iv1 = v1.IsDense ? iiv1 : iiv1 < v1Indices.Length ? v1Indices[iiv1] : v1.Length; - int iv2 = v2.IsDense ? iiv2 : iiv2 < v2Indices.Length ? v2Indices[iiv2] : v2.Length; - T x1, x2; - int iv; - if (iv1 == iv2) - { - if (iv1 == v1.Length) - return true; - x1 = v1Values[iiv1]; - x2 = v2Values[iiv2]; - iv = iv1; - iiv1++; - iiv2++; - } - else if (iv1 < iv2) - { - x1 = v1Values[iiv1]; - x2 = default(T); - iv = iv1; - iiv1++; - } - else - { - x1 = default(T); - x2 = v2Values[iiv2]; - iv = iv2; - iiv2++; - } - if (!fn(iv, x1, x2)) - return false; - } - } } } diff --git a/test/Microsoft.ML.Tests/Transformers/KeyToValueTests.cs b/test/Microsoft.ML.Tests/Transformers/KeyToValueTests.cs index 95726789fd..4ea491f00e 100644 --- a/test/Microsoft.ML.Tests/Transformers/KeyToValueTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/KeyToValueTests.cs @@ -6,6 +6,7 @@ using Microsoft.ML.Data; using Microsoft.ML.Data.IO; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Transforms; using Xunit; using Xunit.Abstractions; @@ -84,7 +85,7 @@ public void KeyToValue() var dataLeft = ML.Transforms.SelectColumns(new[] { "ScalarString", "VectorString" }).Fit(data).Transform(data); var dataRight = ML.Transforms.SelectColumns(new[] { "ScalarString", "VectorString" }).Fit(data2Transformed).Transform(data2Transformed); - CheckSameSchemas(dataLeft.Schema, dataRight.Schema); + TestCommon.CheckSameSchemas(dataLeft.Schema, dataRight.Schema); CheckSameValues(dataLeft, dataRight); Done(); } diff --git a/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs b/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs index 3a25487f15..6f8f0eb440 100644 --- a/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs @@ -12,6 +12,7 @@ using Microsoft.ML.Model; using Microsoft.ML.RunTests; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Tools; using Microsoft.ML.Transforms; using System.Linq; @@ -411,10 +412,10 @@ public void SimpleConstructorsAndExtensions() var data4 = est4.Fit(data).Transform(data); var data5 = est5.Fit(data).Transform(data); - CheckSameSchemas(data1.Schema, data2.Schema); - CheckSameSchemas(data1.Schema, data3.Schema); - CheckSameSchemas(data1.Schema, data4.Schema); - CheckSameSchemas(data1.Schema, data5.Schema); + TestCommon.CheckSameSchemas(data1.Schema, data2.Schema); + TestCommon.CheckSameSchemas(data1.Schema, data3.Schema); + TestCommon.CheckSameSchemas(data1.Schema, data4.Schema); + TestCommon.CheckSameSchemas(data1.Schema, data5.Schema); CheckSameValues(data1, data2); CheckSameValues(data1, data3); CheckSameValues(data1, data4); @@ -428,8 +429,8 @@ public void SimpleConstructorsAndExtensions() var data6 = est6.Fit(data).Transform(data); var data7 = est7.Fit(data).Transform(data); var data8 = est8.Fit(data).Transform(data); - CheckSameSchemas(data6.Schema, data7.Schema); - CheckSameSchemas(data6.Schema, data8.Schema); + TestCommon.CheckSameSchemas(data6.Schema, data7.Schema); + TestCommon.CheckSameSchemas(data6.Schema, data8.Schema); CheckSameValues(data6, data7); CheckSameValues(data6, data8); @@ -441,8 +442,8 @@ public void SimpleConstructorsAndExtensions() var data9 = est9.Fit(data).Transform(data); var data10 = est10.Fit(data).Transform(data); var data11 = est11.Fit(data).Transform(data); - CheckSameSchemas(data9.Schema, data10.Schema); - CheckSameSchemas(data9.Schema, data11.Schema); + TestCommon.CheckSameSchemas(data9.Schema, data10.Schema); + TestCommon.CheckSameSchemas(data9.Schema, data11.Schema); CheckSameValues(data9, data10); CheckSameValues(data9, data11); @@ -454,8 +455,8 @@ public void SimpleConstructorsAndExtensions() var data12 = est12.Fit(data).Transform(data); var data13 = est13.Fit(data).Transform(data); var data14 = est14.Fit(data).Transform(data); - CheckSameSchemas(data12.Schema, data13.Schema); - CheckSameSchemas(data12.Schema, data14.Schema); + TestCommon.CheckSameSchemas(data12.Schema, data13.Schema); + TestCommon.CheckSameSchemas(data12.Schema, data14.Schema); CheckSameValues(data12, data13); CheckSameValues(data12, data14); @@ -467,8 +468,8 @@ public void SimpleConstructorsAndExtensions() var data15 = est15.Fit(data).Transform(data); var data16 = est16.Fit(data).Transform(data); var data17 = est17.Fit(data).Transform(data); - CheckSameSchemas(data15.Schema, data16.Schema); - CheckSameSchemas(data15.Schema, data17.Schema); + TestCommon.CheckSameSchemas(data15.Schema, data16.Schema); + TestCommon.CheckSameSchemas(data15.Schema, data17.Schema); CheckSameValues(data15, data16); CheckSameValues(data15, data17); @@ -517,11 +518,11 @@ public void NormalizerExperimentalExtensions() var data10 = est10.Fit(data).Transform(data); // Schema Checks - CheckSameSchemas(data1.Schema, data6.Schema); - CheckSameSchemas(data2.Schema, data7.Schema); - CheckSameSchemas(data3.Schema, data8.Schema); - CheckSameSchemas(data4.Schema, data9.Schema); - CheckSameSchemas(data5.Schema, data10.Schema); + TestCommon.CheckSameSchemas(data1.Schema, data6.Schema); + TestCommon.CheckSameSchemas(data2.Schema, data7.Schema); + TestCommon.CheckSameSchemas(data3.Schema, data8.Schema); + TestCommon.CheckSameSchemas(data4.Schema, data9.Schema); + TestCommon.CheckSameSchemas(data5.Schema, data10.Schema); // Value Checks CheckSameValues(data1, data6); diff --git a/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs b/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs index 24d2c671f6..46f688f6b5 100644 --- a/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs @@ -11,6 +11,7 @@ using Microsoft.ML.Model; using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Tools; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Text; diff --git a/test/Microsoft.ML.Tests/Transformers/WordEmbeddingsTests.cs b/test/Microsoft.ML.Tests/Transformers/WordEmbeddingsTests.cs index d885e03046..d2eb38768c 100644 --- a/test/Microsoft.ML.Tests/Transformers/WordEmbeddingsTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/WordEmbeddingsTests.cs @@ -5,6 +5,7 @@ using System.IO; using Microsoft.ML.Data; using Microsoft.ML.RunTests; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Transforms.Text; using Xunit; using Xunit.Abstractions;