diff --git a/src/Microsoft.ML.Data/Model/ModelOperationsCatalog.cs b/src/Microsoft.ML.Data/Model/ModelOperationsCatalog.cs index dc4cb551d0..c4fa3d71a2 100644 --- a/src/Microsoft.ML.Data/Model/ModelOperationsCatalog.cs +++ b/src/Microsoft.ML.Data/Model/ModelOperationsCatalog.cs @@ -182,12 +182,25 @@ public ITransformer Load(Stream stream, out DataViewSchema inputSchema) } /// - /// Load the model and its input schema from the stream. + /// Load the model and its input schema from the file. + /// + /// Path to model. + /// Will contain the input schema for the model. If the model was saved using older APIs + /// it may not contain an input schema, in this case will be null. + /// The loaded model. + public ITransformer Load(string modelPath, out DataViewSchema inputSchema) + { + using (var stream = File.OpenRead(modelPath)) + return Load(stream, out inputSchema); + } + + /// + /// Load the model from the stream. /// /// A readable, seekable stream to load from. /// A model of type containing the loader /// and the transformer chain. - public IDataLoader Load(Stream stream) + public IDataLoader LoadDataLoader(Stream stream) { _env.CheckValue(stream, nameof(stream)); @@ -205,6 +218,18 @@ public IDataLoader Load(Stream stream) } } + /// + /// Load the model from the file. + /// + /// Path to model. + /// A model of type containing the loader + /// and the transformer chain. + public IDataLoader LoadDataLoader(string modelPath) + { + using (var stream = File.OpenRead(modelPath)) + return LoadDataLoader(stream); + } + /// /// Load a transformer model and a data loader model from the stream. /// @@ -215,7 +240,7 @@ public ITransformer LoadWithDataLoader(Stream stream, out IDataLoader composite) { loader = composite.Loader; @@ -224,6 +249,18 @@ public ITransformer LoadWithDataLoader(Stream stream, out IDataLoader(); } + /// + /// Load a transformer model and a data loader model from the file. + /// + /// Path to model. + /// The data loader from the model stream. + /// The transformer model from the model stream. + public ITransformer LoadWithDataLoader(string modelPath, out IDataLoader loader) + { + using (var stream = File.OpenRead(modelPath)) + return LoadWithDataLoader(stream, out loader); + } + /// /// Create a prediction engine for one-time prediction. /// diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 0a36b2af62..656ae47840 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -5643,10 +5643,7 @@ public void LoadEntryPointModel() { var modelPath = GetDataPath($"backcompat/ep_model{i}.zip"); ITransformer loadedModel; - using (var stream = File.OpenRead(modelPath)) - { - loadedModel = ml.Model.Load(stream, out var inputSchema); - } + loadedModel = ml.Model.Load(modelPath, out var inputSchema); } } } diff --git a/test/Microsoft.ML.Functional.Tests/ModelLoading.cs b/test/Microsoft.ML.Functional.Tests/ModelLoading.cs index 193ddedad5..25d78c2236 100644 --- a/test/Microsoft.ML.Functional.Tests/ModelLoading.cs +++ b/test/Microsoft.ML.Functional.Tests/ModelLoading.cs @@ -68,24 +68,19 @@ public void LoadModelAndExtractPredictor() ITransformer loadedTransformerModel; IDataLoader loadedCompositeLoader; ITransformer loadedTransformerModel1; - using (var fs = File.OpenRead(modelAndSchemaPath)) - loadedTransformerModel = _ml.Model.Load(fs, out var loadedSchema); - using (var fs = File.OpenRead(compositeLoaderModelPath)) - { - // This model can be loaded either as a composite data loader, - // a transformer model + an input schema, or a transformer model + a data loader. - var t = _ml.Model.LoadWithDataLoader(fs, out IDataLoader l); - var t1 = _ml.Model.Load(fs, out var s); - loadedCompositeLoader = _ml.Model.Load(fs); - } - using (var fs = File.OpenRead(loaderAndTransformerModelPath)) - { - // This model can be loaded either as a composite data loader, - // a transformer model + an input schema, or a transformer model + a data loader. - var t = _ml.Model.Load(fs, out var s); - var c = _ml.Model.Load(fs); - loadedTransformerModel1 = _ml.Model.LoadWithDataLoader(fs, out IDataLoader l); - } + loadedTransformerModel = _ml.Model.Load(modelAndSchemaPath, out var loadedSchema); + + // This model can be loaded either as a composite data loader, + // a transformer model + an input schema, or a transformer model + a data loader. + var t = _ml.Model.LoadWithDataLoader(compositeLoaderModelPath, out IDataLoader l); + var t1 = _ml.Model.Load(compositeLoaderModelPath, out var s); + loadedCompositeLoader = _ml.Model.LoadDataLoader(compositeLoaderModelPath); + + // This model can be loaded either as a composite data loader, + // a transformer model + an input schema, or a transformer model + a data loader. + var tt = _ml.Model.Load(loaderAndTransformerModelPath, out var ss); + var c = _ml.Model.LoadDataLoader(loaderAndTransformerModelPath); + loadedTransformerModel1 = _ml.Model.LoadWithDataLoader(loaderAndTransformerModelPath, out IDataLoader ll); var gam = ((loadedTransformerModel as ISingleFeaturePredictionTransformer).Model as CalibratedModelParametersBase).SubModel @@ -125,11 +120,8 @@ public void SaveAndLoadModelWithLoader() IDataLoader loadedModel; ITransformer loadedModelWithoutLoader; DataViewSchema loadedSchema; - using (var fs = File.OpenRead(modelPath)) - { - loadedModel = _ml.Model.Load(fs); - loadedModelWithoutLoader = _ml.Model.Load(fs, out loadedSchema); - } + loadedModel = _ml.Model.LoadDataLoader(modelPath); + loadedModelWithoutLoader = _ml.Model.Load(modelPath, out loadedSchema); // Without deserializing the loader from the model we lose the slot names. data = _ml.Data.LoadFromEnumerable(new[] { new InputData() }); @@ -171,8 +163,7 @@ public void LoadSchemaAndCreateNewData() ITransformer loadedModel; DataViewSchema loadedSchema; - using (var fs = File.OpenRead(modelPath)) - loadedModel = _ml.Model.Load(fs, out loadedSchema); + loadedModel = _ml.Model.Load(modelPath, out loadedSchema); // Without using the schema from the model we lose the slot names. data = _ml.Data.LoadFromEnumerable(new[] { new InputData() }); @@ -298,26 +289,24 @@ private void Load(string filename, out ITransformer loadedWithSchema, out DataVi out IDataLoader loadedLoader, out ITransformer loadedWithLoader, out IDataLoader loadedLoaderWithTransformer) { - using (var fs = File.OpenRead(filename)) + + try + { + loadedLoader = _ml.Model.LoadDataLoader(filename); + } + catch (Exception) + { + loadedLoader = null; + } + loadedWithSchema = _ml.Model.Load(filename, out loadedSchema); + try + { + loadedWithLoader = _ml.Model.LoadWithDataLoader(filename, out loadedLoaderWithTransformer); + } + catch (Exception) { - try - { - loadedLoader = _ml.Model.Load(fs); - } - catch (Exception) - { - loadedLoader = null; - } - loadedWithSchema = _ml.Model.Load(fs, out loadedSchema); - try - { - loadedWithLoader = _ml.Model.LoadWithDataLoader(fs, out loadedLoaderWithTransformer); - } - catch (Exception) - { - loadedWithLoader = null; - loadedLoaderWithTransformer = null; - } + loadedWithLoader = null; + loadedLoaderWithTransformer = null; } } diff --git a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs index fe813dde2a..12486d6bf7 100644 --- a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs +++ b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipeBase.cs @@ -83,8 +83,7 @@ protected void TestEstimatorCore(IEstimator estimator, ITransformer loadedTransformer; DataViewSchema loadedInputSchema; - using (var fs = File.OpenRead(modelPath)) - loadedTransformer = ML.Model.Load(fs, out loadedInputSchema); + loadedTransformer = ML.Model.Load(modelPath, out loadedInputSchema); DeleteOutputPath(modelPath); // Run on train data. diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamples.cs b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamples.cs index fabcb41205..cc8d841c69 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamples.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamples.cs @@ -153,8 +153,7 @@ private void TrainRegression(string trainDataPath, string testDataPath, string m // When you load the model, it's a 'dynamic' transformer. ITransformer loadedModel; - using (var stream = File.OpenRead(modelPath)) - loadedModel = mlContext.Model.Load(stream, out var schema); + loadedModel = mlContext.Model.Load(modelPath, out var schema); } [Fact] diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs index 63f25dcbce..7abd15d5df 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs @@ -127,8 +127,7 @@ private void TrainRegression(string trainDataPath, string testDataPath, string m // When you load the model, it's a 'dynamic' transformer. ITransformer loadedModel; - using (var stream = File.OpenRead(modelPath)) - loadedModel = mlContext.Model.Load(stream, out var schema); + loadedModel = mlContext.Model.Load(modelPath, out var schema); } [Fact] @@ -530,8 +529,7 @@ private static void RunEndToEnd(MLContext mlContext, IDataView trainData, string // Now we can load the model. ITransformer loadedModel; - using (var fs = File.OpenRead(modelPath)) - loadedModel = newContext.Model.Load(fs, out var schema); + loadedModel = newContext.Model.Load(modelPath, out var schema); } public static IDataView PrepareData(MLContext mlContext, IDataView data) diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainSaveModelAndPredict.cs b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainSaveModelAndPredict.cs index b934c57329..7f9c43a933 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainSaveModelAndPredict.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/Estimators/TrainSaveModelAndPredict.cs @@ -40,8 +40,7 @@ public void TrainSaveModelAndPredict() // Load model. ITransformer loadedModel; DataViewSchema inputSchema; - using (var file = File.OpenRead(modelPath)) - loadedModel = ml.Model.Load(file, out inputSchema); + loadedModel = ml.Model.Load(modelPath, out inputSchema); // Create prediction engine and test predictions. var engine = ml.Model.CreatePredictionEngine(loadedModel, inputSchema); diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 5d1cc5e9c5..52ac76dce5 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -503,12 +503,9 @@ public void MatrixFactorizationBackCompat() ITransformer model; using (var ch = Env.Start("load")) { - using (var fs = File.OpenRead(modelPath)) - { - model = ML.Model.Load(fs, out var schema); - // This model was saved without the input schema. - Assert.Null(schema); - } + model = ML.Model.Load(modelPath, out var schema); + // This model was saved without the input schema. + Assert.Null(schema); } // Create data for testing. Note that the 2nd element is not specified in the training data so it should diff --git a/test/Microsoft.ML.Tests/Transformers/ConvertTests.cs b/test/Microsoft.ML.Tests/Transformers/ConvertTests.cs index e0167a241e..b3ef470a75 100644 --- a/test/Microsoft.ML.Tests/Transformers/ConvertTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/ConvertTests.cs @@ -260,8 +260,7 @@ public void TypeConvertKeyBackCompatTest() ITransformer modelOld; using (var ch = Env.Start("load")) { - using (var fs = File.OpenRead(modelPath)) - modelOld = ML.Model.Load(fs, out var schema); + modelOld = ML.Model.Load(modelPath, out var schema); } var outDataOld = modelOld.Transform(dataView); diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 05055b5613..382871a986 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -178,8 +178,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn() //Load time series model and we will use this to pass two inputs and compare the raw score //with "engine". ITransformer model2 = null; - using (var file = File.OpenRead(modelPath)) - model2 = ml.Model.Load(file, out var schema); + model2 = ml.Model.Load(modelPath, out var schema); //Raw score after state gets updated with two inputs. var engine2 = model2.CreateTimeSeriesPredictionFunction(ml); @@ -198,8 +197,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn() //Save the model with state updated with just one input. engine.CheckPoint(ml, modelPath + 1); ITransformer model3 = null; - using (var file = File.OpenRead(modelPath + 1)) - model3 = ml.Model.Load(file, out var schema); + model3 = ml.Model.Load(modelPath + 1, out var schema1); //Load the model with state updated with just one input, then pass in the second input //and raw score should match the raw score obtained by passing the two input in the first model. @@ -265,8 +263,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngine() // Load Model 1. ITransformer model2 = null; - using (var file = File.OpenRead(modelPath)) - model2 = ml.Model.Load(file, out var schema); + model2 = ml.Model.Load(modelPath, out var schema); //Predict and expect the same result after checkpointing(Prediction #2). engine = model2.CreateTimeSeriesPredictionFunction(ml);