Add API for saving/loading input schema

Author: yaeldMS

docs/samples/Microsoft.ML.Samples/Dynamic/IidChangePointDetectorTransform.cs

@@ -138,7 +138,7 @@ public static void IidChangePointDetectorPrediction()
 
             // Load the model.
             using (var file = File.OpenRead(modelPath))
-                model = ml.Model.Load(file);
+                model = ml.Model.Load(file, out var schema);
 
             // Create a time series prediction engine from the checkpointed model.
             engine = model.CreateTimeSeriesPredictionFunction<IidChangePointData, ChangePointPrediction>(ml);

docs/samples/Microsoft.ML.Samples/Dynamic/IidSpikeDetectorTransform.cs

@@ -122,7 +122,7 @@ public static void IidSpikeDetectorPrediction()
 
             // Load the model.
             using (var file = File.OpenRead(modelPath))
-                model = ml.Model.Load(file);
+                model = ml.Model.Load(file, out var schema);
 
             for (int index = 0; index < 5; index++)
             {

docs/samples/Microsoft.ML.Samples/Dynamic/SsaChangePointDetectorTransform.cs

@@ -142,7 +142,7 @@ public static void SsaChangePointDetectorPrediction()
 
             // Load the model.
             using (var file = File.OpenRead(modelPath))
-                model = ml.Model.Load(file);
+                model = ml.Model.Load(file, out var schema);
 
             // We must create a new prediction engine from the persisted model.
             engine = model.CreateTimeSeriesPredictionFunction<SsaChangePointData, ChangePointPrediction>(ml);

docs/samples/Microsoft.ML.Samples/Dynamic/SsaSpikeDetectorTransform.cs

@@ -150,7 +150,7 @@ public static void SsaSpikeDetectorPrediction()
 
             // Load the model.
             using (var file = File.OpenRead(modelPath))
-                model = ml.Model.Load(file);
+                model = ml.Model.Load(file, out var schema);
 
             // We must create a new prediction engine from the persisted model.
             engine = model.CreateTimeSeriesPredictionFunction<SsaSpikeData, SsaSpikePrediction>(ml);

src/Microsoft.ML.Data/DataLoadSave/CompositeDataLoader.cs

@@ -19,10 +19,6 @@ namespace Microsoft.ML.Data
     public sealed class CompositeDataLoader<TSource, TLastTransformer> : IDataLoader<TSource>
         where TLastTransformer : class, ITransformer
     {
-        private const string LoaderDirectory = "Loader";
-        private const string LegacyLoaderDirectory = "Reader";
-        private const string TransformerDirectory = TransformerChain.LoaderSignature;
-
         /// <summary>
         /// The underlying data loader.
         /// </summary>
@@ -43,9 +39,9 @@ public CompositeDataLoader(IDataLoader<TSource> loader, TransformerChain<TLastTr
 
         private CompositeDataLoader(IHost host, ModelLoadContext ctx)
         {
-            if (!ctx.LoadModelOrNull<IDataLoader<TSource>, SignatureLoadModel>(host, out Loader, LegacyLoaderDirectory))
-                ctx.LoadModel<IDataLoader<TSource>, SignatureLoadModel>(host, out Loader, LoaderDirectory);
-            ctx.LoadModel<TransformerChain<TLastTransformer>, SignatureLoadModel>(host, out Transformer, TransformerDirectory);
+            if (!ctx.LoadModelOrNull<IDataLoader<TSource>, SignatureLoadModel>(host, out Loader, ModelOperationsCatalog.LegacyLoaderDirectory))
+                ctx.LoadModel<IDataLoader<TSource>, SignatureLoadModel>(host, out Loader, ModelOperationsCatalog.LoaderDirectory);
+            ctx.LoadModel<TransformerChain<TLastTransformer>, SignatureLoadModel>(host, out Transformer, ModelOperationsCatalog.TransformerDirectory);
         }
 
         private static CompositeDataLoader<TSource, TLastTransformer> Create(IHostEnvironment env, ModelLoadContext ctx)
@@ -94,8 +90,8 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
             ctx.CheckAtModel();
             ctx.SetVersionInfo(GetVersionInfo());
 
-            ctx.SaveModel(Loader, LoaderDirectory);
-            ctx.SaveModel(Transformer, TransformerDirectory);
+            ctx.SaveModel(Loader, ModelOperationsCatalog.LoaderDirectory);
+            ctx.SaveModel(Transformer, ModelOperationsCatalog.TransformerDirectory);
         }
 
         internal const string Summary = "A loader that encapsulates a loader and a transformer chain.";

src/Microsoft.ML.Data/DataLoadSave/DataOperationsCatalog.cs

@@ -57,6 +57,14 @@ public IDataView LoadFromEnumerable<TRow>(IEnumerable<TRow> data, SchemaDefiniti
             return DataViewConstructionUtils.CreateFromEnumerable(_env, data, schemaDefinition);
         }
 
+        public IDataView LoadFromEnumerable<TRow>(IEnumerable<TRow> data, DataViewSchema schema)
+            where TRow : class
+        {
+            _env.CheckValue(data, nameof(data));
+            _env.CheckValue(schema, nameof(schema));
+            return DataViewConstructionUtils.CreateFromEnumerable(_env, data, schema);
+        }
+
         /// <summary>
         /// Convert an <see cref="IDataView"/> into a strongly-typed <see cref="IEnumerable{TRow}"/>.
         /// </summary>

src/Microsoft.ML.Data/DataLoadSave/TransformerChain.cs

@@ -249,15 +249,15 @@ private static TransformerChain<ITransformer> Create(IHostEnvironment env, Model
         public static void SaveTo(this ITransformer transformer, IHostEnvironment env, Stream outputStream)
             => new TransformerChain<ITransformer>(transformer).SaveTo(env, outputStream);
 
-        public static TransformerChain<ITransformer> LoadFrom(IHostEnvironment env, Stream stream)
+        public static ITransformer LoadFrom(IHostEnvironment env, Stream stream)
         {
             using (var rep = RepositoryReader.Open(stream, env))
             {
                 try
                 {
-                    ModelLoadContext.LoadModelOrNull<TransformerChain<ITransformer>, SignatureLoadModel>(env, out var transformerChain, rep, LoaderSignature);
+                    ModelLoadContext.LoadModelOrNull<ITransformer, SignatureLoadModel>(env, out var transformerChain, rep, LoaderSignature);
                     if (transformerChain == null)
-                        ModelLoadContext.LoadModel<TransformerChain<ITransformer>, SignatureLoadModel>(env, out transformerChain, rep, $@"Model\{LoaderSignature}");
+                        ModelLoadContext.LoadModel<ITransformer, SignatureLoadModel>(env, out transformerChain, rep, $@"Model\{LoaderSignature}");
                     return transformerChain;
                 }
                 catch (FormatException ex)

src/Microsoft.ML.Data/DataView/DataViewConstructionUtils.cs

@@ -45,6 +45,50 @@ public static StreamingDataView<TRow> CreateFromEnumerable<TRow>(IHostEnvironmen
             return new StreamingDataView<TRow>(env, data, internalSchemaDefn);
         }
 
+        public static StreamingDataView<TRow> CreateFromEnumerable<TRow>(IHostEnvironment env, IEnumerable<TRow> data,
+            DataViewSchema schema)
+            where TRow : class
+        {
+            Contracts.AssertValue(env);
+            env.AssertValue(data);
+            env.AssertValueOrNull(schema);
+            schema = schema ?? new DataViewSchema.Builder().ToSchema();
+            return new StreamingDataView<TRow>(env, data, GetInternalSchemaDefinition<TRow>(env, schema));
+        }
+
+        private static InternalSchemaDefinition GetInternalSchemaDefinition<TRow>(IHostEnvironment env, DataViewSchema schema)
+            where TRow : class
+        {
+            Contracts.AssertValue(env);
+            env.AssertValue(schema);
+
+            var isd = InternalSchemaDefinition.Create(typeof(TRow), SchemaDefinition.Direction.Read);
+            foreach (var col in schema)
+            {
+                var name = col.Name;
+                var isdCol = isd.Columns.FirstOrDefault(c => c.ColumnName == name);
+                if (isdCol == null)
+                    throw env.Except($"Type should contain a member named {isdCol.ColumnName}");
+                var annotations = col.Annotations;
+                if (annotations != null)
+                {
+                    foreach (var annotation in annotations.Schema)
+                    {
+                        var info = Utils.MarshalInvoke(GetAnnotationInfo<int>, annotation.Type.RawType, annotation.Name, annotations);
+                        isdCol.Annotations.Add(annotation.Name, info);
+                    }
+                }
+            }
+            return isd;
+        }
+
+        private static AnnotationInfo GetAnnotationInfo<T>(string kind, DataViewSchema.Annotations annotations)
+        {
+            T value = default;
+            annotations.GetValue(kind, ref value);
+            return new AnnotationInfo<T>(kind, value);
+        }
+
         public static InputRow<TRow> CreateInputRow<TRow>(IHostEnvironment env, SchemaDefinition schemaDefinition = null)
             where TRow : class
         {
@@ -604,7 +648,7 @@ public StreamingDataView(IHostEnvironment env, IEnumerable<TRow> data, InternalS
             public override DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
             {
                 var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, Schema);
-                return new WrappedCursor (new Cursor(Host, this, predicate));
+                return new WrappedCursor(new Cursor(Host, this, predicate));
             }
 
             private sealed class Cursor : DataViewCursorBase
@@ -674,7 +718,7 @@ public override DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column
             {
                 Contracts.Assert(_current != null, "The current object must be set prior to cursoring");
                 var predicate = RowCursorUtils.FromColumnsToPredicate(columnsNeeded, Schema);
-                return new WrappedCursor (new Cursor(Host, this, predicate));
+                return new WrappedCursor(new Cursor(Host, this, predicate));
             }
 
             private sealed class Cursor : DataViewCursorBase

src/Microsoft.ML.Data/Model/ModelOperationsCatalog.cs

@@ -2,8 +2,12 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
 using System.IO;
+using Microsoft.Data.DataView;
 using Microsoft.ML.Data;
+using Microsoft.ML.Data.IO;
+using Microsoft.ML.Model;
 
 namespace Microsoft.ML
 {
@@ -12,6 +16,11 @@ namespace Microsoft.ML
     /// </summary>
     public sealed class ModelOperationsCatalog : IInternalCatalog
     {
+        internal const string LoaderDirectory = "Loader";
+        internal const string LegacyLoaderDirectory = "Reader";
+        internal const string TransformerDirectory = TransformerChain.LoaderSignature;
+        internal const string SchemaEntryName = "Schema";
+
         IHostEnvironment IInternalCatalog.Environment => _env;
         private readonly IHostEnvironment _env;
 
@@ -30,28 +39,93 @@ internal ModelOperationsCatalog(IHostEnvironment env)
         /// </summary>
         /// <param name="model">The trained model to be saved.</param>
         /// <param name="stream">A writeable, seekable stream to save to.</param>
-        public void Save(ITransformer model, Stream stream) => model.SaveTo(_env, stream);
-
         public void Save<TSource>(IDataLoader<TSource> model, Stream stream)
         {
             using (var rep = RepositoryWriter.CreateNew(stream))
             {
                 ModelSaveContext.SaveModel(rep, model, "Model");
+                SaveInputSchema(model.GetOutputSchema(), rep);
                 rep.Commit();
             }
         }
 
+        /// <summary>
+        /// Save a transformer model and the loader used to create its input data to the stream.
+        /// </summary>
+        /// <param name="loader">The loader that was used to create data to train the model</param>
+        /// <param name="model">The trained model to be saved</param>
+        /// <param name="stream">A writeable, seekable stream to save to.</param>
         public void Save<TSource>(IDataLoader<TSource> loader, ITransformer model, Stream stream) =>
             Save(new CompositeDataLoader<TSource, ITransformer>(loader, new TransformerChain<ITransformer>(model)), stream);
 
         /// <summary>
-        /// Load the model from the stream.
+        /// Save a transformer model and the schema of the data that was used to train it to the stream.
+        /// </summary>
+        /// <param name="inputSchema">The schema of the input to the transformer.</param>
+        /// <param name="model">The trained model to be saved.</param>
+        /// <param name="stream">A writeable, seekable stream to save to.</param>
+        public void Save(DataViewSchema inputSchema, ITransformer model, Stream stream)
+        {
+            using (var rep = RepositoryWriter.CreateNew(stream))
+            {
+                ModelSaveContext.SaveModel(rep, model, TransformerDirectory);
+                SaveInputSchema(inputSchema, rep);
+                rep.Commit();
+            }
+        }
+
+        private void SaveInputSchema(DataViewSchema inputSchema, RepositoryWriter rep)
+        {
+            using (var ch = _env.Start("Saving Schema"))
+            {
+                var entry = rep.CreateEntry(SchemaEntryName);
+                var saver = new BinarySaver(_env, new BinarySaver.Arguments { Silent = true });
+                DataSaverUtils.SaveDataView(ch, saver, new EmptyDataView(_env, inputSchema), entry.Stream, keepHidden: true);
+            }
+        }
+
+        /// <summary>
+        /// Load the model and its input schema from the stream.
         /// </summary>
         /// <param name="stream">A readable, seekable stream to load from.</param>
+        /// <param name="inputSchema">Will contain the input schema for the model.</param>
         /// <returns>The loaded model.</returns>
-        public ITransformer Load(Stream stream) => TransformerChain.LoadFrom(_env, stream);
+        public ITransformer Load(Stream stream, out DataViewSchema inputSchema)
+        {
+            using (var rep = RepositoryReader.Open(stream, _env))
+            {
+                var entry = rep.OpenEntryOrNull(SchemaEntryName);
+                if (entry != null)
+                {
+                    var loader = new BinaryLoader(_env, new BinaryLoader.Arguments(), entry.Stream);
+                    inputSchema = loader.Schema;
+                }
+                else
+                {
+                    // Try to load from legacy model format.
+                    try
+                    {
+                        var loader = ModelFileUtils.LoadLoader(_env, rep, new MultiFileSource(null), false);
+                        inputSchema = loader.Schema;
+                    }
+                    catch (Exception ex)
+                    {
+                        if (!ex.IsMarked())
+                            throw;
+                        inputSchema = null;
+                    }
+                }
+                return TransformerChain.LoadFrom(_env, stream);
+            }
+        }
 
-        public CompositeDataLoader<IMultiStreamSource, ITransformer> LoadAsCompositeDataLoader(Stream stream)
+        /// <summary>
+        /// Load the model and its input schema from the stream.
+        /// </summary>
+        /// <param name="stream">A readable, seekable stream to load from.</param>
+        /// <returns>A model of type <see cref="CompositeDataLoader{IMultiStreamSource, ITransformer}"/> containing the loader
+        /// and the transformer chain.</returns>
+        public CompositeDataLoader<IMultiStreamSource, ITransformer> Load(Stream stream)
         {
             using (var rep = RepositoryReader.Open(stream))
             {

test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs

@@ -5615,7 +5615,7 @@ public void LoadEntryPointModel()
                 ITransformer loadedModel;
                 using (var stream = File.OpenRead(modelPath))
                 {
-                    loadedModel = ml.Model.Load(stream);
+                    loadedModel = ml.Model.Load(stream, out var inputSchema);
                 }
 
             }