Fixes #4292 about using PFI with BPT and CMPB (#4306)

*Changes in PredictionTransformer.cs and Calibrator.cs to fix the problem of the create methods not being called, to make CMP load its internal calibrator and predictor first so to assign the correct paramaters types and runtimes, and added a PredictionTransformerLoadTypeAttribute so that the binary prediction transformer knows what type to assign when loading a CMP as its internal model.
*Added a working sample for using PFI with BPT and CMPB while loading a model from disk. This is based entirely in the original sample.
*Added file CalibratedModelParametersTests.cs with tests that the CMPs modified in this PR are now being correctly loaded from disk.
*Changed a couple of tests in LbfgsTests.cs that failed because they used casts that now return 'null'.

Author: antoniovs1029

docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PermutationFeatureImportanceLoadFromDisk.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Calibrators;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+
+namespace Samples.Dynamic.Trainers.BinaryClassification
+{
+    public static class PermutationFeatureImportanceLoadFromDisk
+    {
+        public static void Example()
+        {
+
+            var mlContext = new MLContext(seed: 1);
+            var samples = GenerateData();
+            var data = mlContext.Data.LoadFromEnumerable(samples);
+
+            // Create pipeline
+            var featureColumns =
+                new string[] { nameof(Data.Feature1), nameof(Data.Feature2) };
+            var pipeline = mlContext.Transforms
+                .Concatenate("Features", featureColumns)
+                .Append(mlContext.Transforms.NormalizeMinMax("Features"))
+                .Append(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression()
+                );
+
+            // Create and save model
+            var model0 = pipeline.Fit(data);
+            var lt = model0.LastTransformer;
+            var modelPath = "./model.zip";
+            mlContext.Model.Save(model0, data.Schema, modelPath);
+
+            // Load model
+            var model = mlContext.Model.Load(modelPath, out var schema);
+
+            // Transform the dataset.
+            var transformedData = model.Transform(data);
+
+            var linearPredictor = (model as TransformerChain<ITransformer>).LastTransformer as BinaryPredictionTransformer<CalibratedModelParametersBase<LinearBinaryModelParameters, PlattCalibrator>>;
+
+            // Execute PFI with the linearPredictor
+            var permutationMetrics = mlContext.BinaryClassification
+                .PermutationFeatureImportance(linearPredictor, transformedData,
+                permutationCount: 30);
+
+            // Sort indices according to PFI results
+            var sortedIndices = permutationMetrics
+                .Select((metrics, index) => new { index, metrics.AreaUnderRocCurve })
+                .OrderByDescending(
+                feature => Math.Abs(feature.AreaUnderRocCurve.Mean))
+                .Select(feature => feature.index);
+
+            Console.WriteLine("Feature\tModel Weight\tChange in AUC"
+                + "\t95% Confidence in the Mean Change in AUC");
+            var auc = permutationMetrics.Select(x => x.AreaUnderRocCurve).ToArray();
+            foreach (int i in sortedIndices)
+            {
+                Console.WriteLine("{0}\t{1:0.00}\t{2:G4}\t{3:G4}",
+                    featureColumns[i],
+                    linearPredictor.Model.SubModel.Weights[i], // this way we can access the weights inside the submodel
+                    auc[i].Mean,
+                    1.96 * auc[i].StandardError);
+            }
+
+            // Expected output:
+            //  Feature     Model Weight Change in AUC  95% Confidence in the Mean Change in AUC
+            //  Feature2        35.15     -0.387        0.002015
+            //  Feature1        17.94     -0.1514       0.0008963
+        }
+
+        private class Data
+        {
+            public bool Label { get; set; }
+
+            public float Feature1 { get; set; }
+
+            public float Feature2 { get; set; }
+        }
+
+        /// Generate Data
+        private static IEnumerable<Data> GenerateData(int nExamples = 10000,
+            double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1)
+        {
+            var rng = new Random(seed);
+            for (int i = 0; i < nExamples; i++)
+            {
+                var data = new Data
+                {
+                    Feature1 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+                    Feature2 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+                };
+
+                // Create a noisy label.
+                var value = (float)(bias + weight1 * data.Feature1 + weight2 *
+                    data.Feature2 + rng.NextDouble() - 0.5);
+
+                data.Label = Sigmoid(value) > 0.5;
+                yield return data;
+            }
+        }
+
+        private static double Sigmoid(double x) => 1.0 / (1.0 + Math.Exp(-1 * x));
+    }
+}

src/Microsoft.ML.Data/Prediction/Calibrator.cs

@@ -8,6 +8,7 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Reflection;
 using Microsoft.ML;
 using Microsoft.ML.Calibrators;
 using Microsoft.ML.CommandLine;
@@ -396,6 +397,7 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string
     }
 
     [BestFriend]
+    [PredictionTransformerLoadType(typeof(CalibratedModelParametersBase<,>))]
     internal sealed class ValueMapperCalibratedModelParameters<TSubModel, TCalibrator> :
         ValueMapperCalibratedModelParametersBase<TSubModel, TCalibrator>, ICanSaveModel
         where TSubModel : class
@@ -430,8 +432,8 @@ private static VersionInfo GetVersionInfoBulk()
                 loaderAssemblyName: typeof(ValueMapperCalibratedModelParameters<TSubModel, TCalibrator>).Assembly.FullName);
         }
 
-        private ValueMapperCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx)
-            : base(env, RegistrationName, GetPredictor(env, ctx), GetCalibrator(env, ctx))
+        private ValueMapperCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx, TSubModel predictor, TCalibrator calibrator)
+            : base(env, RegistrationName, predictor, calibrator)
         {
         }
 
@@ -443,7 +445,16 @@ private static CalibratedModelParametersBase Create(IHostEnvironment env, ModelL
             var ver2 = GetVersionInfoBulk();
             var ver = ctx.Header.ModelSignature == ver2.ModelSignature ? ver2 : ver1;
             ctx.CheckAtModel(ver);
-            return new ValueMapperCalibratedModelParameters<TSubModel, TCalibrator>(env, ctx);
+
+            // Load first the predictor and calibrator
+            var predictor = GetPredictor(env, ctx);
+            var calibrator = GetCalibrator(env, ctx);
+
+            // Create a generic type using the correct parameter types of predictor and calibrator
+            Type genericType = typeof(ValueMapperCalibratedModelParameters<,>);
+            var genericInstance = CreateCalibratedModelParameters.Create(env, ctx, predictor, calibrator, genericType);
+
+            return (CalibratedModelParametersBase)genericInstance;
         }
 
         void ICanSaveModel.Save(ModelSaveContext ctx)
@@ -456,6 +467,7 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
     }
 
     [BestFriend]
+    [PredictionTransformerLoadType(typeof(CalibratedModelParametersBase<,>))]
     internal sealed class FeatureWeightsCalibratedModelParameters<TSubModel, TCalibrator> :
         ValueMapperCalibratedModelParametersBase<TSubModel, TCalibrator>,
         IPredictorWithFeatureWeights<float>,
@@ -487,8 +499,9 @@ private static VersionInfo GetVersionInfo()
                 loaderAssemblyName: typeof(FeatureWeightsCalibratedModelParameters<TSubModel, TCalibrator>).Assembly.FullName);
         }
 
-        private FeatureWeightsCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx)
-            : base(env, RegistrationName, GetPredictor(env, ctx), GetCalibrator(env, ctx))
+        private FeatureWeightsCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx,
+            TSubModel predictor, TCalibrator calibrator)
+            : base(env, RegistrationName, predictor, calibrator)
         {
             Host.Check(SubModel is IPredictorWithFeatureWeights<float>, "Predictor does not implement " + nameof(IPredictorWithFeatureWeights<float>));
             _featureWeights = (IPredictorWithFeatureWeights<float>)SubModel;
@@ -499,7 +512,16 @@ private static CalibratedModelParametersBase Create(IHostEnvironment env, ModelL
             Contracts.CheckValue(env, nameof(env));
             env.CheckValue(ctx, nameof(ctx));
             ctx.CheckAtModel(GetVersionInfo());
-            return new FeatureWeightsCalibratedModelParameters<TSubModel, TCalibrator>(env, ctx);
+
+            // Load first the predictor and calibrator
+            var predictor = GetPredictor(env, ctx);
+            var calibrator = GetCalibrator(env, ctx);
+
+            // Create a generic type using the correct parameter types of predictor and calibrator
+            Type genericType = typeof(FeatureWeightsCalibratedModelParameters<,>);
+            var genericInstance = CreateCalibratedModelParameters.Create(env, ctx, predictor, calibrator, genericType);
+
+            return (CalibratedModelParametersBase) genericInstance;
         }
 
         void ICanSaveModel.Save(ModelSaveContext ctx)
@@ -520,6 +542,7 @@ public void GetFeatureWeights(ref VBuffer<float> weights)
     /// Encapsulates a predictor and a calibrator that implement <see cref="IParameterMixer"/>.
     /// Its implementation of <see cref="IParameterMixer.CombineParameters"/> combines both the predictors and the calibrators.
     /// </summary>
+    [PredictionTransformerLoadType(typeof(CalibratedModelParametersBase <,>))]
     internal sealed class ParameterMixingCalibratedModelParameters<TSubModel, TCalibrator> :
         ValueMapperCalibratedModelParametersBase<TSubModel, TCalibrator>,
         IParameterMixer<float>,
@@ -553,8 +576,8 @@ private static VersionInfo GetVersionInfo()
                 loaderAssemblyName: typeof(ParameterMixingCalibratedModelParameters<TSubModel, TCalibrator>).Assembly.FullName);
         }
 
-        private ParameterMixingCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx)
-            : base(env, RegistrationName, GetPredictor(env, ctx), GetCalibrator(env, ctx))
+        private ParameterMixingCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx, TSubModel predictor, TCalibrator calibrator)
+            : base(env, RegistrationName, predictor, calibrator)
         {
             Host.Check(SubModel is IParameterMixer<float>, "Predictor does not implement " + nameof(IParameterMixer));
             Host.Check(SubModel is IPredictorWithFeatureWeights<float>, "Predictor does not implement " + nameof(IPredictorWithFeatureWeights<float>));
@@ -566,7 +589,16 @@ private static CalibratedModelParametersBase Create(IHostEnvironment env, ModelL
             Contracts.CheckValue(env, nameof(env));
             env.CheckValue(ctx, nameof(ctx));
             ctx.CheckAtModel(GetVersionInfo());
-            return new ParameterMixingCalibratedModelParameters<TSubModel, TCalibrator>(env, ctx);
+
+            // Load first the predictor and calibrator
+            var predictor = GetPredictor(env, ctx);
+            var calibrator = GetCalibrator(env, ctx);
+
+            // Create a generic type using the correct parameter types of predictor and calibrator
+            Type genericType = typeof(ParameterMixingCalibratedModelParameters<,>);
+            object genericInstance = CreateCalibratedModelParameters.Create(env, ctx, predictor, calibrator, genericType);
+
+            return (CalibratedModelParametersBase) genericInstance;
         }
 
         void ICanSaveModel.Save(ModelSaveContext ctx)
@@ -777,6 +809,28 @@ ValueMapper<TSrc, VBuffer<float>> IFeatureContributionMapper.GetFeatureContribut
         }
     }
 
+    internal static class CreateCalibratedModelParameters
+    {
+        internal static object Create(IHostEnvironment env, ModelLoadContext ctx, object predictor, ICalibrator calibrator, Type calibratedModelParametersType)
+        {
+            Type[] genericTypeArgs = { predictor.GetType(), calibrator.GetType() };
+            Type constructed = calibratedModelParametersType.MakeGenericType(genericTypeArgs);
+
+            Type[] constructorArgs = {
+                typeof(IHostEnvironment),
+                typeof(ModelLoadContext),
+                predictor.GetType(),
+                calibrator.GetType()
+            };
+
+            // Call the appropiate constructor of the created generic type passing on the previously loaded predictor and calibrator
+            var genericCtor = constructed.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, constructorArgs, null);
+            object genericInstance = genericCtor.Invoke(new object[] { env, ctx, predictor, calibrator });
+
+            return genericInstance;
+        }
+    }
+
     [BestFriend]
     internal static class CalibratorUtils
     {