Update tests.

Author: codemzs

src/Microsoft.ML.Data/MLContext.cs

@@ -46,6 +46,11 @@ public sealed class MLContext : IHostEnvironment
         /// </summary>
         public AnomalyDetectionCatalog AnomalyDetection { get; }
 
+        /// <summary>
+        /// Trainers and tasks specific to forecasting problems.
+        /// </summary>
+        public ForecastingCatalog Forecasting { get; }
+
         /// <summary>
         /// Data processing operations.
         /// </summary>
@@ -113,6 +118,7 @@ public MLContext(int? seed = null)
             Clustering = new ClusteringCatalog(_env);
             Ranking = new RankingCatalog(_env);
             AnomalyDetection = new AnomalyDetectionCatalog(_env);
+            Forecasting = new ForecastingCatalog(_env);
             Transforms = new TransformsCatalog(_env);
             Model = new ModelOperationsCatalog(_env);
             Data = new DataOperationsCatalog(_env);

src/Microsoft.ML.Data/TrainCatalog.cs

@@ -704,4 +704,31 @@ public AnomalyDetectionMetrics Evaluate(IDataView data, string labelColumnName =
             return eval.Evaluate(data, labelColumnName, scoreColumnName, predictedLabelColumnName);
         }
     }
+
+    /// <summary>
+    /// Class used by <see cref="MLContext"/> to create instances of forecasting components.
+    /// </summary>
+    public sealed class ForecastingCatalog : TrainCatalogBase
+    {
+        /// <summary>
+        /// The list of trainers for performing forecasting.
+        /// </summary>
+        public Forecasters Trainers { get; }
+
+        internal ForecastingCatalog(IHostEnvironment env) : base(env, nameof(ForecastingCatalog))
+        {
+            Trainers = new Forecasters(this);
+        }
+
+        /// <summary>
+        /// Class used by <see cref="MLContext"/> to create instances of forecasting trainers.
+        /// </summary>
+        public sealed class Forecasters : CatalogInstantiatorBase
+        {
+            internal Forecasters(ForecastingCatalog catalog)
+                : base(catalog)
+            {
+            }
+        }
+    }
 }

src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs

@@ -179,7 +179,7 @@ public static SrCnnAnomalyEstimator DetectAnomalyBySrCnn(this TransformsCatalog
         /// </format>
         /// </example>
         public static SsaForecastingEstimator ForecastBySsa(
-            this TransformsCatalog catalog, string outputColumnName, string inputColumnName, int windowSize, int seriesLength, int trainSize, int horizon,
+            this ForecastingCatalog catalog, string outputColumnName, string inputColumnName, int windowSize, int seriesLength, int trainSize, int horizon,
             bool isAdaptive = false, float discountFactor = 1, RankSelectionMethod rankSelectionMethod = RankSelectionMethod.Exact, int? rank = null,
             int? maxRank = null, bool shouldStablize = true, bool shouldMaintainInfo = false, GrowthRatio? maxGrowth = null, string forecastingConfidenceIntervalMinOutputColumnName = null,
             string forecastingConfidenceIntervalMaxOutputColumnName = null, float confidenceLevel = 0.95f) =>

src/Microsoft.ML.TimeSeries/SSaForecasting.cs

@@ -224,7 +224,6 @@ private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, Dat
     /// ]]>
     /// </format>
     /// </remarks>
-    /// <seealso cref="Microsoft.ML.TimeSeriesCatalog.ForecastBySsa(TransformsCatalog, string, string, int, int, int, int, bool, float, RankSelectionMethod, int?, int?, bool, bool, GrowthRatio?, string, string, float)" />
     public sealed class SsaForecastingEstimator : IEstimator<SsaForecasting>
     {
         private readonly IHost _host;

src/Microsoft.ML.TimeSeries/SequentialForecastingTransformBase.cs

@@ -256,13 +256,13 @@ private protected sealed override void InitializeStateCore(bool disk = false)
             {
                 Parent = (SequentialForecastingTransformBase<TInput, TState>)ParentTransform;
                 Host.Assert(WindowSize >= 0);
-                InitializeAnomalyDetector();
+                InitializeForecaster();
             }
 
             /// <summary>
-            /// The abstract method that realizes the initialization functionality for the anomaly detector.
+            /// The abstract method that realizes the initialization functionality for the forecaster.
             /// </summary>
-            private protected abstract void InitializeAnomalyDetector();
+            private protected abstract void InitializeForecaster();
         }
     }
 }

src/Microsoft.ML.TimeSeries/SequentialTransformerBase.cs

@@ -168,6 +168,7 @@ public void UpdateStateCore(ref TInput input, bool buffer = true)
 
             public void Process(ref TInput input, ref TOutput output1, ref TOutput output2, ref TOutput output3)
             {
+                //Using prediction engine will not evaluate the below condition to true.
                 if (PreviousPosition == -1)
                     UpdateStateCore(ref input);
 
@@ -186,8 +187,9 @@ public void Process(ref TInput input, ref TOutput output1, ref TOutput output2,
 
             public void ProcessWithoutBuffer(ref TInput input, ref TOutput output1, ref TOutput output2, ref TOutput output3)
             {
+                //Using prediction engine will not evaluate the below condition to true.
                 if (PreviousPosition == -1)
-                    UpdateStateCore(ref input, false);
+                    UpdateStateCore(ref input);
 
                 if (InitialWindowedBuffer.Count < InitialWindowSize)
                 {
@@ -202,6 +204,7 @@ public void ProcessWithoutBuffer(ref TInput input, ref TOutput output1, ref TOut
 
             public void Process(ref TInput input, ref TOutput output)
             {
+                //Using prediction engine will not evaluate the below condition to true.
                 if (PreviousPosition == -1)
                     UpdateStateCore(ref input);
 
@@ -220,8 +223,9 @@ public void Process(ref TInput input, ref TOutput output)
 
             public void ProcessWithoutBuffer(ref TInput input, ref TOutput output)
             {
+                //Using prediction engine will not evaluate the below condition to true.
                 if (PreviousPosition == -1)
-                    UpdateStateCore(ref input, false);
+                    UpdateStateCore(ref input);
 
                 if (InitialWindowedBuffer.Count < InitialWindowSize)
                 {

src/Microsoft.ML.TimeSeries/SsaForecastingBase.cs

@@ -250,7 +250,7 @@ internal void SaveThis(ModelSaveContext ctx)
             internal sealed class State : ForecastingStateBase
             {
                 private SequenceModelerBase<Single, Single> _model;
-                private SsaForecastingBase _parentAnomalyDetector;
+                private SsaForecastingBase _parentForecaster;
 
                 public State()
                 {
@@ -278,8 +278,8 @@ private protected override void CloneCore(State state)
                     stateLocal.InitialWindowedBuffer = InitialWindowedBuffer.Clone();
                     if (_model != null)
                     {
-                        _parentAnomalyDetector.Model = _parentAnomalyDetector.Model.Clone();
-                        _model = _parentAnomalyDetector.Model;
+                        _parentForecaster.Model = _parentForecaster.Model.Clone();
+                        _model = _parentForecaster.Model;
                     }
                 }
 
@@ -288,30 +288,40 @@ private protected override void LearnStateFromDataCore(FixedSizeQueue<Single> da
                     // This method is empty because there is no need to implement a training logic here.
                 }
 
-                private protected override void InitializeAnomalyDetector()
+                private protected override void InitializeForecaster()
                 {
-                    _parentAnomalyDetector = (SsaForecastingBase)Parent;
-                    _model = _parentAnomalyDetector.Model;
+                    _parentForecaster = (SsaForecastingBase)Parent;
+                    _model = _parentForecaster.Model;
                 }
 
                 private protected override void TransformCore(ref float input, FixedSizeQueue<float> windowedBuffer, long iteration, ref VBuffer<float> dst)
                 {
-                    dst = new VBuffer<float>(_parentAnomalyDetector.Horizon,
-                        ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).Forecast(_parentAnomalyDetector.Horizon));
+                    // Forecasting is being done without prediction engine. Update the model
+                    // with the observation.
+                    if (PreviousPosition == -1)
+                        Consume(input);
+
+                    dst = new VBuffer<float>(_parentForecaster.Horizon,
+                        ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).Forecast(_parentForecaster.Horizon));
                 }
 
                 private protected override void TransformCore(ref float input, FixedSizeQueue<float> windowedBuffer, long iteration,
                     ref VBuffer<float> dst1, ref VBuffer<float> dst2, ref VBuffer<float> dst3)
                 {
-                    ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).ForecastWithConfidenceIntervals(_parentAnomalyDetector.Horizon,
-                        out float[] forecast, out float[] min, out float[] max, _parentAnomalyDetector.ConfidenceLevel);
+                    // Forecasting is being done without prediction engine. Update the model
+                    // with the observation.
+                    if (PreviousPosition == -1)
+                        Consume(input);
+
+                    ((AdaptiveSingularSpectrumSequenceModelerInternal)_model).ForecastWithConfidenceIntervals(_parentForecaster.Horizon,
+                        out float[] forecast, out float[] min, out float[] max, _parentForecaster.ConfidenceLevel);
 
-                    dst1 = new VBuffer<float>(_parentAnomalyDetector.Horizon, forecast);
-                    dst2 = new VBuffer<float>(_parentAnomalyDetector.Horizon, min);
-                    dst3 = new VBuffer<float>(_parentAnomalyDetector.Horizon, max);
+                    dst1 = new VBuffer<float>(_parentForecaster.Horizon, forecast);
+                    dst2 = new VBuffer<float>(_parentForecaster.Horizon, min);
+                    dst3 = new VBuffer<float>(_parentForecaster.Horizon, max);
                 }
 
-                public override void Consume(Single input) => _model.Consume(ref input, _parentAnomalyDetector.IsAdaptive);
+                public override void Consume(Single input) => _model.Consume(ref input, _parentForecaster.IsAdaptive);
             }
         }
     }

test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs

@@ -26,11 +26,11 @@ private sealed class ForecastPrediction
         {
 #pragma warning disable CS0649
             [VectorType(4)]
-            public float[] Change;
+            public float[] Forecast;
             [VectorType(4)]
-            public float[] Min;
+            public float[] MinCnf;
             [VectorType(4)]
-            public float[] Max;
+            public float[] MaxCnf;
 #pragma warning restore CS0649
         }
 
@@ -182,13 +182,14 @@ public void Forecast()
             {
                 ConfidenceLevel = 0.95f,
                 Source = "Value",
-                Name = "Change",
-                ForecastingConfidenceIntervalMinOutputColumnName = "Min",
-                ForecastingConfidenceIntervalMaxOutputColumnName = "Max",
+                Name = "Forecast",
+                ForecastingConfidenceIntervalMinOutputColumnName = "MinCnf",
+                ForecastingConfidenceIntervalMaxOutputColumnName = "MaxCnf",
                 WindowSize = 10,
                 SeriesLength = 11,
                 TrainSize = 22,
-                Horizon = 4
+                Horizon = 4,
+                IsAdaptive = true
             };
 
             for (int j = 0; j < NumberOfSeasonsInTraining; j++)
@@ -205,18 +206,19 @@ public void Forecast()
             // Get predictions
             var enumerator = env.Data.CreateEnumerable<ForecastPrediction>(output, true).GetEnumerator();
             ForecastPrediction row = null;
-            List<double> expectedValues = new List<double>() { 0, -3.31410598754883, 0.5, 5.12000000000001E-08, 0, 1.5700820684432983, 5.2001145245395008E-07,
-                    0.012414560443710681, 0, 1.2854313254356384, 0.28810801662678009, 0.02038940454467935, 0, -1.0950627326965332, 0.36663890634019225, 0.026956459625565483};
+            List<float> expectedForecast = new List<float>() { 0.191491723f, 2.53994083f, 5.26454258f, 7.37313938f };
+            List<float> minCnf = new List<float>() { -3.9741993f, -2.36872721f, 0.09407653f, 2.18899345f };
+            List<float> maxCnf = new List<float>() { 4.3571825f, 7.448609f, 10.435009f, 12.5572853f };
+            enumerator.MoveNext();
+            row = enumerator.Current;
 
-            int index = 0;
-            while (enumerator.MoveNext() && index < expectedValues.Count)
+            for (int localIndex = 0; localIndex < 4; localIndex++)
             {
-                row = enumerator.Current;
-                /*Assert.Equal(expectedValues[index++], row.Change[0], precision: 7);  // Alert
-                Assert.Equal(expectedValues[index++], row.Change[1], precision: 7);  // Raw score
-                Assert.Equal(expectedValues[index++], row.Change[2], precision: 7);  // P-Value score
-                Assert.Equal(expectedValues[index++], row.Change[3], precision: 7);  // Martingale score*/
+                Assert.Equal(expectedForecast[localIndex], row.Forecast[localIndex], precision: 7);
+                Assert.Equal(minCnf[localIndex], row.MinCnf[localIndex], precision: 7);
+                Assert.Equal(maxCnf[localIndex], row.MaxCnf[localIndex], precision: 7);
             }
+
         }
 
         [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
@@ -385,7 +387,7 @@ public void ForecastingPredictionEngine()
                 WindowSize = 10,
                 SeriesLength = 11,
                 TrainSize = 22,
-                Horizon = 4
+                Horizon = 4,
             };
 
             for (int j = 0; j < NumberOfSeasonsInTraining; j++)
@@ -403,37 +405,16 @@ public void ForecastingPredictionEngine()
             engine.Update(new Data(1));
             engine.Update(new Data(2));
             var forecast = engine.Forecast(horizon: 5);
-            var prediction = ml.Data.CreateEnumerable<ForecastResult>(forecast, reuseRowObject: false);
-
-
-            /*Assert.Equal(0, prediction.Change[0], precision: 7); // Alert
-            Assert.Equal(1.1661833524703979, prediction.Change[1], precision: 5); // Raw score
-            Assert.Equal(0.5, prediction.Change[2], precision: 7); // P-Value score
-            Assert.Equal(5.1200000000000114E-08, prediction.Change[3], precision: 7); // Martingale score
-
-            //Model 1: Checkpoint.
-            var modelPath = "temp.zip";
-            engine.CheckPoint(ml, modelPath);
-
-            //Model 1: Prediction #2
-            prediction = engine.Predict(new Data(1));
-            Assert.Equal(0, prediction.Change[0], precision: 7); // Alert
-            Assert.Equal(0.12216401100158691, prediction.Change[1], precision: 5); // Raw score
-            Assert.Equal(0.14823824685192111, prediction.Change[2], precision: 5); // P-Value score
-            Assert.Equal(1.5292508189989167E-07, prediction.Change[3], precision: 7); // Martingale score
+            var prediction = ml.Data.CreateEnumerable<ForecastResult>(forecast, reuseRowObject: false).GetEnumerator();
 
-            // Load Model 1.
-            ITransformer model2 = null;
-            using (var file = File.OpenRead(modelPath))
-                model2 = ml.Model.Load(file, out var schema);
+            int index = 0;
+            List<float> expectedForecast = new List<float>() { 3.9516573f, 6.212672f, 7.732854f, 8.125769f, 7.22453928f };
+            while (prediction.MoveNext())
+            {
+                Assert.Equal(prediction.Current.Forecast, expectedForecast[index++]);
+            }
 
-            //Predict and expect the same result after checkpointing(Prediction #2).
-            engine = model2.CreateTimeSeriesPredictionFunction<Data, Prediction>(ml);
-            prediction = engine.Predict(new Data(1));
-            Assert.Equal(0, prediction.Change[0], precision: 7); // Alert
-            Assert.Equal(0.12216401100158691, prediction.Change[1], precision: 5); // Raw score
-            Assert.Equal(0.14823824685192111, prediction.Change[2], precision: 5); // P-Value score
-            Assert.Equal(1.5292508189989167E-07, prediction.Change[3], precision: 5); // Martingale score*/
+            Assert.Equal(5, index);
         }
 
         [Fact]