Speed up the inference of the saved_model(s). Fixes #5847 (#5848)

* Speed up of the inference of saved_model(s).

Signed-off-by: darth-vader-lg <[email protected]>

* Fixed TensorFlowTransform fitting problem.
- Fixed the exception while fitting data with more than one input tensor. Followed the OnnxTransformer schema for the data view getters creation.

Signed-off-by: darth-vader-lg <[email protected]>

* Dispose of the cached tensors in the TensorFlowTransformer.
- The cached tensors are disposed at the end of inference operations.

Signed-off-by: darth-vader-lg <[email protected]>

Author: darth-vader-lg

src/Microsoft.ML.TensorFlow/TensorflowTransform.cs

@@ -637,9 +637,41 @@ public Mapper(TensorFlowTransformer parent, DataViewSchema inputSchema) :
                 _runners = new ConcurrentBag<Runner>();
             }
 
+            private Delegate CreateGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, OutputCache outputCache)
+            {
+                Host.AssertValue(input);
+
+                var activeOutputColNames = _parent.Outputs.Where((x, i) => activeOutput(i)).ToArray();
+
+                var type = Tf2MlNetType(_parent.TFOutputTypes[iinfo]).RawType;
+                Host.Assert(type == _parent.OutputTypes[iinfo].GetItemType().RawType);
+                var srcTensorGetters = GetTensorValueGetters(input, _inputColIndices, _isInputVector, _parent.TFInputTypes, _fullySpecifiedShapes);
+                return Utils.MarshalInvoke(MakeGetter<int>, type, input, iinfo, srcTensorGetters, activeOutputColNames, outputCache);
+            }
+
+            public override Delegate[] CreateGetters(DataViewRow input, Func<int, bool> activeOutput, out Action disposer)
+            {
+                Contracts.Assert(input.Schema == InputSchema);
+
+                OutputCache outputCacher = new OutputCache();
+
+                int n = OutputColumns.Value.Length;
+                var result = new Delegate[n];
+                for (int i = 0; i < n; i++) {
+                    if (!activeOutput(i))
+                        continue;
+                    result[i] = CreateGetter(input, i, activeOutput, outputCacher);
+                }
+                disposer = () =>
+                {
+                    outputCacher.Dispose();
+                };
+                return result;
+            }
+
             private protected override void SaveModel(ModelSaveContext ctx) => _parent.SaveModel(ctx);
 
-            private class OutputCache
+            private class OutputCache : IDisposable
             {
                 public long Position;
                 public Dictionary<string, Tensor> Outputs;
@@ -648,22 +680,22 @@ public OutputCache()
                     Position = -1;
                     Outputs = new Dictionary<string, Tensor>();
                 }
-            }
-
-            protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
-            {
-                disposer = null;
-                Host.AssertValue(input);
 
-                var outputCache = new OutputCache();
-                var activeOutputColNames = _parent.Outputs.Where((x, i) => activeOutput(i)).ToArray();
+                private bool _isDisposed;
 
-                var type = Tf2MlNetType(_parent.TFOutputTypes[iinfo]).RawType;
-                Host.Assert(type == _parent.OutputTypes[iinfo].GetItemType().RawType);
-                var srcTensorGetters = GetTensorValueGetters(input, _inputColIndices, _isInputVector, _parent.TFInputTypes, _fullySpecifiedShapes);
-                return Utils.MarshalInvoke(MakeGetter<int>, type, input, iinfo, srcTensorGetters, activeOutputColNames, outputCache);
+                public void Dispose()
+                {
+                    if (_isDisposed)
+                        return;
+                    foreach (var tensor in Outputs.Values)
+                        tensor.Dispose();
+                    _isDisposed = true;
+                }
             }
 
+            protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
+                => throw new NotImplementedException("This should never be called!");
+
             private Delegate MakeGetter<T>(DataViewRow input, int iinfo, ITensorValueGetter[] srcTensorGetters, string[] activeOutputColNames, OutputCache outputCache) where T : unmanaged
             {
                 Host.AssertValue(input);