fixed IDataview so each has its own transformer

updates with correct native nuget package:

including signed native nuget package. Null fix

added tests, removed dll check

Author: michaelgsharp

src/Microsoft.ML.Featurizers/CategoryImputer.cs

@@ -88,15 +88,6 @@ internal static CategoryImputerTransformerEstimator Create(IHostEnvironment env,
 
         internal CategoryImputerTransformerEstimator(IHostEnvironment env, string outputColumnName, string inputColumnName)
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             Contracts.CheckValue(env, nameof(env));
             _host = env.Register(nameof(CategoryImputerTransformerEstimator));
             _options = new Options
@@ -107,15 +98,6 @@ internal CategoryImputerTransformerEstimator(IHostEnvironment env, string output
 
         internal CategoryImputerTransformerEstimator(IHostEnvironment env, Options options)
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             Contracts.CheckValue(env, nameof(env));
             _host = env.Register(nameof(CategoryImputerTransformerEstimator));
 
@@ -169,15 +151,6 @@ public sealed class CategoryImputerTransformer : RowToRowTransformerBase, IDispo
         internal CategoryImputerTransformer(IHostEnvironment host, IDataView input, CategoryImputerTransformerEstimator.Column[] columns) :
             base(host.Register(nameof(CategoryImputerTransformerEstimator)))
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             var schema = input.Schema;
 
             _columns = columns.Select(x => TypedColumn.CreateTypedColumn(x.Name, x.Source, schema[x.Source].Type.RawType.ToString())).ToArray();
@@ -191,15 +164,6 @@ internal CategoryImputerTransformer(IHostEnvironment host, IDataView input, Cate
         internal CategoryImputerTransformer(IHostEnvironment host, ModelLoadContext ctx) :
             base(host.Register(nameof(CategoryImputerTransformer)))
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             Host.CheckValue(ctx, nameof(ctx));
             ctx.CheckAtModel(GetVersionInfo());
             // *** Binary format ***

src/Microsoft.ML.Featurizers/TimeSeriesImputer.cs

@@ -107,14 +107,14 @@ internal TimeSeriesImputerEstimator(IHostEnvironment env, string timeSeriesColum
             _host = Contracts.CheckRef(env, nameof(env)).Register("TimeSeriesImputerEstimator");
             _host.CheckValue(timeSeriesColumn, nameof(timeSeriesColumn), "TimePoint column should not be null.");
             _host.CheckNonEmpty(grainColumns, nameof(grainColumns), "Need at least one grain column.");
-            if(filterMode != FilterMode.NoFilter)
+            if (filterMode == FilterMode.Include)
                 _host.CheckNonEmpty(filterColumns, nameof(filterColumns), "Need at least 1 filter column if a FilterMode is specified");
 
             _options = new Options
             {
                 TimeSeriesColumn = timeSeriesColumn,
                 GrainColumns = grainColumns,
-                FilterColumns = filterColumns,
+                FilterColumns = filterColumns == null ? new string[] { } : filterColumns,
                 FilterMode = filterMode,
                 ImputeMode = imputeMode,
                 SupressTypeErrors = supressTypeErrors
@@ -262,7 +262,7 @@ private unsafe TransformerEstimatorSafeHandle CreateTransformerFromSavedData(byt
                 if (!result)
                     throw new Exception(CommonExtensions.GetErrorDetailsAndFreeNativeMemory(errorHandle));
 
-               return new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative);
+                return new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative);
             }
         }
 
@@ -433,6 +433,8 @@ internal TimeSeriesImputerDataView MakeDataTransform(IDataView input)
             return new TimeSeriesImputerDataView(_host, input, _timeSeriesColumn, _grainColumns, _dataColumns, _allColumnNames, this);
         }
 
+        internal TransformerEstimatorSafeHandle CloneTransformer() => CreateTransformerFromSavedData(CreateTransformerSaveData());
+
         public void Dispose()
         {
             if (!TransformerHandle.IsClosed)

src/Microsoft.ML.Featurizers/TimeSeriesImputerDataView.cs

@@ -49,7 +49,7 @@ internal TypedColumn(DataViewSchema.Column column, bool isImputed, SharedColumnS
             }
 
             internal abstract Delegate GetGetter();
-            internal abstract void InitializeGetter(DataViewRowCursor cursor, TimeSeriesImputerTransformer transformerParent, string timeSeriesColumn,
+            internal abstract void InitializeGetter(DataViewRowCursor cursor, TimeSeriesImputerTransformer.TransformerEstimatorSafeHandle transformerParent, string timeSeriesColumn,
                 string[] grainColumns, string[] dataColumns, string[] allColumnNames, Dictionary<string, TypedColumn> allColumns);
 
             internal abstract TypeId GetTypeId();
@@ -129,7 +129,7 @@ internal override Delegate GetGetter()
                 return _getter;
             }
 
-            internal override unsafe void InitializeGetter(DataViewRowCursor cursor, TimeSeriesImputerTransformer transformerParent, string timeSeriesColumn,
+            internal override unsafe void InitializeGetter(DataViewRowCursor cursor, TimeSeriesImputerTransformer.TransformerEstimatorSafeHandle transformer, string timeSeriesColumn,
                 string[] grainColumns, string[] dataColumns, string[] allImputedColumnNames, Dictionary<string, TypedColumn> allColumns)
             {
                 if (Column.Name != IsRowImputedColumnName)
@@ -154,7 +154,7 @@ internal override unsafe void InitializeGetter(DataViewRowCursor cursor, TimeSer
                             fixed (byte* bufferPointer = SharedState.ColumnBuffer)
                             {
                                 var binaryArchiveData = new NativeBinaryArchiveData() { Data = bufferPointer, DataSize = new IntPtr(bufferLength) };
-                                success = TransformDataNative(transformerParent.TransformerHandle, binaryArchiveData, out outputData, out outputDataSize, out errorHandle);
+                                success = TransformDataNative(transformer, binaryArchiveData, out outputData, out outputDataSize, out errorHandle);
                                 if (!success)
                                     throw new Exception(CommonExtensions.GetErrorDetailsAndFreeNativeMemory(errorHandle));
                             }
@@ -633,7 +633,7 @@ protected override bool ReleaseHandle()
         private readonly string[] _allImputedColumnNames;
         private readonly DataViewSchema _schema;
 
-        public TimeSeriesImputerDataView(IHostEnvironment env, IDataView input, string timeSeriesColumn, string[] grainColumns, string[] dataColumns, string[] allColumnNames, TimeSeriesImputerTransformer parent)
+        internal TimeSeriesImputerDataView(IHostEnvironment env, IDataView input, string timeSeriesColumn, string[] grainColumns, string[] dataColumns, string[] allColumnNames, TimeSeriesImputerTransformer parent)
         {
             _host = env;
             _source = input;
@@ -664,7 +664,7 @@ public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columns
 
             // Column WasColumnImputed is new, don't get its value from source
             var input = _source.GetRowCursorForAllColumns();//.GetRowCursor(columnsNeeded.Where(x => x.Name != IsRowImputedColumnName));
-            return new Cursor(_host, input, _parent, _timeSeriesColumn, _grainColumns, _dataColumns, _allImputedColumnNames, _schema);
+            return new Cursor(_host, input, _parent.CloneTransformer(), _timeSeriesColumn, _grainColumns, _dataColumns, _allImputedColumnNames, _schema);
         }
 
         // Can't use parallel cursors so this defaults to calling non-parallel version
@@ -687,9 +687,9 @@ private sealed class Cursor : DataViewRowCursor
             private bool _isGood;
             private readonly Dictionary<string, TypedColumn> _allColumns;
             private readonly DataViewSchema _schema;
-            private readonly TimeSeriesImputerTransformer _transformerParent;
+            private readonly TimeSeriesImputerTransformer.TransformerEstimatorSafeHandle _transformer;
 
-            public Cursor(IChannelProvider provider, DataViewRowCursor input, TimeSeriesImputerTransformer transformerParent, string timeSeriesColumn,
+            public Cursor(IChannelProvider provider, DataViewRowCursor input, TimeSeriesImputerTransformer.TransformerEstimatorSafeHandle transformer, string timeSeriesColumn,
                 string[] grainColumns, string[] dataColumns, string[] allImputedColumnNames, DataViewSchema schema)
             {
                 _ch = provider;
@@ -699,7 +699,7 @@ public Cursor(IChannelProvider provider, DataViewRowCursor input, TimeSeriesImpu
                 var length = input.Schema.Count;
                 _position = -1;
                 _schema = schema;
-                _transformerParent = transformerParent;
+                _transformer = transformer;
 
                 var sharedState = new SharedColumnState()
                 {
@@ -712,7 +712,7 @@ public Cursor(IChannelProvider provider, DataViewRowCursor input, TimeSeriesImpu
 
                 foreach (var column in _allColumns.Values)
                 {
-                    column.InitializeGetter(_input, transformerParent, timeSeriesColumn, grainColumns, dataColumns, allImputedColumnNames, _allColumns);
+                    column.InitializeGetter(_input, transformer, timeSeriesColumn, grainColumns, dataColumns, allImputedColumnNames, _allColumns);
                 }
             }
 
@@ -736,6 +736,12 @@ public override bool IsColumnActive(DataViewSchema.Column column)
                 return true;
             }
 
+            protected override void Dispose(bool disposing)
+            {
+                if (!_transformer.IsClosed)
+                    _transformer.Close();
+            }
+
             /// <summary>
             /// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
             /// This throws if the column is not active in this row, or if the type

src/Microsoft.ML.Featurizers/ToStringTransformer.cs

@@ -130,15 +130,6 @@ internal static ToStringTransformerEstimator Create(IHostEnvironment env, params
 
         internal ToStringTransformerEstimator(IHostEnvironment env, string outputColumnName, string inputColumnName)
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             Contracts.CheckValue(env, nameof(env));
             _host = env.Register(nameof(ToStringTransformerEstimator));
 
@@ -150,15 +141,6 @@ internal ToStringTransformerEstimator(IHostEnvironment env, string outputColumnN
 
         internal ToStringTransformerEstimator(IHostEnvironment env, Options options)
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             Contracts.CheckValue(env, nameof(env));
             _host = env.Register(nameof(ToStringTransformerEstimator));
 
@@ -211,15 +193,6 @@ public sealed class ToStringTransformer : RowToRowTransformerBase, IDisposable
         internal ToStringTransformer(IHostEnvironment host, ToStringTransformerEstimator.Column[] columns, IDataView input) :
             base(host.Register(nameof(ToStringTransformer)))
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             var schema = input.Schema;
 
             _columns = columns.Select(x => TypedColumn.CreateTypedColumn(x.Name, x.Source, schema[x.Source].Type.RawType.ToString())).ToArray();
@@ -234,15 +207,6 @@ internal ToStringTransformer(IHostEnvironment host, ToStringTransformerEstimator
         internal ToStringTransformer(IHostEnvironment host, ModelLoadContext ctx) :
             base(host.Register(nameof(ToStringTransformer)))
         {
-            try
-            {
-                CheckIfDllExists(IntPtr.Zero, out IntPtr errorHandleString, out IntPtr errorHandleStringSize);
-            }
-            catch
-            {
-                throw new Exception("Featurizers library not found");
-            }
-
             Host.CheckValue(ctx, nameof(ctx));
             ctx.CheckAtModel(GetVersionInfo());
             // *** Binary format ***

temp-nuget-folder/MicrosoftMLFeaturizers.0.1.0.nupkg

@@ -257,17 +257,6 @@ public void HolidayTest()
             Assert.Equal("Christmas Day", row[20].Value.ToString());  // HolidayName
             Assert.Equal((byte)0, row[21].Value);                       // IsPaidTimeOff
 
-            mlContext.Model.Save(model, data.Schema, "D:/Repos/machinelearning/temp-nuget-folder/model.zip");
-            var loadedModel = mlContext.Model.Load("D:/Repos/machinelearning/temp-nuget-folder/model.zip", out DataViewSchema loadedSchema);
-
-            var loadedOutput = loadedModel.Transform(data);
-            var loadedRow = loadedOutput.Preview(1).RowView[0].Values;
-
-            // Assert the data from the first row from loaded data for holidays is what we expect
-            Assert.Equal("Christmas Day", loadedRow[20].Value.ToString());  // HolidayName
-            Assert.Equal((byte)0, loadedRow[21].Value);                       // IsPaidTimeOff
-
-
             TestEstimatorCore(pipeline, data);
             Done();
         }