Adding transform extensions (#1460)

* adding all extensions for the Text related transformation

* adding keyToVector extensions and renaming the estimatorto conform to 1318

* Adding SelectColumns xtensions. Renaming SelectEstimator and Textnormalizer

* KeyToBinary extensions

* Adding extensions for the Image transforms. Some renaming

Author: sfilipi

docs/samples/Microsoft.ML.Samples/Dynamic/KeyToValue_Term.cs

@@ -7,6 +7,7 @@
         using Microsoft.ML.Runtime.Api;
         using Microsoft.ML.Runtime.Data;
         using Microsoft.ML.Transforms.Categorical;
+        using Microsoft.ML.Transforms.Conversions;
         using Microsoft.ML.Transforms.Text;
         using System;
         using System.Collections.Generic;

src/Microsoft.ML.Data/Evaluators/EvaluatorUtils.cs

@@ -6,7 +6,7 @@
 using Microsoft.ML.Runtime.Data.IO;
 using Microsoft.ML.Runtime.Internal.Utilities;
 using Microsoft.ML.Transforms;
-using Microsoft.ML.Transforms.Categorical;
+using Microsoft.ML.Transforms.Conversions;
 using System;
 using System.Collections.Generic;
 using System.Globalization;

src/Microsoft.ML.Data/Transforms/CategoricalsCatalog.cs

@@ -55,4 +55,50 @@ public static ConvertingEstimator ConvertTo(this TransformsCatalog.Conversions c
         public static ConvertingEstimator ConvertTo(this TransformsCatalog.Conversions catalog, params ConvertingTransform.ColumnInfo[] columns)
             => new ConvertingEstimator(CatalogUtils.GetEnvironment(catalog), columns);
     }
+
+    public static class ToValueCatalog
+    {
+        /// <summary>
+        /// Convert the key types back to their original values.
+        /// </summary>
+        /// <param name="catalog">The categorical transform's catalog.</param>
+        /// <param name="inputColumn">Name of the input column.</param>
+        public static KeyToValueEstimator MapKeyToValue(this TransformsCatalog.Conversions catalog, string inputColumn)
+            => new KeyToValueEstimator(CatalogUtils.GetEnvironment(catalog), inputColumn);
+
+        /// <summary>
+        ///  Convert the key types (name of the column specified in the first item of the tuple) back to their original values
+        ///  (named as specified in the second item of the tuple).
+        /// </summary>
+        /// <param name="catalog">The categorical transform's catalog</param>
+        /// <param name="columns">The pairs of input and output columns.</param>
+        public static KeyToValueEstimator MapKeyToValue(this TransformsCatalog.Conversions catalog, params (string input, string output)[] columns)
+             => new KeyToValueEstimator(CatalogUtils.GetEnvironment(catalog), columns);
+    }
+
+    /// <summary>
+    /// Extensions for KeyToVectorMappingEstimator.
+    /// </summary>
+    public static class ToVectorCatalog
+    {
+        /// <summary>
+        /// Convert the key types back to their original vectors.
+        /// </summary>
+        /// <param name="catalog">The categorical transform's catalog.</param>
+        /// <param name="columns">The input column to map back to vectors.</param>
+        public static KeyToVectorMappingEstimator MapKeyToVector(this TransformsCatalog.Conversions catalog,
+            params KeyToVectorTransform.ColumnInfo[] columns)
+            => new KeyToVectorMappingEstimator(CatalogUtils.GetEnvironment(catalog), columns);
+
+        /// <summary>
+        /// Convert the key types back to their original vectors.
+        /// </summary>
+        /// <param name="catalog">The categorical transform's catalog.</param>
+        /// <param name="inputColumn">The name of the input column.</param>
+        /// <param name="outputColumn">The name of the output column.</param>
+        /// <param name="bag">Whether bagging is used for the conversion. </param>
+        public static KeyToVectorMappingEstimator MapKeyToVector(this TransformsCatalog.Conversions catalog,
+            string inputColumn, string outputColumn = null, bool bag = KeyToVectorMappingEstimator.Defaults.Bag)
+            => new KeyToVectorMappingEstimator(CatalogUtils.GetEnvironment(catalog), inputColumn, outputColumn, bag);
+    }
 }

src/Microsoft.ML.Data/Transforms/ConversionsCatalog.cs

@@ -0,0 +1,93 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Transforms;
+
+namespace Microsoft.ML
+{
+    /// <summary>
+    /// Extensions for Column Copying Estimator.
+    /// </summary>
+    public static class ColumnCopyingCatalog
+    {
+        /// <summary>
+        /// Copies the input column to another column named as specified in <paramref name="outputColumn"/>.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="inputColumn">Name of the input column.</param>
+        /// <param name="outputColumn">Name of the new column, resulting from copying.</param>
+        public static CopyColumnsEstimator CopyColumns(this TransformsCatalog catalog, string inputColumn, string outputColumn)
+            => new CopyColumnsEstimator(CatalogUtils.GetEnvironment(catalog), inputColumn, outputColumn);
+
+        /// <summary>
+        /// Copies the input column, name specified in the first item of the tuple,
+        /// to another column, named as specified in the second item of the tuple.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog</param>
+        /// <param name="columns">The pairs of input and output columns.</param>
+        public static CopyColumnsEstimator CopyColumns(this TransformsCatalog catalog, params (string source, string name)[] columns)
+            => new CopyColumnsEstimator(CatalogUtils.GetEnvironment(catalog), columns);
+
+    }
+
+    /// <summary>
+    /// Extensions for ColumnConcatenatingEstimator.
+    /// </summary>
+    public static class ColumnConcatenatingEstimatorCatalog
+    {
+        /// <summary>
+        /// Concatenates two columns together.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="outputColumn">The name of the output column.</param>
+        /// <param name="inputColumns">The names of the columns to concatenate together.</param>
+        public static ColumnConcatenatingEstimator Concatenate(this TransformsCatalog catalog, string outputColumn, params string[] inputColumns)
+            => new ColumnConcatenatingEstimator(CatalogUtils.GetEnvironment(catalog), outputColumn, inputColumns);
+
+    }
+
+    /// <summary>
+    /// Extensions for ColumnSelectingEstimator.
+    /// </summary>
+    public static class ColumnSelectingCatalog
+    {
+        /// <summary>
+        /// KeepColumns is used to select a list of columns that the user wants to keep on a given an input. Any column not specified
+        /// will be dropped from the output output schema.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="columnsToKeep">The array of column names to keep.</param>
+        public static ColumnSelectingEstimator KeepColumns(this TransformsCatalog catalog, params string[] columnsToKeep)
+            => ColumnSelectingEstimator.KeepColumns(CatalogUtils.GetEnvironment(catalog), columnsToKeep);
+
+        /// <summary>
+        /// DropColumns is used to select a list of columns that user wants to drop from a given input. Any column not specified will
+        /// be maintained in the output schema.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="columnsToDrop">The array of column names to drop.</param>
+        public static ColumnSelectingEstimator DropColumns(this TransformsCatalog catalog, params string[] columnsToDrop)
+            => ColumnSelectingEstimator.DropColumns(CatalogUtils.GetEnvironment(catalog), columnsToDrop);
+
+        /// <summary>
+        /// ColumnSelectingEstimator is used to select a list of columns that user wants to drop from a given input.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="keepColumns">The array of column names to keep, cannot be set with <paramref name="dropColumns"/>.</param>
+        /// <param name="dropColumns">The array of column names to drop, cannot be set with <paramref name="keepColumns"/>.</param>
+        /// <param name="keepHidden">If true will keep hidden columns and false will remove hidden columns.</param>
+        /// <param name="ignoreMissing">If false will check for any columns given in <paramref name="keepColumns"/>
+        /// or <paramref name="dropColumns"/> that are missing from the input. If a missing colums exists a
+        /// SchemaMistmatch exception is thrown. If true, the check is not made.</param>
+        public static ColumnSelectingEstimator SelectColumns(this TransformsCatalog catalog,
+            string[] keepColumns,
+            string[] dropColumns,
+            bool keepHidden = SelectColumnsTransform.Defaults.KeepHidden,
+            bool ignoreMissing = SelectColumnsTransform.Defaults.IgnoreMissing)
+            => new ColumnSelectingEstimator(CatalogUtils.GetEnvironment(catalog),
+                keepColumns, dropColumns, keepHidden, ignoreMissing);
+    }
+}

src/Microsoft.ML.Data/Transforms/ExtensionsCatalog.cs

@@ -12,7 +12,7 @@
 using Microsoft.ML.Runtime.Model.Pfa;
 using Microsoft.ML.StaticPipe;
 using Microsoft.ML.StaticPipe.Runtime;
-using Microsoft.ML.Transforms.Categorical;
+using Microsoft.ML.Transforms.Conversions;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
@@ -32,7 +32,7 @@
 [assembly: LoadableClass(typeof(IRowMapper), typeof(KeyToValueTransform), null, typeof(SignatureLoadRowMapper),
     KeyToValueTransform.UserName, KeyToValueTransform.LoaderSignature)]
 
-namespace Microsoft.ML.Transforms.Categorical
+namespace Microsoft.ML.Transforms.Conversions
 {
     /// <summary>
     /// KeyToValueTransform utilizes KeyValues metadata to map key indices to the corresponding values in the KeyValues metadata.

src/Microsoft.ML.Data/Transforms/KeyToValueTransform.cs

@@ -12,7 +12,7 @@
 using Microsoft.ML.Runtime.Model.Pfa;
 using Microsoft.ML.StaticPipe;
 using Microsoft.ML.StaticPipe.Runtime;
-using Microsoft.ML.Transforms.Categorical;
+using Microsoft.ML.Transforms.Conversions;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
@@ -31,7 +31,7 @@
 [assembly: LoadableClass(typeof(IRowMapper), typeof(KeyToVectorTransform), null, typeof(SignatureLoadRowMapper),
    KeyToVectorTransform.UserName, KeyToVectorTransform.LoaderSignature)]
 
-namespace Microsoft.ML.Transforms.Categorical
+namespace Microsoft.ML.Transforms.Conversions
 {
     public sealed class KeyToVectorTransform : OneToOneTransformerBase
     {
@@ -84,7 +84,7 @@ public sealed class Arguments
 
             [Argument(ArgumentType.AtMostOnce,
                 HelpText = "Whether to combine multiple indicator vectors into a single bag vector instead of concatenating them. This is only relevant when the input is a vector.")]
-            public bool Bag = KeyToVectorEstimator.Defaults.Bag;
+            public bool Bag = KeyToVectorMappingEstimator.Defaults.Bag;
         }
 
         public class ColumnInfo
@@ -93,7 +93,7 @@ public class ColumnInfo
             public readonly string Output;
             public readonly bool Bag;
 
-            public ColumnInfo(string input, string output, bool bag = KeyToVectorEstimator.Defaults.Bag)
+            public ColumnInfo(string input, string output, bool bag = KeyToVectorMappingEstimator.Defaults.Bag)
             {
                 Input = input;
                 Output = output;
@@ -733,25 +733,25 @@ private bool SaveAsOnnxCore(OnnxContext ctx, int iinfo, ColInfo info, string src
         }
     }
 
-    public sealed class KeyToVectorEstimator : TrivialEstimator<KeyToVectorTransform>
+    public sealed class KeyToVectorMappingEstimator : TrivialEstimator<KeyToVectorTransform>
     {
         internal static class Defaults
         {
             public const bool Bag = false;
         }
 
-        public KeyToVectorEstimator(IHostEnvironment env, params KeyToVectorTransform.ColumnInfo[] columns)
+        public KeyToVectorMappingEstimator(IHostEnvironment env, params KeyToVectorTransform.ColumnInfo[] columns)
             : this(env, new KeyToVectorTransform(env, columns))
         {
         }
 
-        public KeyToVectorEstimator(IHostEnvironment env, string name, string source = null, bool bag = Defaults.Bag)
-            : this(env, new KeyToVectorTransform(env, new KeyToVectorTransform.ColumnInfo(source ?? name, name, bag)))
+        public KeyToVectorMappingEstimator(IHostEnvironment env, string inputColumn, string outputColumn = null, bool bag = Defaults.Bag)
+            : this(env, new KeyToVectorTransform(env, new KeyToVectorTransform.ColumnInfo(inputColumn, outputColumn ?? inputColumn, bag)))
         {
         }
 
-        private KeyToVectorEstimator(IHostEnvironment env, KeyToVectorTransform transformer)
-            : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(KeyToVectorEstimator)), transformer)
+        private KeyToVectorMappingEstimator(IHostEnvironment env, KeyToVectorTransform transformer)
+            : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(KeyToVectorMappingEstimator)), transformer)
         {
         }
 
@@ -891,7 +891,7 @@ public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
                     var col = (IColInput)toOutput[i];
                     infos[i] = new KeyToVectorTransform.ColumnInfo(inputNames[col.Input], outputNames[toOutput[i]], col.Bag);
                 }
-                return new KeyToVectorEstimator(env, infos);
+                return new KeyToVectorMappingEstimator(env, infos);
             }
         }
 

src/Microsoft.ML.Data/Transforms/KeyToVectorTransform.cs

@@ -34,7 +34,10 @@ public static class NormalizerCatalog
         /// ]]>
         /// </format>
         /// </example>
-        public static NormalizingEstimator Normalize(this TransformsCatalog catalog, string inputName, string outputName = null, NormalizingEstimator.NormalizerMode mode = NormalizingEstimator.NormalizerMode.MinMax)
+        public static NormalizingEstimator Normalize(this TransformsCatalog catalog,
+            string inputName,
+            string outputName = null,
+            NormalizingEstimator.NormalizerMode mode = NormalizingEstimator.NormalizerMode.MinMax)
             => new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), inputName, outputName, mode);
 
         /// <summary>
@@ -57,15 +60,18 @@ public static NormalizingEstimator Normalize(this TransformsCatalog catalog, str
         /// ]]>
         /// </format>
         /// </example>
-        public static NormalizingEstimator Normalize(this TransformsCatalog catalog, NormalizingEstimator.NormalizerMode mode, params (string input, string output)[] columns)
+        public static NormalizingEstimator Normalize(this TransformsCatalog catalog,
+            NormalizingEstimator.NormalizerMode mode,
+            params (string input, string output)[] columns)
             => new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), mode, columns);
 
         /// <summary>
         /// Normalize (rescale) columns according to specified custom parameters.
         /// </summary>
         /// <param name="catalog">The transform catalog</param>
         /// <param name="columns">The normalization settings for all the columns</param>
-        public static NormalizingEstimator Normalize(this TransformsCatalog catalog, params NormalizingEstimator.ColumnBase[] columns)
+        public static NormalizingEstimator Normalize(this TransformsCatalog catalog,
+            params NormalizingEstimator.ColumnBase[] columns)
             => new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columns);
     }
 }