Scrub text normalizer (#2918)

Author: wschin

src/Microsoft.ML.StaticPipe/TextStaticExtensions.cs

@@ -176,7 +176,7 @@ private sealed class OutPipelineColumn : Scalar<string>
         {
             public readonly Scalar<string> Input;
 
-            public OutPipelineColumn(Scalar<string> input, TextNormalizingEstimator.CaseNormalizationMode textCase, bool keepDiacritics, bool keepPunctuations, bool keepNumbers)
+            public OutPipelineColumn(Scalar<string> input, TextNormalizingEstimator.CaseMode textCase, bool keepDiacritics, bool keepPunctuations, bool keepNumbers)
                 : base(new Reconciler(textCase, keepDiacritics, keepPunctuations, keepNumbers), input)
             {
                 Input = input;
@@ -185,12 +185,12 @@ public OutPipelineColumn(Scalar<string> input, TextNormalizingEstimator.CaseNorm
 
         private sealed class Reconciler : EstimatorReconciler, IEquatable<Reconciler>
         {
-            private readonly TextNormalizingEstimator.CaseNormalizationMode _textCase;
+            private readonly TextNormalizingEstimator.CaseMode _textCase;
             private readonly bool _keepDiacritics;
             private readonly bool _keepPunctuations;
             private readonly bool _keepNumbers;
 
-            public Reconciler(TextNormalizingEstimator.CaseNormalizationMode textCase, bool keepDiacritics, bool keepPunctuations, bool keepNumbers)
+            public Reconciler(TextNormalizingEstimator.CaseMode textCase, bool keepDiacritics, bool keepPunctuations, bool keepNumbers)
             {
                 _textCase = textCase;
                 _keepDiacritics = keepDiacritics;
@@ -227,15 +227,15 @@ public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
         /// Normalizes input text by changing case, removing diacritical marks, punctuation marks and/or numbers.
         /// </summary>
         /// <param name="input">The column to apply to.</param>
-        /// <param name="textCase">Casing text using the rules of the invariant culture.</param>
+        /// <param name="caseMode">Casing text using the rules of the invariant culture.</param>
         /// <param name="keepDiacritics">Whether to keep diacritical marks or remove them.</param>
         /// <param name="keepPunctuations">Whether to keep punctuation marks or remove them.</param>
         /// <param name="keepNumbers">Whether to keep numbers or remove them.</param>
         public static Scalar<string> NormalizeText(this Scalar<string> input,
-            TextNormalizingEstimator.CaseNormalizationMode textCase = TextNormalizingEstimator.CaseNormalizationMode.Lower,
+            TextNormalizingEstimator.CaseMode caseMode = TextNormalizingEstimator.CaseMode.Lower,
             bool keepDiacritics = false,
             bool keepPunctuations = true,
-            bool keepNumbers = true) => new OutPipelineColumn(input, textCase, keepDiacritics, keepPunctuations, keepNumbers);
+            bool keepNumbers = true) => new OutPipelineColumn(input, caseMode, keepDiacritics, keepPunctuations, keepNumbers);
     }
 
     /// <summary>

src/Microsoft.ML.Transforms/Text/TextCatalog.cs

@@ -83,19 +83,19 @@ public static TokenizingByCharactersEstimator TokenizeCharacters(this Transforms
         /// <param name="catalog">The text-related transform's catalog.</param>
         /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
         /// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
-        /// <param name="textCase">Casing text using the rules of the invariant culture.</param>
+        /// <param name="caseMode">Casing text using the rules of the invariant culture.</param>
         /// <param name="keepDiacritics">Whether to keep diacritical marks or remove them.</param>
         /// <param name="keepPunctuations">Whether to keep punctuation marks or remove them.</param>
         /// <param name="keepNumbers">Whether to keep numbers or remove them.</param>
         public static TextNormalizingEstimator NormalizeText(this TransformsCatalog.TextTransforms catalog,
             string outputColumnName,
             string inputColumnName = null,
-            TextNormalizingEstimator.CaseNormalizationMode textCase = TextNormalizeDefaults.TextCase,
+            TextNormalizingEstimator.CaseMode caseMode = TextNormalizeDefaults.Mode,
             bool keepDiacritics = TextNormalizeDefaults.KeepDiacritics,
             bool keepPunctuations = TextNormalizeDefaults.KeepPunctuations,
             bool keepNumbers = TextNormalizeDefaults.KeepNumbers)
             => new TextNormalizingEstimator(Contracts.CheckRef(catalog, nameof(catalog)).GetEnvironment(),
-                outputColumnName, inputColumnName, textCase, keepDiacritics, keepPunctuations, keepNumbers);
+                outputColumnName, inputColumnName, caseMode, keepDiacritics, keepPunctuations, keepNumbers);
 
         /// <include file='doc.xml' path='doc/members/member[@name="WordEmbeddings"]/*' />
         /// <param name="catalog">The text-related transform's catalog.</param>

src/Microsoft.ML.Transforms/Text/TextFeaturizingEstimator.cs

@@ -24,7 +24,7 @@
 
 namespace Microsoft.ML.Transforms.Text
 {
-    using CaseNormalizationMode = TextNormalizingEstimator.CaseNormalizationMode;
+    using CaseMode = TextNormalizingEstimator.CaseMode;
     // A transform that turns a collection of text documents into numerical feature vectors. The feature vectors are counts
     // of (word or character) ngrams in a given text. It offers ngram hashing (finding the ngram token string name to feature
     // integer index mapping through hashing) as an option.
@@ -100,7 +100,7 @@ internal sealed class Arguments : TransformInputBase
             public bool UsePredefinedStopWordRemover = false;
 
             [Argument(ArgumentType.AtMostOnce, HelpText = "Casing text using the rules of the invariant culture.", ShortName = "case", SortOrder = 5)]
-            public CaseNormalizationMode TextCase = TextNormalizingEstimator.Defaults.TextCase;
+            public CaseMode TextCase = TextNormalizingEstimator.Defaults.Mode;
 
             [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to keep diacritical marks or remove them.", ShortName = "diac", SortOrder = 6)]
             public bool KeepDiacritics = TextNormalizingEstimator.Defaults.KeepDiacritics;
@@ -142,7 +142,7 @@ public sealed class Options
             /// <summary>
             /// Casing used for the text.
             /// </summary>
-            public CaseNormalizationMode TextCase { get; set; } = CaseNormalizationMode.Lower;
+            public CaseMode TextCase { get; set; } = CaseMode.Lower;
             /// <summary>
             /// Whether to keep diacritical marks or remove them.
             /// </summary>
@@ -203,7 +203,7 @@ private sealed class TransformApplierParams
             public readonly NormFunction VectorNormalizer;
             public readonly Language Language;
             public readonly bool UsePredefinedStopWordRemover;
-            public readonly CaseNormalizationMode TextCase;
+            public readonly CaseMode TextCase;
             public readonly bool KeepDiacritics;
             public readonly bool KeepPunctuations;
             public readonly bool KeepNumbers;
@@ -241,7 +241,7 @@ public bool NeedsNormalizeTransform
                 get
                 {
                     return
-                        TextCase != CaseNormalizationMode.None ||
+                        TextCase != CaseMode.None ||
                         !KeepDiacritics ||
                         !KeepPunctuations ||
                         !KeepNumbers;
@@ -275,7 +275,7 @@ public TransformApplierParams(TextFeaturizingEstimator parent)
             {
                 var host = parent._host;
                 host.Check(Enum.IsDefined(typeof(Language), parent.OptionalSettings.TextLanguage));
-                host.Check(Enum.IsDefined(typeof(CaseNormalizationMode), parent.OptionalSettings.TextCase));
+                host.Check(Enum.IsDefined(typeof(CaseMode), parent.OptionalSettings.TextCase));
                 WordExtractorFactory = parent._wordFeatureExtractor?.CreateComponent(host, parent._dictionary);
                 CharExtractorFactory = parent._charFeatureExtractor?.CreateComponent(host, parent._dictionary);
                 VectorNormalizer = parent.OptionalSettings.VectorNormalizer;