Changed default NGram length from 1 to 2. (#5248)

* make the FastTree work better with sparse data. Is much more performant with sparse data then before, slightly less performant if its dense data

* reverted FastTree changes, fixed incorrect test

* Added catagorical column to wiki detox data

Author: michaelgsharp

src/Microsoft.ML.Transforms/Text/WrappedTextTransformers.cs

@@ -79,7 +79,7 @@ public class Options
 
             public Options()
             {
-                NgramLength = 1;
+                NgramLength = 2;
                 SkipLength = NgramExtractingEstimator.Defaults.SkipLength;
                 UseAllLengths = NgramExtractingEstimator.Defaults.UseAllLengths;
                 MaximumNgramsCount = new int[] { NgramExtractingEstimator.Defaults.MaximumNgramsCount };

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -23696,6 +23696,7 @@
           "Default": {
             "Name": "NGram",
             "Settings": {
+              "NgramLength": 2,
               "MaxNumTerms": [
                 10000000
               ]

test/BaselineOutput/Common/Text/featurized.tsv

@@ -103,7 +103,7 @@ CalibratorTestData GetCalibratorTestData()
             var transformer = pipeline.Fit(data);
             var scoredData = transformer.Transform(data);
             var scoredDataPreview = scoredData.Preview();
-            Assert.True(scoredDataPreview.ColumnView.Length == 5);
+            Assert.True(scoredDataPreview.ColumnView.Length == 6);
 
             return new CalibratorTestData
             {
@@ -128,11 +128,11 @@ private void CheckValidCalibratedData(IDataView scoredData, ITransformer transfo
 
             var calibratedData = transformer.Transform(scoredData).Preview();
 
-            Assert.True(calibratedData.ColumnView.Length == 6);
+            Assert.True(calibratedData.ColumnView.Length == 7);
 
             for (int i = 0; i < 10; i++)
             {
-                var probability = calibratedData.RowView[i].Values[5];
+                var probability = calibratedData.RowView[i].Values[6];
                 Assert.InRange((float)probability.Value, 0, 1);
             }
         }

test/Microsoft.ML.Tests/TrainerEstimators/CalibratorEstimators.cs

@@ -101,14 +101,14 @@ public void TestLRWithStats()
 
                 Assert.NotNull(biasStats);
 
-                CompareNumbersWithTolerance(biasStats.StandardError, 0.25, digitsOfPrecision: 2);
-                CompareNumbersWithTolerance(biasStats.ZScore, 7.97, digitsOfPrecision: 2);
+                CompareNumbersWithTolerance(biasStats.StandardError, 0.24, digitsOfPrecision: 2);
+                CompareNumbersWithTolerance(biasStats.ZScore, 8.32, digitsOfPrecision: 2);
 
                 var scoredData = transformer.Transform(dataView);
 
                 var coefficients = stats.GetWeightsCoefficientStatistics(100);
 
-                Assert.Equal(18, coefficients.Length);
+                Assert.Equal(17, coefficients.Length);
 
                 foreach (var coefficient in coefficients)
                     Assert.True(coefficient.StandardError < 1.0);

test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs

@@ -197,7 +197,8 @@ public void TestEstimatorLdSvmTrainer()
                         Columns = new[]
                         {
                             new TextLoader.Column("Label", DataKind.Boolean, 0),
-                            new TextLoader.Column("SentimentText", DataKind.String, 1)
+                            new TextLoader.Column("SentimentText", DataKind.String, 1),
+                            new TextLoader.Column("LoggedIn", DataKind.Boolean, 2)
                         }
                     }).Load(GetDataPath(TestDatasets.Sentiment.trainFilename));
 
@@ -214,7 +215,8 @@ public void TestEstimatorLdSvmTrainer()
         private (IEstimator<ITransformer>, IDataView) GetOneHotBinaryClassificationPipeline()
         {
             var (pipeline, data) = GetBinaryClassificationPipeline();
-            var oneHotPipeline = pipeline.Append(ML.Transforms.Categorical.OneHotEncoding("Features"));
+            var oneHotPipeline = pipeline.Append(ML.Transforms.Categorical.OneHotEncoding("LoggedIn"));
+            oneHotPipeline.Append(ML.Transforms.Concatenate("Features", "Features", "LoggedIn"));
 
             return (oneHotPipeline, data);
         }

test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs

@@ -221,14 +221,14 @@ public void TextFeaturizerWithL2NormTest()
             var prediction = engine.Predict(data[0]);
             Assert.Equal(data[0].A, string.Join(" ", prediction.OutputTokens));
             var exp1 = 0.333333343f;
-            var exp2 = 0.707106769f;
-            var expected = new float[] { exp1, exp1, exp1, exp1, exp1, exp1, exp1, exp1, exp1, exp2, exp2 };
+            var exp2 = 0.577350259f;
+            var expected = new float[] { exp1, exp1, exp1, exp1, exp1, exp1, exp1, exp1, exp1, exp2, exp2, exp2 };
             Assert.Equal(expected, prediction.Features);
 
             prediction = engine.Predict(data[1]);
             exp1 = 0.4472136f;
             Assert.Equal(data[1].A, string.Join(" ", prediction.OutputTokens));
-            expected = new float[] { exp1, 0.0f, 0.0f, 0.0f, 0.0f, exp1, exp1, exp1, exp1, 0.0f, 1.0f };
+            expected = new float[] { exp1, 0.0f, 0.0f, 0.0f, 0.0f, exp1, exp1, exp1, exp1, 0.0f, 0.0f, 1.0f };
             Assert.Equal(expected, prediction.Features);
         }
 

test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs

@@ -10,5 +10,11 @@
     "ItemKind": "TX",
     "Purpose": "TextFeature",
     "ColumnRangeSelector": "1"
+  },
+  {
+    "SuggestedName": "LoggedIn",
+    "ItemKind": "Bool",
+    "Purpose": "Categorical",
+    "ColumnRangeSelector": "2"
   }
-]
+]