Combine multiple tree ensemble models into a single tree ensemble (#364)

* Add a way to create a single tree ensemble model from multiple tree ensemble models.

* Address PR comments, and fix bugs in serializing/deserializing RegressionTrees.

* Address PR comments.

Author: yaeldMS

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -74,6 +74,8 @@ public interface IModelCombiner<TModel, TPredictor>
         TPredictor CombineModels(IEnumerable<TModel> models);
     }
 
+    public delegate void SignatureModelCombiner(PredictionKind kind);
+
     /// <summary>
     /// Weakly typed interface for a trainer "session" that produces a predictor.
     /// </summary>

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -57,6 +57,7 @@
     <Compile Include="TreeEnsemble\Ensemble.cs" />
     <Compile Include="TreeEnsemble\QuantileRegressionTree.cs" />
     <Compile Include="TreeEnsemble\RegressionTree.cs" />
+    <Compile Include="TreeEnsemble\TreeEnsembleCombiner.cs" />
     <Compile Include="Training\Applications\GradientWrappers.cs" />
     <Compile Include="Training\Applications\ObjectiveFunction.cs" />
     <Compile Include="Training\BaggingProvider.cs" />

src/Microsoft.ML.FastTree/TreeEnsemble/RegressionTree.cs

@@ -105,22 +105,21 @@ public RegressionTree(byte[] buffer, ref int position)
             LteChild = buffer.ToIntArray(ref position);
             GtChild = buffer.ToIntArray(ref position);
             SplitFeatures = buffer.ToIntArray(ref position);
-            int[] categoricalNodeIndices = buffer.ToIntArray(ref position);
-            CategoricalSplit = GetCategoricalSplitFromIndices(categoricalNodeIndices);
-            if (categoricalNodeIndices?.Length > 0)
+            byte[] categoricalSplitAsBytes = buffer.ToByteArray(ref position);
+            CategoricalSplit = categoricalSplitAsBytes.Select(b => b > 0).ToArray();
+            if (CategoricalSplit.Any(b => b))
             {
                 CategoricalSplitFeatures = new int[NumNodes][];
                 CategoricalSplitFeatureRanges = new int[NumNodes][];
-                foreach (var index in categoricalNodeIndices)
+                for (int index = 0; index < NumNodes; index++)
                 {
-                    Contracts.Assert(CategoricalSplit[index]);
-
                     CategoricalSplitFeatures[index] = buffer.ToIntArray(ref position);
-                    CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position, 2);
+                    CategoricalSplitFeatureRanges[index] = buffer.ToIntArray(ref position);
                 }
             }
 
             Thresholds = buffer.ToUIntArray(ref position);
+            RawThresholds = buffer.ToFloatArray(ref position);
             _splitGain = buffer.ToDoubleArray(ref position);
             _gainPValue = buffer.ToDoubleArray(ref position);
             _previousLeafValue = buffer.ToDoubleArray(ref position);
@@ -144,6 +143,23 @@ private bool[] GetCategoricalSplitFromIndices(int[] indices)
             return categoricalSplit;
         }
 
+        private bool[] GetCategoricalSplitFromBytes(byte[] indices)
+        {
+            bool[] categoricalSplit = new bool[NumNodes];
+            if (indices == null)
+                return categoricalSplit;
+
+            Contracts.Assert(indices.Length <= NumNodes);
+
+            foreach (int index in indices)
+            {
+                Contracts.Assert(index >= 0 && index < NumNodes);
+                categoricalSplit[index] = true;
+            }
+
+            return categoricalSplit;
+        }
+
         /// <summary>
         /// Create a Regression Tree object from raw tree contents.
         /// </summary>
@@ -192,7 +208,7 @@ internal RegressionTree(int[] splitFeatures, Double[] splitGain, Double[] gainPV
             LeafValues = leafValues;
             CategoricalSplitFeatures = categoricalSplitFeatures;
             CategoricalSplitFeatureRanges = new int[CategoricalSplitFeatures.Length][];
-            for(int i= 0; i < CategoricalSplitFeatures.Length; ++i)
+            for (int i = 0; i < CategoricalSplitFeatures.Length; ++i)
             {
                 if (CategoricalSplitFeatures[i] != null && CategoricalSplitFeatures[i].Length > 0)
                 {
@@ -500,6 +516,7 @@ public virtual int SizeInBytes()
                 NumNodes * sizeof(int) +
                 CategoricalSplit.Length * sizeof(bool) +
                 Thresholds.SizeInBytes() +
+                RawThresholds.SizeInBytes() +
                 _splitGain.SizeInBytes() +
                 _gainPValue.SizeInBytes() +
                 _previousLeafValue.SizeInBytes() +
@@ -514,22 +531,22 @@ public virtual void ToByteArray(byte[] buffer, ref int position)
             LteChild.ToByteArray(buffer, ref position);
             GtChild.ToByteArray(buffer, ref position);
             SplitFeatures.ToByteArray(buffer, ref position);
+            CategoricalSplit.Length.ToByteArray(buffer, ref position);
             foreach (var split in CategoricalSplit)
                 Convert.ToByte(split).ToByteArray(buffer, ref position);
 
             if (CategoricalSplitFeatures != null)
             {
-                foreach (var splits in CategoricalSplitFeatures)
-                    splits.ToByteArray(buffer, ref position);
-            }
-
-            if (CategoricalSplitFeatureRanges != null)
-            {
-                foreach (var ranges in CategoricalSplitFeatureRanges)
-                    ranges.ToByteArray(buffer, ref position);
+                Contracts.AssertValue(CategoricalSplitFeatureRanges);
+                for (int i = 0; i < CategoricalSplitFeatures.Length; i++)
+                {
+                    CategoricalSplitFeatures[i].ToByteArray(buffer, ref position);
+                    CategoricalSplitFeatureRanges[i].ToByteArray(buffer, ref position);
+                }
             }
 
             Thresholds.ToByteArray(buffer, ref position);
+            RawThresholds.ToByteArray(buffer, ref position);
             _splitGain.ToByteArray(buffer, ref position);
             _gainPValue.ToByteArray(buffer, ref position);
             _previousLeafValue.ToByteArray(buffer, ref position);

src/Microsoft.ML.FastTree/TreeEnsemble/TreeEnsembleCombiner.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.FastTree.Internal;
+using Microsoft.ML.Runtime.Internal.Calibration;
+
+[assembly: LoadableClass(typeof(TreeEnsembleCombiner), null, typeof(SignatureModelCombiner), "Fast Tree Model Combiner", "FastTreeCombiner")]
+
+namespace Microsoft.ML.Runtime.FastTree.Internal
+{
+    public sealed class TreeEnsembleCombiner : IModelCombiner<IPredictorProducing<float>, IPredictorProducing<float>>
+    {
+        private readonly IHost _host;
+        private readonly PredictionKind _kind;
+
+        public TreeEnsembleCombiner(IHostEnvironment env, PredictionKind kind)
+        {
+            _host = env.Register("TreeEnsembleCombiner");
+            switch (kind)
+            {
+                case PredictionKind.BinaryClassification:
+                case PredictionKind.Regression:
+                case PredictionKind.Ranking:
+                    _kind = kind;
+                    break;
+                default:
+                    throw _host.ExceptUserArg(nameof(kind), $"Tree ensembles can be either of type {nameof(PredictionKind.BinaryClassification)}, " +
+                        $"{nameof(PredictionKind.Regression)} or {nameof(PredictionKind.Ranking)}");
+            }
+        }
+
+        public IPredictorProducing<float> CombineModels(IEnumerable<IPredictorProducing<float>> models)
+        {
+            _host.CheckValue(models, nameof(models));
+
+            var ensemble = new Ensemble();
+            int modelCount = 0;
+            int featureCount = -1;
+            bool binaryClassifier = false;
+            foreach (var model in models)
+            {
+                modelCount++;
+
+                var predictor = model;
+                _host.CheckValue(predictor, nameof(models), "One of the models is null");
+
+                var calibrated = predictor as CalibratedPredictorBase;
+                double paramA = 1;
+                if (calibrated != null)
+                {
+                    _host.Check(calibrated.Calibrator is PlattCalibrator,
+                        "Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
+                    predictor = calibrated.SubPredictor;
+                    paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
+                }
+                var tree = predictor as FastTreePredictionWrapper;
+                if (tree == null)
+                    throw _host.Except("Model is not a tree ensemble");
+                foreach (var t in tree.TrainedEnsemble.Trees)
+                {
+                    var bytes = new byte[t.SizeInBytes()];
+                    int position = -1;
+                    t.ToByteArray(bytes, ref position);
+                    position = -1;
+                    var tNew = new RegressionTree(bytes, ref position);
+                    if (paramA != 1)
+                    {
+                        for (int i = 0; i < tNew.NumLeaves; i++)
+                            tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
+                    }
+                    ensemble.AddTree(tNew);
+                }
+
+                if (modelCount == 1)
+                {
+                    binaryClassifier = calibrated != null;
+                    featureCount = tree.InputType.ValueCount;
+                }
+                else
+                {
+                    _host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
+                    _host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
+                }
+            }
+
+            var scale = 1 / (double)modelCount;
+
+            foreach (var t in ensemble.Trees)
+            {
+                for (int i = 0; i < t.NumLeaves; i++)
+                    t.SetOutput(i, t.LeafValues[i] * scale);
+            }
+
+            switch (_kind)
+            {
+                case PredictionKind.BinaryClassification:
+                    if (!binaryClassifier)
+                        return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
+
+                    var cali = new PlattCalibrator(_host, -1, 0);
+                    return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
+                case PredictionKind.Regression:
+                    return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
+                case PredictionKind.Ranking:
+                    return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
+                default:
+                    _host.Assert(false);
+                    throw _host.ExceptNotSupp();
+            }
+        }
+    }
+}

src/Microsoft.ML.FastTree/Utils/ToByteArrayExtensions.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Linq;
 using System.Text;
+using Microsoft.ML.Runtime.Internal.Utilities;
 
 namespace Microsoft.ML.Runtime.FastTree.Internal
 {
@@ -290,7 +291,7 @@ public static string ToString(this byte[] buffer, ref int position)
 
         public static int SizeInBytes(this byte[] a)
         {
-            return sizeof(int) + a.Length * sizeof(byte);
+            return sizeof(int) + Utils.Size(a) * sizeof(byte);
         }
 
         public static void ToByteArray(this byte[] a, byte[] buffer, ref int position)
@@ -314,7 +315,7 @@ public static byte[] ToByteArray(this byte[] buffer, ref int position)
 
         public static int SizeInBytes(this short[] a)
         {
-            return sizeof(int) + a.Length * sizeof(short);
+            return sizeof(int) + Utils.Size(a) * sizeof(short);
         }
 
         public unsafe static void ToByteArray(this short[] a, byte[] buffer, ref int position)
@@ -353,7 +354,7 @@ public unsafe static short[] ToShortArray(this byte[] buffer, ref int position)
 
         public static int SizeInBytes(this ushort[] a)
         {
-            return sizeof(int) + a.Length * sizeof(ushort);
+            return sizeof(int) + Utils.Size(a) * sizeof(ushort);
         }
 
         public unsafe static void ToByteArray(this ushort[] a, byte[] buffer, ref int position)
@@ -392,12 +393,12 @@ public unsafe static ushort[] ToUShortArray(this byte[] buffer, ref int position
 
         public static int SizeInBytes(this int[] array)
         {
-            return sizeof(int) + array.Length * sizeof(int);
+            return sizeof(int) + Utils.Size(array) * sizeof(int);
         }
 
         public unsafe static void ToByteArray(this int[] a, byte[] buffer, ref int position)
         {
-            int length = a.Length;
+            int length = Utils.Size(a);
             length.ToByteArray(buffer, ref position);
 
             fixed (byte* tmpBuffer = buffer)
@@ -415,6 +416,9 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position)
 
         public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int length)
         {
+            if (length == 0)
+                return null;
+
             int[] a = new int[length];
 
             fixed (byte* tmpBuffer = buffer)
@@ -433,7 +437,7 @@ public unsafe static int[] ToIntArray(this byte[] buffer, ref int position, int
 
         public static int SizeInBytes(this uint[] array)
         {
-            return sizeof(int) + array.Length * sizeof(uint);
+            return sizeof(int) + Utils.Size(array) * sizeof(uint);
         }
 
         public unsafe static void ToByteArray(this uint[] a, byte[] buffer, ref int position)
@@ -472,7 +476,7 @@ public unsafe static uint[] ToUIntArray(this byte[] buffer, ref int position)
 
         public static int SizeInBytes(this long[] array)
         {
-            return sizeof(int) + array.Length * sizeof(long);
+            return sizeof(int) + Utils.Size(array) * sizeof(long);
         }
 
         public unsafe static void ToByteArray(this long[] a, byte[] buffer, ref int position)
@@ -511,7 +515,7 @@ public unsafe static long[] ToLongArray(this byte[] buffer, ref int position)
 
         public static int SizeInBytes(this ulong[] array)
         {
-            return sizeof(int) + array.Length * sizeof(ulong);
+            return sizeof(int) + Utils.Size(array) * sizeof(ulong);
         }
 
         public unsafe static void ToByteArray(this ulong[] a, byte[] buffer, ref int position)
@@ -550,7 +554,7 @@ public unsafe static ulong[] ToULongArray(this byte[] buffer, ref int position)
 
         public static int SizeInBytes(this MD5Hash[] array)
         {
-            return sizeof(int) + array.Length * MD5Hash.SizeInBytes();
+            return sizeof(int) + Utils.Size(array) * MD5Hash.SizeInBytes();
         }
 
         public static void ToByteArray(this MD5Hash[] a, byte[] buffer, ref int position)
@@ -577,7 +581,7 @@ public unsafe static MD5Hash[] ToUInt128Array(this byte[] buffer, ref int positi
 
         public static int SizeInBytes(this float[] array)
         {
-            return sizeof(int) + array.Length * sizeof(float);
+            return sizeof(int) + Utils.Size(array) * sizeof(float);
         }
 
         public unsafe static void ToByteArray(this float[] a, byte[] buffer, ref int position)
@@ -616,7 +620,7 @@ public unsafe static float[] ToFloatArray(this byte[] buffer, ref int position)
 
         public static int SizeInBytes(this double[] array)
         {
-            return sizeof(int) + array.Length * sizeof(double);
+            return sizeof(int) + Utils.Size(array) * sizeof(double);
         }
 
         public unsafe static void ToByteArray(this double[] a, byte[] buffer, ref int position)
@@ -655,6 +659,8 @@ public unsafe static double[] ToDoubleArray(this byte[] buffer, ref int position
 
         public static int SizeInBytes(this double[][] array)
         {
+            if (Utils.Size(array) == 0)
+                return sizeof(int);
             return sizeof(int) + array.Sum(x => x.SizeInBytes());
         }
 
@@ -683,7 +689,7 @@ public static double[][] ToDoubleJaggedArray(this byte[] buffer, ref int positio
         public static long SizeInBytes(this string[] array)
         {
             long length = sizeof(int);
-            for (int i = 0; i < array.Length; ++i)
+            for (int i = 0; i < Utils.Size(array); ++i)
             {
                 length += array[i].SizeInBytes();
             }
@@ -692,8 +698,8 @@ public static long SizeInBytes(this string[] array)
 
         public static void ToByteArray(this string[] a, byte[] buffer, ref int position)
         {
-            a.Length.ToByteArray(buffer, ref position);
-            for (int i = 0; i < a.Length; ++i)
+            Utils.Size(a).ToByteArray(buffer, ref position);
+            for (int i = 0; i < Utils.Size(a); ++i)
             {
                 a[i].ToByteArray(buffer, ref position);
             }