Merge master branch in to temp/docs branch.

build.cmd

@@ -21,6 +21,7 @@ set BoostRoot=%DependenciesDir%BoostDbg3.7
 set PythonVersion=3.7
 set PythonTag=cp37
 set RunTests=False
+set InstallPythonPackages=False
 set RunExtendedTests=False
 set BuildDotNetBridgeOnly=False
 set SkipDotNetBridge=False
@@ -33,6 +34,11 @@ if /i [%1] == [--configuration] (
 )
 if /i [%1] == [--runTests]     (
     set RunTests=True
+    set InstallPythonPackages=True
+    shift && goto :Arg_Loop
+)
+if /i [%1] == [--installPythonPackages]     (
+    set InstallPythonPackages=True
     shift && goto :Arg_Loop
 )
 if /i [%1] == [--includeExtendedTests]     (
@@ -58,6 +64,7 @@ echo ""
 echo "Options:"
 echo "  --configuration <Configuration>   Build Configuration (DbgWinPy3.7,DbgWinPy3.6,DbgWinPy3.5,DbgWinPy2.7,RlsWinPy3.7,RlsWinPy3.6,RlsWinPy3.5,RlsWinPy2.7)"
 echo "  --runTests                        Run tests after build"
+echo "  --installPythonPackages           Install python packages after build"
 echo "  --includeExtendedTests            Include the extended tests if the tests are run"
 echo "  --buildDotNetBridgeOnly           Build only DotNetBridge"
 echo "  --skipDotNetBridge                Build everything except DotNetBridge"
@@ -157,7 +164,7 @@ if /i [%1] == [DbgWinPy2.7]     (
 :Build
 :: Install dotnet SDK version, see https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script
 echo Installing dotnet SDK ... 
-powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Version 2.1.200 -InstallDir ./cli"
+powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Version 2.1.701 -InstallDir ./cli"
 
 set _dotnetRoot=%__currentScriptDir%cli
 
@@ -328,6 +335,19 @@ md "%__currentScriptDir%target"
 copy "%__currentScriptDir%src\python\dist\%WheelFile%" "%__currentScriptDir%target\%WheelFile%"
 echo Python package successfully created: %__currentScriptDir%target\%WheelFile%
 
+if "%InstallPythonPackages%" == "True" (
+    echo ""
+    echo "#################################"
+    echo "Installing python packages ... "
+    echo "#################################"
+    call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
+    if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq )
+    :: Run azureml-dataprep tests only in pyhon 3.7 as its an optional dependency
+    if %PythonVersion% == 3.7 ( call "%PythonExe%" -m pip install --upgrade azureml-dataprep )
+    call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%"
+    call "%PythonExe%" -m pip install "scikit-learn==0.19.2"
+)
+
 if "%RunTests%" == "False" ( 
     goto :Exit_Success
 )
@@ -337,11 +357,6 @@ echo ""
 echo "#################################"
 echo "Running tests ... "
 echo "#################################"
-call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
-if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq )
-call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%"
-call "%PythonExe%" -m pip install "scikit-learn==0.19.2"
-
 set PackagePath=%PythonRoot%\Lib\site-packages\nimbusml
 set TestsPath1=%PackagePath%\tests
 set TestsPath2=%__currentScriptDir%src\python\tests

build.sh

@@ -16,6 +16,7 @@ usage()
     echo "Options:"
     echo "  --configuration <Configuration>   Build Configuration (DbgLinPy3.7,DbgLinPy3.6,DbgLinPy3.5,DbgLinPy2.7,RlsLinPy3.7,RlsLinPy3.6,RlsLinPy3.5,RlsLinPy2.7,DbgMacPy3.7,DbgMacPy3.6,DbgMacPy3.5,DbgMacPy2.7,RlsMacPy3.7,RlsMacPy3.6,RlsMacPy3.5,RlsMacPy2.7)"
     echo "  --runTests                        Run tests after build"
+    echo "  --installPythonPackages           Install python packages after build"
     echo "  --runTestsOnly                    Run tests on a wheel file in default build location (<repo>/target/)"
     echo "  --includeExtendedTests            Include the extended tests if the tests are run"
     echo "  --buildNativeBridgeOnly           Build only the native bridge code"
@@ -31,6 +32,7 @@ else
     __configuration=DbgLinPy3.7
 fi
 __runTests=false
+__installPythonPackages=false
 __runExtendedTests=false
 __buildNativeBridge=true
 __buildDotNetBridge=true
@@ -48,6 +50,10 @@ while [ "$1" != "" ]; do
             ;;
         --runtests)
             __runTests=true
+            __installPythonPackages=true
+            ;;
+        --installPythonPackages)
+            __installPythonPackages=true
             ;;
         --includeextendedtests)
             __runExtendedTests=true
@@ -56,6 +62,7 @@ while [ "$1" != "" ]; do
             __buildNativeBridge=false
             __buildDotNetBridge=false
             __runTests=true
+            __installPythonPackages=true
             ;;
         --buildnativebridgeonly)
             __buildDotNetBridge=false
@@ -166,7 +173,7 @@ if [ ${__buildDotNetBridge} = true ]
 then 
     # Install dotnet SDK version, see https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script
     echo "Installing dotnet SDK ... "
-    curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin -Version 2.1.200 -InstallDir ./cli
+    curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin -Version 2.1.701 -InstallDir ./cli
 
     # Build managed code
     echo "Building managed code ... "
@@ -199,6 +206,7 @@ then
     cp  "${BuildOutputDir}/${__configuration}"/DotNetBridge.dll "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
     cp  "${BuildOutputDir}/${__configuration}"/pybridge.so "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
 
+	# ls "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/
     if [ ${PythonVersion} = 2.7 ]
     then
         cp  "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/*.dll "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
@@ -246,11 +254,11 @@ then
     echo Python package successfully created: ${__currentScriptDir}/target/${WheelFile}
 fi
 
-if [ ${__runTests} = true ]
-then 
+if [ ${__installPythonPackages} = true ]
+then
     echo ""
     echo "#################################"
-    echo "Running tests ... "
+    echo "Installing Python packages ... "
     echo "#################################"
     Wheel=${__currentScriptDir}/target/nimbusml-${ProductVersion}-${PythonTag}-none-${PlatName}.whl
     if [ ! -f ${Wheel} ]
@@ -266,10 +274,20 @@ then
     elif [ ${PythonVersion} = 3.6 ] && [ "$(uname -s)" = "Darwin" ]
     then
         "${PythonExe}" -m pip install --upgrade pytest-remotedata
-    fi
+    elif [ ${PythonVersion} = 3.7 ]
+    then
+        "${PythonExe}" -m pip install --upgrade azureml-dataprep
+	fi
     "${PythonExe}" -m pip install --upgrade "${Wheel}"
     "${PythonExe}" -m pip install "scikit-learn==0.19.2"
+fi
 
+if [ ${__runTests} = true ]
+then 
+    echo ""
+    echo "#################################"
+    echo "Running tests ... "
+    echo "#################################"
     PackagePath=${PythonRoot}/lib/python${PythonVersion}/site-packages/nimbusml
     TestsPath1=${PackagePath}/tests
     TestsPath2=${__currentScriptDir}/src/python/tests

build/libs_linux.txt

@@ -1,12 +1,17 @@
 Newtonsoft.Json.dll
 libCpuMathNative.so
-libFactorizationMachineNative.so
 libFastTreeNative.so
 libLdaNative.so
 libMklImports.so
+libMklProxyNative.so
 libSymSgdNative.so
 lib_lightgbm.so
 libtensorflow.so
-libtensorflow_framework.so
+libtensorflow_framework.so.1
+libonnxruntime.so
 System.Drawing.Common.dll
+TensorFlow.NET.dll
+NumSharp.Core.dll
+Microsoft.DataPrep.dll
+Microsoft.DPrep.*
 Microsoft.ML.*

build/libs_mac.txt

@@ -1,12 +1,17 @@
 Newtonsoft.Json.dll
 libCpuMathNative.dylib
-libFactorizationMachineNative.dylib
 libFastTreeNative.dylib
 libLdaNative.dylib
 libMklImports.dylib
+libMklProxyNative.dylib
 libSymSgdNative.dylib
 lib_lightgbm.dylib
 libtensorflow.dylib
-libtensorflow_framework.dylib
+libonnxruntime.dylib
+libtensorflow_framework.1.dylib
 System.Drawing.Common.dll
+TensorFlow.NET.dll
+NumSharp.Core.dll
+Microsoft.DataPrep.dll
+Microsoft.DPrep.*
 Microsoft.ML.*

build/libs_win.txt

@@ -1,13 +1,17 @@
 Google.Protobuf.dll
 Newtonsoft.Json.dll
 CpuMathNative.dll
-FactorizationMachineNative.dll
 FastTreeNative.dll
 LdaNative.dll
 lib_lightgbm.dll
 libiomp5md.dll
 MklImports.dll
+MklProxyNative.dll
 SymSgdNative.dll
 tensorflow.dll
+TensorFlow.NET.dll
+NumSharp.Core.dll
 System.Drawing.Common.dll
+Microsoft.DataPrep.dll
+Microsoft.DPrep.*
 Microsoft.ML.*

build/sign.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net461</TargetFramework>
     <OutDir Condition="'$(OutDir)' == ''">../x64/</OutDir>
   </PropertyGroup>
 
@@ -15,7 +15,7 @@
 
   <ItemGroup>
       <FilesToSign Include="$(OutDir)**/DotNetBridge.dll">
-        <Authenticode>Microsoft</Authenticode>
+        <Authenticode>Microsoft400</Authenticode>
       </FilesToSign>
   </ItemGroup>
 

docs/release-notes/release-1.3.0.md

@@ -0,0 +1,101 @@
+# [NimbusML](https://docs.microsoft.com/en-us/nimbusml/overview) 1.3.0
+
+## **New Features**
+
+- **Save/Restore model when pickling Pipeline**
+
+   [PR#189](https://github.com/microsoft/NimbusML/pull/189) Save and restore
+   the underlying model file when pickling a nimbusml Pipeline.
+
+- **Feature Contributions**
+
+  [PR#196](https://github.com/microsoft/NimbusML/pull/196) Added support for
+  observation level feature contributions. Exposes an API
+  `Pipeline.get_feature_contributions()` that provides scores for how much
+  each feature influenced a particular prediction, thereby allowing users to
+  inspect which features were most important in making the prediction.
+
+- **Add `classes_` to Pipeline**
+
+   [PR#200](https://github.com/microsoft/NimbusML/pull/200) Add a `classes_`
+   attribute to a Pipeline and/or predictor instance when calling
+   `Pipeline.predict_proba()`.
+
+- **Automatically Convert Input Of Handler, Filter and Indicator**
+
+   [PR#204](https://github.com/microsoft/NimbusML/pull/204) Update Handler,
+   Filter, and Indicator to automatically convert the input columns to float
+   before performing the transform.
+
+- **Combine Models**
+
+   [PR#208](https://github.com/microsoft/NimbusML/pull/208) Add support for
+   combining models from transforms, predictors and pipelines in to one model.
+
+- **Azureml-Dataprep integration**
+
+   [PR#181](https://github.com/microsoft/NimbusML/pull/181) Added support for
+   dataflow objects as a datasource for pipeline training/testing.
+
+- **Linear SVM Binary Classifier**
+  [PR#180](https://github.com/microsoft/NimbusML/pull/180) Added
+  `LinearSvmBinaryClassifier` in `nimbusml.linear_model`.
+
+- **Ensemble Training**
+
+  [PR#207](https://github.com/microsoft/NimbusML/pull/207) Enabled training of
+  Ensemble models by adding `nimbusml.ensemble.EnsembleRegressor` and 
+  `nimbusml.ensemble.EnsembleClassifier`. Added components needed
+  to create ensemble models as new modules in `nimbusml.ensemble`. These
+  components are passed as arguments to the ensemble trainers.
+  - Preprocessing components for training multiple models to ensemble in 
+  `nimbusml.ensemble.subset_selector` and  `nimbusml.ensemble.feature_selector`.
+  - Post training components to create the ensemble from the trained models in
+  `nimbusml.ensemble.sub_model_selector` and `nimbusml.ensemble.output_combiner`.
+
+## **Bug Fixes**
+
+- **Fixed memory leak**
+
+   The [PR#184](https://github.com/microsoft/NimbusML/pull/184) fixed potentially
+   large memory leak when transforming pandas dataframe.
+
+- **Remove Stored References To `X` and `y`**
+
+   [PR#195](https://github.com/microsoft/NimbusML/pull/195) Remove the stored
+   references to X and y in BasePredictor.
+
+- **Fixed Explicit `evaltype`**
+
+   The [issue](https://github.com/microsoft/NimbusML/issues/193) where passing
+   in an explicit `evaltype` to `_predict` in a NimbusML pipeline causes errors
+   has been fixed with this
+   [commit](https://github.com/microsoft/NimbusML/commit/1f97c9ef55f5e257f989db5f375cca5c55880258).
+
+## **Breaking Changes**
+
+None.
+
+## **Enhancements**
+
+None.
+
+## **Documentation and Samples**
+
+[Feature Contributions Example](https://github.com/microsoft/NimbusML/blob/master/src/python/nimbusml/examples/PipelineWithFeatureContributions.py)
+
+LinearSvmBinaryClassifier Examples:
+- [FileDataStream example](https://github.com/microsoft/NimbusML/blob/master/src/python/nimbusml/examples/LinearSvmBinaryClassifier.py)
+- [DataFrame example](https://github.com/microsoft/NimbusML/blob/master/src/python/nimbusml/examples/examples_from_dataframe/LinearSvmBinaryClassifier_df.py)
+
+EnsembleClassifier Examples:
+- [FileDataStream example](https://github.com/microsoft/NimbusML/blob/master/src/python/nimbusml/examples/EnsembleClassifier.py)
+- [DataFrame example](https://github.com/microsoft/NimbusML/blob/master/src/python/nimbusml/examples/examples_from_dataframe/EnsembleClassifier_iris_df.py)
+
+EnsembleRegressor Examples:
+- [FileDataStream example](https://github.com/microsoft/NimbusML/blob/master/src/python/nimbusml/examples/EnsembleRegressor.py)
+- [DataFrame example](https://github.com/microsoft/NimbusML/blob/master/src/python/nimbusml/examples/examples_from_dataframe/EnsembleRegressor_airquality_df.py)
+
+## **Remarks**
+
+None.

release-next.md

@@ -18,7 +18,7 @@ None.
 
 ## **Documentation and Samples**
 
-None.
+None. 
 
 ## **Remarks**
 

src/DotNetBridge/Bridge.cs

@@ -314,7 +314,7 @@ private static unsafe int GenericExec(EnvironmentBlock* penv, sbyte* psz, int cd
             env.ComponentCatalog.RegisterAssembly(typeof(CategoricalCatalog).Assembly); // ML.Transforms
             env.ComponentCatalog.RegisterAssembly(typeof(FastTreeRegressionTrainer).Assembly); // ML.FastTree
 
-            //env.ComponentCatalog.RegisterAssembly(typeof(EnsembleModelParameters).Assembly); // ML.Ensemble
+            env.ComponentCatalog.RegisterAssembly(typeof(EnsembleModelParameters).Assembly); // ML.Ensemble
             env.ComponentCatalog.RegisterAssembly(typeof(KMeansModelParameters).Assembly); // ML.KMeansClustering
             env.ComponentCatalog.RegisterAssembly(typeof(PcaModelParameters).Assembly); // ML.PCA
             env.ComponentCatalog.RegisterAssembly(typeof(CVSplit).Assembly); // ML.EntryPoints

src/DotNetBridge/DotNetBridge.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Platforms>x64</Platforms>
     <DefineConstants>CORECLR</DefineConstants>
@@ -31,16 +31,19 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.ML" Version="1.2.0" />
-    <PackageReference Include="Microsoft.ML.CpuMath" Version="1.2.0" />    
-    <PackageReference Include="Microsoft.ML.EntryPoints" Version="0.14.0" />
-    <PackageReference Include="Microsoft.ML.Mkl.Components" Version="1.2.0" />
-    <PackageReference Include="Microsoft.ML.Mkl.Redist" Version="1.2.0" />
-    <PackageReference Include="Microsoft.ML.ImageAnalytics" Version="1.2.0" />
-    <PackageReference Include="Microsoft.ML.LightGBM" Version="1.2.0" />
-    <PackageReference Include="Microsoft.ML.OnnxTransformer" Version="1.2.0" />
-    <PackageReference Include="Microsoft.ML.TensorFlow" Version="1.2.0" />
-    <PackageReference Include="Microsoft.ML.Ensemble" Version="0.14.0" />
-    <PackageReference Include="Microsoft.ML.TimeSeries" Version="1.2.0" />
+    <PackageReference Include="Microsoft.ML" Version="1.3.1" />
+    <PackageReference Include="Microsoft.ML.CpuMath" Version="1.3.1" />    
+    <PackageReference Include="Microsoft.ML.EntryPoints" Version="0.15.1" />
+    <PackageReference Include="Microsoft.ML.Mkl.Components" Version="1.3.1" />
+    <PackageReference Include="Microsoft.ML.ImageAnalytics" Version="1.3.1" />
+    <PackageReference Include="Microsoft.ML.LightGBM" Version="1.3.1" />
+    <PackageReference Include="Microsoft.ML.OnnxTransformer" Version="1.3.1" />
+    <PackageReference Include="Microsoft.ML.TensorFlow" Version="1.3.1" />
+    <PackageReference Include="Microsoft.ML.Dnn" Version="0.15.1" />
+    <PackageReference Include="Microsoft.ML.Ensemble" Version="0.15.1" />
+    <PackageReference Include="Microsoft.ML.TimeSeries" Version="1.3.1" />
+    <PackageReference Include="Microsoft.DataPrep" Version="0.0.1.5-preview" />
+    <PackageReference Include="TensorFlow.NET" Version="0.10.10" />
+    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="1.14.0" />
   </ItemGroup>
 </Project>